diff --git a/.codeclimate.yml b/.codeclimate.yml index 81b8bd3c4c..b5165b6887 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -1,8 +1,16 @@ # Save as .codeclimate.yml (note leading .) in project root directory +version: "2" languages: Ruby: true JavaScript: true PHP: true +checks: + file-lines: + config: + threshold: 500 + method-lines: + config: + threshold: 50 exclude_paths: - "public/vendor/*" - "test/*" \ No newline at end of file diff --git a/.eslintrc b/.eslintrc index ef9f48dedc..9414543335 100644 --- a/.eslintrc +++ b/.eslintrc @@ -36,6 +36,10 @@ "no-restricted-globals": "off", "function-paren-newline": "off", "import/no-unresolved": "error", + "quotes": ["error", "single", { + "avoidEscape": true, + "allowTemplateLiterals": true + }], // ES6 "prefer-rest-params": "off", diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index d025ff3ff5..49f0a84909 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -10,9 +10,9 @@ - **NodeBB version:** - **NodeBB git hash:** -- **Database type:** mongo or redis +- **Database type:** mongo, redis, or postgres - **Database version:** - + - **Exact steps to cause this issue:** ', nested[i].html())); - } + nested.forEach(function (nestedEl, i) { + result.html(result.html().replace('', function () { + return nestedEl.html(); + })); + }); }); $('.search-result-text').find('img:not(.not-responsive)').addClass('img-responsive'); diff --git a/public/src/client/topic.js b/public/src/client/topic.js index 63c63e3051..a73f884482 100644 --- a/public/src/client/topic.js +++ b/public/src/client/topic.js @@ -59,8 +59,8 @@ define('forum/topic', [ } addBlockQuoteHandler(); - addParentHandler(); + addDropupHandler(); navigator.init('[component="post"]', ajaxify.data.postcount, Topic.toTop, Topic.toBottom, Topic.navigatorCallback, Topic.calculateIndex); @@ -166,6 +166,17 @@ define('forum/topic', [ }); } + function addDropupHandler() { + // Locate all dropdowns + var target = $('#content .dropdown-menu').parent(); + + // Toggle dropup if past 50% of screen + $(target).on('show.bs.dropdown', function () { + var dropUp = this.getBoundingClientRect().top > ($(window).height() / 2); + $(this).toggleClass('dropup', dropUp); + }); + } + function updateTopicTitle() { var span = components.get('navbar/title').find('span'); if ($(window).scrollTop() > 50 && span.hasClass('hidden')) { diff --git a/public/src/client/topic/delete-posts.js b/public/src/client/topic/delete-posts.js index fb2c9f7f43..803d957d18 100644 --- a/public/src/client/topic/delete-posts.js +++ b/public/src/client/topic/delete-posts.js @@ -18,6 +18,10 @@ define('forum/topic/delete-posts', ['components', 'postSelect'], function (compo } function onDeletePostsClicked() { + if (modal) { + return; + } + app.parseAndTranslate('partials/delete_posts_modal', {}, function (html) { modal = html; diff --git a/public/src/client/topic/events.js b/public/src/client/topic/events.js index ac4c41e96b..544d8039a6 100644 --- a/public/src/client/topic/events.js +++ b/public/src/client/topic/events.js @@ -159,6 +159,8 @@ define('forum/topic/events', [ }); }); } + + postTools.removeMenu(components.get('post', 'pid', data.post.pid)); } function tagsUpdated(tags) { diff --git a/public/src/client/topic/fork.js b/public/src/client/topic/fork.js index f4190b61ac..134a41a1ac 100644 --- a/public/src/client/topic/fork.js +++ b/public/src/client/topic/fork.js @@ -17,6 +17,10 @@ define('forum/topic/fork', ['components', 'postSelect'], function (components, p } function onForkThreadClicked() { + if (forkModal) { + return; + } + app.parseAndTranslate('partials/fork_thread_modal', {}, function (html) { forkModal = html; diff --git a/public/src/client/topic/merge.js b/public/src/client/topic/merge.js index f250e19f50..fd4eb592ed 100644 --- a/public/src/client/topic/merge.js +++ b/public/src/client/topic/merge.js @@ -25,7 +25,7 @@ define('forum/topic/merge', function () { modal.find('.close,#merge_topics_cancel').on('click', closeModal); - $('[component="category"]').on('click', '[component="category/topic"] a', onTopicClicked); + $('#content').on('click', '[component="category"] [component="category/topic"] a', onTopicClicked); showTopicsSelected(); @@ -101,7 +101,7 @@ define('forum/topic/merge', function () { modal = null; } selectedTids = {}; - $('[component="category"]').off('click', '[component="category/topic"] a', onTopicClicked); + $('#content').off('click', '[component="category"] [component="category/topic"] a', onTopicClicked); } return Merge; diff --git a/public/src/client/topic/postTools.js b/public/src/client/topic/postTools.js index 8d460d7ae3..1608811166 100644 --- a/public/src/client/topic/postTools.js +++ b/public/src/client/topic/postTools.js @@ -66,6 +66,10 @@ define('forum/topic/postTools', [ postEl.find('[component="post/restore"]').toggleClass('hidden', !isDeleted); postEl.find('[component="post/purge"]').toggleClass('hidden', !isDeleted); + PostTools.removeMenu(postEl); + }; + + PostTools.removeMenu = function (postEl) { postEl.find('[component="post/tools"] .dropdown-menu').html(''); }; @@ -338,7 +342,7 @@ define('forum/topic/postTools', [ } if (post.length) { - slug = post.attr('data-userslug'); + slug = utils.slugify(post.attr('data-username'), true); } if (post.length && post.attr('data-uid') !== '0') { slug = '@' + slug; diff --git a/public/src/client/topic/votes.js b/public/src/client/topic/votes.js index ee4813d173..1d17e03d46 100644 --- a/public/src/client/topic/votes.js +++ b/public/src/client/topic/votes.js @@ -72,6 +72,10 @@ define('forum/topic/votes', ['components', 'translator', 'benchpress'], function if (err) { app.alertError(err.message); } + + if (err && err.message === '[[error:not-logged-in]]') { + ajaxify.go('login'); + } }); return false; diff --git a/public/src/client/users.js b/public/src/client/users.js index 3d541e0863..bf6873e693 100644 --- a/public/src/client/users.js +++ b/public/src/client/users.js @@ -130,7 +130,7 @@ define('forum/users', ['translator', 'benchpress'], function (translator, Benchp function handleInvite() { $('[component="user/invite"]').on('click', function () { - bootbox.prompt('Email: ', function (email) { + bootbox.prompt('[[users:prompt-email]]', function (email) { if (!email) { return; } diff --git a/public/src/modules/chat.js b/public/src/modules/chat.js index 558229c4a6..8ec6cd0300 100644 --- a/public/src/modules/chat.js +++ b/public/src/modules/chat.js @@ -75,7 +75,7 @@ define('chat', [ sounds.play('chat-incoming', 'chat.incoming:' + data.message.mid); taskbar.push('chat', modal.attr('data-uuid'), { - title: data.roomName || username, + title: '[[modules:chat.chatting_with]] ' + (data.roomName || username), touid: data.message.fromUser.uid, roomId: data.roomId, }); @@ -136,7 +136,8 @@ define('chat', [ rooms: rooms, }, function (html) { translator.translate(html, function (translated) { - chatsListEl.empty().html(translated); + chatsListEl.find('*').not('.navigation-link').remove(); + chatsListEl.prepend(translated); app.createUserTooltips(chatsListEl, 'right'); }); }); @@ -211,12 +212,12 @@ define('chat', [ chatModal.find('.modal-header').on('dblclick', gotoChats); chatModal.find('button[data-action="maximize"]').on('click', gotoChats); chatModal.find('button[data-action="minimize"]').on('click', function () { - var uuid = chatModal.attr('uuid'); + var uuid = chatModal.attr('data-uuid'); module.minimize(uuid); }); - chatModal.on('click', function () { - taskbar.updateActive(this.getAttribute('data-uuid')); + chatModal.on('click', ':not(.close)', function () { + taskbar.updateActive(chatModal.attr('data-uuid')); if (dragged) { dragged = false; @@ -250,7 +251,7 @@ define('chat', [ Chats.addIPHandler(chatModal); taskbar.push('chat', chatModal.attr('data-uuid'), { - title: data.roomName || (data.users.length ? data.users[0].username : ''), + title: '[[modules:chat.chatting_with]] ' + (data.roomName || (data.users.length ? data.users[0].username : '')), roomId: data.roomId, icon: 'fa-comment', state: '', diff --git a/public/src/modules/handleBack.js b/public/src/modules/handleBack.js index 6a11ebc45e..94e7125f52 100644 --- a/public/src/modules/handleBack.js +++ b/public/src/modules/handleBack.js @@ -35,10 +35,14 @@ define('handleBack', [ storage.removeItem('category:bookmark'); storage.removeItem('category:bookmark:clicked'); + if (!utils.isNumber(bookmarkIndex)) { + return; + } bookmarkIndex = Math.max(0, parseInt(bookmarkIndex, 10) || 0); clickedIndex = Math.max(0, parseInt(clickedIndex, 10) || 0); - if (!utils.isNumber(bookmarkIndex)) { + + if (!bookmarkIndex && !clickedIndex) { return; } diff --git a/public/src/modules/helpers.js b/public/src/modules/helpers.js index 2171eedfba..9e0eb2088c 100644 --- a/public/src/modules/helpers.js +++ b/public/src/modules/helpers.js @@ -185,7 +185,10 @@ } } return states.map(function (priv) { - return ''; + var guestDisabled = ['groups:moderate', 'groups:posts:upvote', 'groups:posts:downvote']; + var disabled = member === 'guests' && guestDisabled.includes(priv.name); + + return ''; }).join(''); } diff --git a/public/src/modules/pictureCropper.js b/public/src/modules/pictureCropper.js index 60e7f2708c..0d8fe3a3ca 100644 --- a/public/src/modules/pictureCropper.js +++ b/public/src/modules/pictureCropper.js @@ -50,6 +50,7 @@ define('pictureCropper', ['translator', 'cropper', 'benchpress'], function (tran aspectRatio: data.aspectRatio, autoCropArea: 1, viewMode: 1, + checkCrossOrigin: false, cropmove: function () { if (data.restrictImageDimension) { if (cropperTool.cropBoxData.width > data.imageDimension) { @@ -96,7 +97,17 @@ define('pictureCropper', ['translator', 'cropper', 'benchpress'], function (tran cropperModal.find('.crop-btn').on('click', function () { $(this).addClass('disabled'); - var imageData = data.imageType ? cropperTool.getCroppedCanvas().toDataURL(data.imageType) : cropperTool.getCroppedCanvas().toDataURL(); + var imageData; + try { + imageData = data.imageType ? cropperTool.getCroppedCanvas().toDataURL(data.imageType) : cropperTool.getCroppedCanvas().toDataURL(); + } catch (err) { + if (err.message === 'Failed to execute \'toDataURL\' on \'HTMLCanvasElement\': Tainted canvases may not be exported.') { + app.alertError('[[error:cors-error]]'); + } else { + app.alertError(err.message); + } + return; + } cropperModal.find('#upload-progress-bar').css('width', '100%'); cropperModal.find('#upload-progress-box').show().removeClass('hide'); @@ -138,13 +149,11 @@ define('pictureCropper', ['translator', 'cropper', 'benchpress'], function (tran function onSubmit(data, callback) { function showAlert(type, message) { - module.hideAlerts(data.uploadModal); if (type === 'error') { data.uploadModal.find('#fileUploadSubmitBtn').removeClass('disabled'); } data.uploadModal.find('#alert-' + type).translateText(message).removeClass('hide'); } - var fileInput = data.uploadModal.find('#fileInput'); if (!fileInput.val()) { return showAlert('error', '[[uploads:select-file-to-upload]]'); @@ -154,7 +163,10 @@ define('pictureCropper', ['translator', 'cropper', 'benchpress'], function (tran var reader = new FileReader(); var imageUrl; var imageType = file.type; - + var fileSize = data.hasOwnProperty('fileSize') && data.fileSize !== undefined ? parseInt(data.fileSize, 10) : false; + if (fileSize && file.size > fileSize * 1024) { + return app.alertError('[[error:file-too-big, ' + fileSize + ']]'); + } reader.addEventListener('load', function () { imageUrl = reader.result; diff --git a/public/src/modules/search.js b/public/src/modules/search.js index f135271d50..ea0f775fd1 100644 --- a/public/src/modules/search.js +++ b/public/src/modules/search.js @@ -7,24 +7,14 @@ define('search', ['navigator', 'translator', 'storage'], function (nav, translat }; Search.query = function (data, callback) { - var term = data.term; - // Detect if a tid was specified - var topicSearch = term.match(/^in:topic-([\d]+) /); + var topicSearch = data.term.match(/^in:topic-([\d]+) /); if (!topicSearch) { - term = term.replace(/^[ ?#]*/, ''); - - try { - term = encodeURIComponent(term); - } catch (e) { - return app.alertError('[[error:invalid-search-term]]'); - } - ajaxify.go('search?' + createQueryString(data)); callback(); } else { - var cleanedTerm = term.replace(topicSearch[0], ''); + var cleanedTerm = data.term.replace(topicSearch[0], ''); var tid = topicSearch[1]; if (cleanedTerm.length > 0) { @@ -36,8 +26,15 @@ define('search', ['navigator', 'translator', 'storage'], function (nav, translat function createQueryString(data) { var searchIn = data.in || 'titlesposts'; var postedBy = data.by || ''; + var term = data.term.replace(/^[ ?#]*/, ''); + try { + term = encodeURIComponent(term); + } catch (e) { + return app.alertError('[[error:invalid-search-term]]'); + } + var query = { - term: data.term, + term: term, in: searchIn, }; diff --git a/public/src/modules/taskbar.js b/public/src/modules/taskbar.js index 1e3ea17e10..5a1c0d886f 100644 --- a/public/src/modules/taskbar.js +++ b/public/src/modules/taskbar.js @@ -1,7 +1,7 @@ 'use strict'; -define('taskbar', ['benchpress'], function (Benchpress) { +define('taskbar', ['benchpress', 'translator'], function (Benchpress, translator) { var taskbar = {}; taskbar.init = function () { @@ -111,32 +111,35 @@ define('taskbar', ['benchpress'], function (Benchpress) { } function createTaskbar(data) { - var title = $('
').text(data.options.title || 'NodeBB Task').html(); + translator.translate(data.options.title, function (taskTitle) { + var title = $('
').text(taskTitle || 'NodeBB Task').html(); - var taskbarEl = $('
  • ') - .addClass(data.options.className) - .html('' + - (data.options.icon ? ' ' : '') + - (data.options.image ? ' ' : '') + - '' + title + '' + - '') - .attr({ - 'data-module': data.module, - 'data-uuid': data.uuid, - }) - .addClass(data.options.state !== undefined ? data.options.state : 'active'); + var taskbarEl = $('
  • ') + .addClass(data.options.className) + .html('' + + (data.options.icon ? ' ' : '') + + (data.options.image ? ' ' : '') + + '' + title + '' + + '') + .attr({ + title: title, + 'data-module': data.module, + 'data-uuid': data.uuid, + }) + .addClass(data.options.state !== undefined ? data.options.state : 'active'); - if (!data.options.state || data.options.state === 'active') { - minimizeAll(); - } + if (!data.options.state || data.options.state === 'active') { + minimizeAll(); + } - taskbar.tasklist.append(taskbarEl); - update(); + taskbar.tasklist.append(taskbarEl); + update(); - data.element = taskbarEl; + data.element = taskbarEl; - taskbarEl.data(data); - $(window).trigger('action:taskbar.pushed', data); + taskbarEl.data(data); + $(window).trigger('action:taskbar.pushed', data); + }); } taskbar.updateTitle = function (module, uuid, newTitle) { diff --git a/public/src/require-config.js b/public/src/require-config.js index a7c70ac70e..2657d4aa33 100644 --- a/public/src/require-config.js +++ b/public/src/require-config.js @@ -2,7 +2,7 @@ require.config({ baseUrl: config.relative_path + '/assets/src/modules', - waitSeconds: 7, + waitSeconds: 0, urlArgs: config['cache-buster'], paths: { forum: '../client', diff --git a/src/categories.js b/src/categories.js index 1270924f0e..93d2c87e54 100644 --- a/src/categories.js +++ b/src/categories.js @@ -95,7 +95,7 @@ Categories.getAllCategories = function (uid, callback) { ], callback); }; -Categories.getCategoriesByPrivilege = function (set, uid, privilege, callback) { +Categories.getCidsByPrivilege = function (set, uid, privilege, callback) { async.waterfall([ function (next) { db.getSortedSetRange(set, 0, -1, next); @@ -103,6 +103,14 @@ Categories.getCategoriesByPrivilege = function (set, uid, privilege, callback) { function (cids, next) { privileges.categories.filterCids(privilege, cids, uid, next); }, + ], callback); +}; + +Categories.getCategoriesByPrivilege = function (set, uid, privilege, callback) { + async.waterfall([ + function (next) { + Categories.getCidsByPrivilege(set, uid, privilege, next); + }, function (cids, next) { Categories.getCategories(cids, uid, next); }, @@ -378,3 +386,5 @@ Categories.filterIgnoringUids = function (cid, uids, callback) { }, ], callback); }; + +Categories.async = require('./promisify')(Categories); diff --git a/src/categories/create.js b/src/categories/create.js index 1d6fc6c09f..835515c2b4 100644 --- a/src/categories/create.js +++ b/src/categories/create.js @@ -35,7 +35,7 @@ module.exports = function (Categories) { parentCid: parentCid, topic_count: 0, post_count: 0, - disabled: 0, + disabled: data.disabled ? 1 : 0, order: order, link: data.link || '', numRecentReplies: 1, @@ -84,10 +84,24 @@ module.exports = function (Categories) { ], next); }, function (results, next) { - if (data.cloneFromCid && parseInt(data.cloneFromCid, 10)) { - return Categories.copySettingsFrom(data.cloneFromCid, category.cid, !data.parentCid, next); - } - next(null, category); + async.series([ + function (next) { + if (data.cloneFromCid && parseInt(data.cloneFromCid, 10)) { + return Categories.copySettingsFrom(data.cloneFromCid, category.cid, !data.parentCid, next); + } + + next(); + }, + function (next) { + if (data.cloneChildren) { + return duplicateCategoriesChildren(category.cid, data.cloneFromCid, data.uid, next); + } + + next(); + }, + ], function (err) { + next(err, category); + }); }, function (category, next) { plugins.fireHook('action:category.create', { category: category }); @@ -96,6 +110,27 @@ module.exports = function (Categories) { ], callback); }; + function duplicateCategoriesChildren(parentCid, cid, uid, callback) { + Categories.getChildren([cid], uid, function (err, children) { + if (err || !children.length) { + return callback(err); + } + + children = children[0]; + + children.forEach(function (child) { + child.parentCid = parentCid; + child.cloneFromCid = child.cid; + child.cloneChildren = true; + child.name = utils.decodeHTMLEntities(child.name); + child.description = utils.decodeHTMLEntities(child.description); + child.uid = uid; + }); + + async.each(children, Categories.create, callback); + }); + } + Categories.assignColours = function () { var backgrounds = ['#AB4642', '#DC9656', '#F7CA88', '#A1B56C', '#86C1B9', '#7CAFC2', '#BA8BAF', '#A16946']; var text = ['#fff', '#fff', '#333', '#fff', '#333', '#fff', '#fff', '#fff']; diff --git a/src/categories/data.js b/src/categories/data.js index a1a9d5c587..8eb24b2179 100644 --- a/src/categories/data.js +++ b/src/categories/data.js @@ -28,9 +28,15 @@ module.exports = function (Categories) { return; } - category.name = validator.escape(String(category.name || '')); - category.disabled = category.hasOwnProperty('disabled') ? parseInt(category.disabled, 10) === 1 : undefined; - category.isSection = category.hasOwnProperty('isSection') ? parseInt(category.isSection, 10) === 1 : undefined; + if (category.hasOwnProperty('name')) { + category.name = validator.escape(String(category.name || '')); + } + if (category.hasOwnProperty('disabled')) { + category.disabled = parseInt(category.disabled, 10) === 1; + } + if (category.hasOwnProperty('isSection')) { + category.isSection = parseInt(category.isSection, 10) === 1; + } if (category.hasOwnProperty('icon')) { category.icon = category.icon || 'hidden'; diff --git a/src/cli/index.js b/src/cli/index.js index cece70d554..20f7c85271 100644 --- a/src/cli/index.js +++ b/src/cli/index.js @@ -186,8 +186,9 @@ program program .command('build [targets...]') .description('Compile static assets ' + '(JS, CSS, templates, languages, sounds)'.red) - .action(function (targets) { - require('./manage').build(targets.length ? targets : true); + .option('-s, --series', 'Run builds in series without extra processes') + .action(function (targets, options) { + require('./manage').build(targets.length ? targets : true, options); }) .on('--help', function () { require('./manage').buildTargets(); diff --git a/src/cli/reset.js b/src/cli/reset.js index 44d78df961..d3675d2ee4 100644 --- a/src/cli/reset.js +++ b/src/cli/reset.js @@ -11,6 +11,7 @@ var events = require('../events'); var meta = require('../meta'); var plugins = require('../plugins'); var widgets = require('../widgets'); +var privileges = require('../privileges'); var dirname = require('./paths').baseDir; @@ -86,9 +87,13 @@ exports.reset = function (options, callback) { }; function resetSettings(callback) { - meta.configs.set('allowLocalLogin', 1, function (err) { + privileges.global.give(['local:login'], 'registered-users', function (err) { + if (err) { + return callback(err); + } + winston.info('[reset] registered-users given login privilege'); winston.info('[reset] Settings reset to default'); - callback(err); + callback(); }); } diff --git a/src/cli/upgrade.js b/src/cli/upgrade.js index 38df2594cb..618183111e 100644 --- a/src/cli/upgrade.js +++ b/src/cli/upgrade.js @@ -41,6 +41,7 @@ var steps = { handler: function (next) { async.series([ db.init, + require('../meta').configs.init, upgrade.run, ], next); }, diff --git a/src/controllers/accounts.js b/src/controllers/accounts.js index 0f31595ef4..8a7f1f1db6 100644 --- a/src/controllers/accounts.js +++ b/src/controllers/accounts.js @@ -4,13 +4,14 @@ var accountsController = { profile: require('./accounts/profile'), edit: require('./accounts/edit'), info: require('./accounts/info'), + categories: require('./accounts/categories'), settings: require('./accounts/settings'), groups: require('./accounts/groups'), follow: require('./accounts/follow'), posts: require('./accounts/posts'), notifications: require('./accounts/notifications'), chats: require('./accounts/chats'), - session: require('./accounts/session'), + sessions: require('./accounts/sessions'), blocks: require('./accounts/blocks'), uploads: require('./accounts/uploads'), consent: require('./accounts/consent'), diff --git a/src/controllers/accounts/categories.js b/src/controllers/accounts/categories.js new file mode 100644 index 0000000000..43eff07889 --- /dev/null +++ b/src/controllers/accounts/categories.js @@ -0,0 +1,43 @@ +'use strict'; + +var async = require('async'); + +var user = require('../../user'); +var categories = require('../../categories'); +var accountHelpers = require('./helpers'); + +var categoriesController = module.exports; + +categoriesController.get = function (req, res, callback) { + var userData; + async.waterfall([ + function (next) { + accountHelpers.getUserDataByUserSlug(req.params.userslug, req.uid, next); + }, + function (_userData, next) { + userData = _userData; + if (!userData) { + return callback(); + } + + async.parallel({ + ignored: function (next) { + user.getIgnoredCategories(userData.uid, next); + }, + categories: function (next) { + categories.buildForSelect(userData.uid, 'find', next); + }, + }, next); + }, + function (results) { + results.categories.forEach(function (category) { + if (category) { + category.isIgnored = results.ignored.includes(String(category.cid)); + } + }); + userData.categories = results.categories; + userData.title = '[[pages:account/watched_categories, ' + userData.username + ']]'; + res.render('account/categories', userData); + }, + ], callback); +}; diff --git a/src/controllers/accounts/helpers.js b/src/controllers/accounts/helpers.js index 1d2a30272e..12f7674563 100644 --- a/src/controllers/accounts/helpers.js +++ b/src/controllers/accounts/helpers.js @@ -65,6 +65,17 @@ helpers.getUserDataByUserSlug = function (userslug, callerUID, callback) { globalMod: true, admin: true, }, + }, { + id: 'sessions', + route: 'sessions', + name: '[[pages:account/sessions]]', + visibility: { + self: true, + other: false, + moderator: false, + globalMod: false, + admin: false, + }, }, { id: 'consent', route: 'consent', diff --git a/src/controllers/accounts/session.js b/src/controllers/accounts/sessions.js similarity index 57% rename from src/controllers/accounts/session.js rename to src/controllers/accounts/sessions.js index 809cdb6dad..1609fa2383 100644 --- a/src/controllers/accounts/session.js +++ b/src/controllers/accounts/sessions.js @@ -4,8 +4,36 @@ var async = require('async'); var db = require('../../database'); var user = require('../../user'); +var helpers = require('../helpers'); +var accountHelpers = require('./helpers'); -var sessionController = {}; +var sessionController = module.exports; + +sessionController.get = function (req, res, callback) { + var userData; + + async.waterfall([ + function (next) { + accountHelpers.getUserDataByUserSlug(req.params.userslug, req.uid, next); + }, + function (_userData, next) { + userData = _userData; + if (!userData) { + return callback(); + } + + user.auth.getSessions(userData.uid, req.sessionID, next); + }, + function (sessions) { + userData.sessions = sessions; + + userData.title = '[[pages:account/sessions]]'; + userData.breadcrumbs = helpers.buildBreadcrumbs([{ text: userData.username, url: '/user/' + userData.userslug }, { text: '[[pages:account/sessions]]' }]); + + res.render('account/sessions', userData); + }, + ], callback); +}; sessionController.revoke = function (req, res, next) { if (!req.params.hasOwnProperty('uuid')) { @@ -50,5 +78,3 @@ sessionController.revoke = function (req, res, next) { return res.sendStatus(200); }); }; - -module.exports = sessionController; diff --git a/src/controllers/accounts/settings.js b/src/controllers/accounts/settings.js index 5f5bb57aa7..0beab3f77a 100644 --- a/src/controllers/accounts/settings.js +++ b/src/controllers/accounts/settings.js @@ -148,7 +148,9 @@ settingsController.get = function (req, res, callback) { var notifFreqOptions = [ 'all', + 'first', 'everyTen', + 'threshold', 'logarithmic', 'disabled', ]; @@ -156,7 +158,7 @@ settingsController.get = function (req, res, callback) { userData.upvoteNotifFreq = notifFreqOptions.map(function (name) { return { name: name, - selected: name === userData.notifFreqOptions, + selected: name === userData.settings.upvoteNotifFreq, }; }); diff --git a/src/controllers/admin/database.js b/src/controllers/admin/database.js index efec771ee6..147747822c 100644 --- a/src/controllers/admin/database.js +++ b/src/controllers/admin/database.js @@ -25,6 +25,14 @@ databaseController.get = function (req, res, next) { next(); } }, + postgres: function (next) { + if (nconf.get('postgres')) { + var pdb = require('../../database/postgres'); + pdb.info(pdb.pool, next); + } else { + next(); + } + }, }, next); }, function (results) { diff --git a/src/controllers/admin/events.js b/src/controllers/admin/events.js index f6988ef72e..88a2848f37 100644 --- a/src/controllers/admin/events.js +++ b/src/controllers/admin/events.js @@ -14,24 +14,35 @@ eventsController.get = function (req, res, next) { var start = (page - 1) * itemsPerPage; var stop = start + itemsPerPage - 1; + var currentFilter = req.query.filter || ''; + async.waterfall([ function (next) { async.parallel({ eventCount: function (next) { - db.sortedSetCard('events:time', next); + db.sortedSetCard('events:time' + (currentFilter ? ':' + currentFilter : ''), next); }, events: function (next) { - events.getEvents(start, stop, next); + events.getEvents(currentFilter, start, stop, next); }, }, next); }, function (results) { + var types = [''].concat(events.types); + var filters = types.map(function (type) { + return { + value: type, + name: type || 'all', + selected: type === currentFilter, + }; + }); + var pageCount = Math.max(1, Math.ceil(results.eventCount / itemsPerPage)); res.render('admin/advanced/events', { events: results.events, - pagination: pagination.create(page, pageCount), - next: 20, + pagination: pagination.create(page, pageCount, req.query), + filters: filters, }); }, ], next); diff --git a/src/controllers/admin/postqueue.js b/src/controllers/admin/postqueue.js index 66ce1e237f..1812d70e95 100644 --- a/src/controllers/admin/postqueue.js +++ b/src/controllers/admin/postqueue.js @@ -1,6 +1,7 @@ 'use strict'; var async = require('async'); +var validator = require('validator'); var db = require('../../database'); var user = require('../../user'); @@ -80,7 +81,8 @@ function getQueuedPosts(ids, callback) { }); async.map(postData, function (postData, next) { - postData.data.rawContent = postData.data.content; + postData.data.rawContent = validator.escape(String(postData.data.content)); + postData.data.title = validator.escape(String(postData.data.title)); async.waterfall([ function (next) { if (postData.data.cid) { diff --git a/src/controllers/admin/privileges.js b/src/controllers/admin/privileges.js index 92dbe27ef9..08bfe3183a 100644 --- a/src/controllers/admin/privileges.js +++ b/src/controllers/admin/privileges.js @@ -2,6 +2,7 @@ var async = require('async'); +var db = require('../../database'); var categories = require('../../categories'); var privileges = require('../../privileges'); @@ -19,7 +20,19 @@ privilegesController.get = function (req, res, callback) { privileges.categories.list(cid, next); } }, - allCategories: async.apply(categories.buildForSelect, req.uid, 'read'), + allCategories: function (next) { + async.waterfall([ + function (next) { + db.getSortedSetRange('cid:0:children', 0, -1, next); + }, + function (cids, next) { + categories.getCategories(cids, req.uid, next); + }, + function (categoriesData, next) { + categories.buildForSelectCategories(categoriesData, next); + }, + ], next); + }, }, next); }, function (data) { diff --git a/src/controllers/admin/uploads.js b/src/controllers/admin/uploads.js index 21dbbbca99..c7952ee98c 100644 --- a/src/controllers/admin/uploads.js +++ b/src/controllers/admin/uploads.js @@ -5,7 +5,6 @@ var async = require('async'); var nconf = require('nconf'); var mime = require('mime'); var fs = require('fs'); -var jimp = require('jimp'); var meta = require('../../meta'); var posts = require('../../posts'); @@ -177,16 +176,13 @@ uploadsController.uploadTouchIcon = function (req, res, next) { } // Resize the image into squares for use as touch icons at various DPIs - async.each(sizes, function (size, next) { - async.series([ - async.apply(file.saveFileToLocal, 'touchicon-' + size + '.png', 'system', uploadedFile.path), - async.apply(image.resizeImage, { - path: path.join(nconf.get('upload_path'), 'system', 'touchicon-' + size + '.png'), - extension: 'png', - width: size, - height: size, - }), - ], next); + async.eachSeries(sizes, function (size, next) { + image.resizeImage({ + path: uploadedFile.path, + target: path.join(nconf.get('upload_path'), 'system', 'touchicon-' + size + '.png'), + width: size, + height: size, + }, next); }, function (err) { file.delete(uploadedFile.path); @@ -291,7 +287,6 @@ function uploadImage(filename, folder, uploadedFile, req, res, next) { async.apply(image.resizeImage, { path: uploadedFile.path, target: uploadPath, - extension: 'png', height: 50, }), async.apply(meta.configs.set, 'brand:emailLogo', path.join(nconf.get('upload_url'), 'system/site-logo-x50.png')), @@ -299,15 +294,16 @@ function uploadImage(filename, folder, uploadedFile, req, res, next) { next(err, imageData); }); } else if (path.basename(filename, path.extname(filename)) === 'og:image' && folder === 'system') { - jimp.read(imageData.path).then(function (image) { + image.size(imageData.path, function (err, size) { + if (err) { + next(err); + } meta.configs.setMultiple({ - 'og:image:height': image.bitmap.height, - 'og:image:width': image.bitmap.width, + 'og:image:width': size.width, + 'og:image:height': size.height, }, function (err) { next(err, imageData); }); - }).catch(function (err) { - next(err); }); } else { setImmediate(next, null, imageData); diff --git a/src/controllers/authentication.js b/src/controllers/authentication.js index 643b898843..941ec5bcc9 100644 --- a/src/controllers/authentication.js +++ b/src/controllers/authentication.js @@ -12,10 +12,9 @@ var meta = require('../meta'); var user = require('../user'); var plugins = require('../plugins'); var utils = require('../utils'); -var Password = require('../password'); var translator = require('../translator'); var helpers = require('./helpers'); - +var privileges = require('../privileges'); var sockets = require('../socket.io'); var authenticationController = module.exports; @@ -398,23 +397,25 @@ authenticationController.localLogin = function (req, username, password, next) { uid = _uid; async.parallel({ - userData: function (next) { - db.getObjectFields('user:' + uid, ['password', 'passwordExpiry'], next); - }, + userData: async.apply(db.getObjectFields, 'user:' + uid, ['passwordExpiry']), isAdminOrGlobalMod: function (next) { user.isAdminOrGlobalMod(uid, next); }, banned: function (next) { user.isBanned(uid, next); }, + hasLoginPrivilege: function (next) { + privileges.global.can('local:login', uid, next); + }, }, next); }, function (result, next) { - userData = result.userData; - userData.uid = uid; - userData.isAdminOrGlobalMod = result.isAdminOrGlobalMod; + userData = Object.assign(result.userData, { + uid: uid, + isAdminOrGlobalMod: result.isAdminOrGlobalMod, + }); - if (!result.isAdminOrGlobalMod && parseInt(meta.config.allowLocalLogin, 10) === 0) { + if (parseInt(uid, 10) && !result.hasLoginPrivilege) { return next(new Error('[[error:local-login-disabled]]')); } @@ -422,16 +423,13 @@ authenticationController.localLogin = function (req, username, password, next) { return getBanInfo(uid, next); } - user.auth.logAttempt(uid, req.ip, next); - }, - function (next) { - Password.compare(password, userData.password, next); + user.isPasswordCorrect(uid, password, req.ip, next); }, function (passwordMatch, next) { if (!passwordMatch) { return next(new Error('[[error:invalid-login-credentials]]')); } - user.auth.clearLoginAttempts(uid); + next(null, userData, '[[success:authentication-successful]]'); }, ], next); diff --git a/src/controllers/categories.js b/src/controllers/categories.js index 1eb46a0041..b15db0bbff 100644 --- a/src/controllers/categories.js +++ b/src/controllers/categories.js @@ -47,11 +47,12 @@ categoriesController.list = function (req, res, next) { } data.categories.forEach(function (category) { - if (category && Array.isArray(category.posts) && category.posts.length) { + if (category && Array.isArray(category.posts) && category.posts.length && category.posts[0]) { category.teaser = { url: nconf.get('relative_path') + '/post/' + category.posts[0].pid, timestampISO: category.posts[0].timestampISO, pid: category.posts[0].pid, + topic: category.posts[0].topic, }; } }); diff --git a/src/controllers/groups.js b/src/controllers/groups.js index 0703915e6f..6063653fc0 100644 --- a/src/controllers/groups.js +++ b/src/controllers/groups.js @@ -7,8 +7,9 @@ var meta = require('../meta'); var groups = require('../groups'); var user = require('../user'); var helpers = require('./helpers'); +var pagination = require('../pagination'); -var groupsController = {}; +var groupsController = module.exports; groupsController.list = function (req, res, next) { var sort = req.query.sort || 'alpha'; @@ -113,7 +114,12 @@ groupsController.details = function (req, res, callback) { }; groupsController.members = function (req, res, callback) { + var page = parseInt(req.query.page, 10) || 1; + var usersPerPage = 50; + var start = Math.max(0, (page - 1) * usersPerPage); + var stop = start + usersPerPage - 1; var groupName; + var groupData; async.waterfall([ function (next) { groups.getGroupNameByGroupSlug(req.params.slug, next); @@ -127,14 +133,16 @@ groupsController.members = function (req, res, callback) { isAdminOrGlobalMod: async.apply(user.isAdminOrGlobalMod, req.uid), isMember: async.apply(groups.isMember, req.uid, groupName), isHidden: async.apply(groups.isHidden, groupName), + groupData: async.apply(groups.getGroupData, groupName), }, next); }, function (results, next) { if (results.isHidden && !results.isMember && !results.isAdminOrGlobalMod) { return callback(); } + groupData = results.groupData; - user.getUsersFromSet('group:' + groupName + ':members', req.uid, 0, 49, next); + user.getUsersFromSet('group:' + groupName + ':members', req.uid, start, stop, next); }, function (users) { var breadcrumbs = helpers.buildBreadcrumbs([ @@ -143,10 +151,10 @@ groupsController.members = function (req, res, callback) { { text: '[[groups:details.members]]' }, ]); + var pageCount = Math.max(1, Math.ceil(groupData.memberCount / usersPerPage)); res.render('groups/members', { users: users, - nextStart: 50, - loadmore_display: users.length > 50 ? 'block' : 'hide', + pagination: pagination.create(page, pageCount, req.query), breadcrumbs: breadcrumbs, }); }, @@ -177,5 +185,3 @@ groupsController.uploadCover = function (req, res, next) { res.json([{ url: image.url }]); }); }; - -module.exports = groupsController; diff --git a/src/controllers/helpers.js b/src/controllers/helpers.js index 4b41c62dfb..636abd77bb 100644 --- a/src/controllers/helpers.js +++ b/src/controllers/helpers.js @@ -12,6 +12,7 @@ var categories = require('../categories'); var plugins = require('../plugins'); var meta = require('../meta'); var middleware = require('../middleware'); +var utils = require('../utils'); var helpers = module.exports; @@ -227,19 +228,38 @@ helpers.buildTitle = function (pageTitle) { return title; }; +helpers.getCategories = function (set, uid, privilege, selectedCid, callback) { + async.waterfall([ + function (next) { + categories.getCidsByPrivilege(set, uid, privilege, next); + }, + function (cids, next) { + getCategoryData(cids, uid, selectedCid, next); + }, + ], callback); +}; + helpers.getWatchedCategories = function (uid, selectedCid, callback) { - if (selectedCid && !Array.isArray(selectedCid)) { - selectedCid = [selectedCid]; - } async.waterfall([ function (next) { user.getWatchedCategories(uid, next); }, function (cids, next) { + getCategoryData(cids, uid, selectedCid, next); + }, + ], callback); +}; + +function getCategoryData(cids, uid, selectedCid, callback) { + if (selectedCid && !Array.isArray(selectedCid)) { + selectedCid = [selectedCid]; + } + async.waterfall([ + function (next) { privileges.categories.filterCids('read', cids, uid, next); }, function (cids, next) { - categories.getCategoriesFields(cids, ['cid', 'name', 'slug', 'icon', 'link', 'color', 'bgColor', 'parentCid'], next); + categories.getCategoriesFields(cids, ['cid', 'name', 'slug', 'icon', 'link', 'color', 'bgColor', 'parentCid', 'image', 'imageClass'], next); }, function (categoryData, next) { categoryData = categoryData.filter(function (category) { @@ -249,6 +269,7 @@ helpers.getWatchedCategories = function (uid, selectedCid, callback) { var selectedCids = []; categoryData.forEach(function (category) { category.selected = selectedCid ? selectedCid.indexOf(String(category.cid)) !== -1 : false; + category.parentCid = category.hasOwnProperty('parentCid') && utils.isNumber(category.parentCid) ? category.parentCid : 0; if (category.selected) { selectedCategory.push(category); selectedCids.push(parseInt(category.cid, 10)); @@ -280,7 +301,7 @@ helpers.getWatchedCategories = function (uid, selectedCid, callback) { next(null, { categories: categoriesData, selectedCategory: selectedCategory, selectedCids: selectedCids }); }, ], callback); -}; +} function recursive(category, categoriesData, level) { category.level = level; diff --git a/src/controllers/home.js b/src/controllers/home.js index 35a6cfe6a0..49cee7e192 100644 --- a/src/controllers/home.js +++ b/src/controllers/home.js @@ -1,6 +1,8 @@ 'use strict'; var async = require('async'); +var url = require('url'); + var plugins = require('../plugins'); var meta = require('../meta'); var user = require('../user'); @@ -18,7 +20,7 @@ function getUserHomeRoute(uid, callback) { var route = adminHomePageRoute(); if (settings.homePageRoute !== 'undefined' && settings.homePageRoute !== 'none') { - route = settings.homePageRoute || route; + route = (settings.homePageRoute || route).replace(/^\/+/, ''); } next(null, route); @@ -40,14 +42,22 @@ function rewrite(req, res, next) { } }, function (route, next) { - var hook = 'action:homepage.get:' + route; - - if (!plugins.hasListeners(hook)) { - req.url = req.path + (!req.path.endsWith('/') ? '/' : '') + route; - } else { - res.locals.homePageRoute = route; + var parsedUrl; + try { + parsedUrl = url.parse(route, true); + } catch (err) { + return next(err); } + var pathname = parsedUrl.pathname; + var hook = 'action:homepage.get:' + pathname; + if (!plugins.hasListeners(hook)) { + req.url = req.path + (!req.path.endsWith('/') ? '/' : '') + pathname; + } else { + res.locals.homePageRoute = pathname; + } + req.query = Object.assign(parsedUrl.query, req.query); + next(); }, ], next); diff --git a/src/controllers/index.js b/src/controllers/index.js index a91636dfc7..a183692d4e 100644 --- a/src/controllers/index.js +++ b/src/controllers/index.js @@ -7,6 +7,7 @@ var validator = require('validator'); var meta = require('../meta'); var user = require('../user'); var plugins = require('../plugins'); +var privileges = require('../privileges'); var helpers = require('./helpers'); var Controllers = module.exports; @@ -106,7 +107,6 @@ Controllers.login = function (req, res, next) { data.alternate_logins = loginStrategies.length > 0; data.authentication = loginStrategies; - data.allowLocalLogin = parseInt(meta.config.allowLocalLogin, 10) === 1 || parseInt(req.query.local, 10) === 1; data.allowRegistration = registrationType === 'normal' || registrationType === 'admin-approval' || registrationType === 'admin-approval-ip'; data.allowLoginWith = '[[login:' + allowLoginWith + ']]'; data.breadcrumbs = helpers.buildBreadcrumbs([{ @@ -115,26 +115,33 @@ Controllers.login = function (req, res, next) { data.error = req.flash('error')[0] || errorText; data.title = '[[pages:login]]'; - if (!data.allowLocalLogin && !data.allowRegistration && data.alternate_logins && data.authentication.length === 1) { - if (res.locals.isAPI) { - return helpers.redirect(res, { - external: nconf.get('relative_path') + data.authentication[0].url, - }); + privileges.global.canGroup('local:login', 'registered-users', function (err, hasLoginPrivilege) { + if (err) { + return next(err); } - return res.redirect(nconf.get('relative_path') + data.authentication[0].url); - } - if (req.loggedIn) { - user.getUserFields(req.uid, ['username', 'email'], function (err, user) { - if (err) { - return next(err); + + data.allowLocalLogin = hasLoginPrivilege || parseInt(req.query.local, 10) === 1; + if (!data.allowLocalLogin && !data.allowRegistration && data.alternate_logins && data.authentication.length === 1) { + if (res.locals.isAPI) { + return helpers.redirect(res, { + external: nconf.get('relative_path') + data.authentication[0].url, + }); } - data.username = allowLoginWith === 'email' ? user.email : user.username; - data.alternate_logins = false; + return res.redirect(nconf.get('relative_path') + data.authentication[0].url); + } + if (req.loggedIn) { + user.getUserFields(req.uid, ['username', 'email'], function (err, user) { + if (err) { + return next(err); + } + data.username = allowLoginWith === 'email' ? user.email : user.username; + data.alternate_logins = false; + res.render('login', data); + }); + } else { res.render('login', data); - }); - } else { - res.render('login', data); - } + } + }); }; Controllers.register = function (req, res, next) { diff --git a/src/controllers/unread.js b/src/controllers/unread.js index 500cfe11a6..96dc4f66ce 100644 --- a/src/controllers/unread.js +++ b/src/controllers/unread.js @@ -54,12 +54,6 @@ unreadController.get = function (req, res, next) { cutoff: cutoff, }, next); }, - function (data, next) { - user.blocks.filter(req.uid, data.topics, function (err, filtered) { - data.topics = filtered; - next(err, data); - }); - }, function (data) { data.title = meta.config.homePageTitle || '[[pages:home]]'; data.pageCount = Math.max(1, Math.ceil(data.topicCount / settings.topicsPerPage)); diff --git a/src/controllers/uploads.js b/src/controllers/uploads.js index f837d58529..4421438cda 100644 --- a/src/controllers/uploads.js +++ b/src/controllers/uploads.js @@ -25,7 +25,7 @@ uploadsController.upload = function (req, res, filesIterator) { files = files[0]; } - async.map(files, filesIterator, function (err, images) { + async.mapSeries(files, filesIterator, function (err, images) { deleteTempFiles(files); if (err) { @@ -56,6 +56,9 @@ function uploadAsImage(req, uploadedFile, callback) { if (!canUpload) { return next(new Error('[[error:no-privileges]]')); } + image.checkDimensions(uploadedFile.path, next); + }, + function (next) { if (plugins.hasListeners('filter:uploadImage')) { return plugins.fireHook('filter:uploadImage', { image: uploadedFile, @@ -113,25 +116,16 @@ function resizeImage(fileObj, callback) { return callback(null, fileObj); } - var dirname = path.dirname(fileObj.path); - var extname = path.extname(fileObj.path); - var basename = path.basename(fileObj.path, extname); - image.resizeImage({ path: fileObj.path, - target: path.join(dirname, basename + '-resized' + extname), - extension: extname, + target: file.appendToFileName(fileObj.path, '-resized'), width: parseInt(meta.config.maximumImageWidth, 10) || 760, quality: parseInt(meta.config.resizeImageQuality, 10) || 60, }, next); }, function (next) { // Return the resized version to the composer/postData - var dirname = path.dirname(fileObj.url); - var extname = path.extname(fileObj.url); - var basename = path.basename(fileObj.url, extname); - - fileObj.url = dirname + '/' + basename + '-resized' + extname; + fileObj.url = file.appendToFileName(fileObj.url, '-resized'); next(null, fileObj); }, @@ -157,7 +151,6 @@ uploadsController.uploadThumb = function (req, res, next) { var size = parseInt(meta.config.topicThumbSize, 10) || 120; image.resizeImage({ path: uploadedFile.path, - extension: path.extname(uploadedFile.name), width: size, height: size, }, next); diff --git a/src/database/mongo.js b/src/database/mongo.js index 5b264253f5..9ab35feba2 100644 --- a/src/database/mongo.js +++ b/src/database/mongo.js @@ -98,6 +98,7 @@ mongoModule.getConnectionOptions = function () { reconnectTries: 3600, reconnectInterval: 1000, autoReconnect: true, + useNewUrlParser: true, }; return _.merge(connOptions, nconf.get('mongo:options') || {}); @@ -118,7 +119,6 @@ mongoModule.init = function (callback) { } client = _client; db = client.db(); - mongoModule.client = db; require('./mongo/main')(db, mongoModule); @@ -126,6 +126,10 @@ mongoModule.init = function (callback) { require('./mongo/sets')(db, mongoModule); require('./mongo/sorted')(db, mongoModule); require('./mongo/list')(db, mongoModule); + require('./mongo/transaction')(db, mongoModule); + + mongoModule.async = require('../promisify')(mongoModule, ['client', 'sessionStore']); + callback(); }); }; diff --git a/src/database/mongo/hash.js b/src/database/mongo/hash.js index bfbeb5cb6c..1298e6e292 100644 --- a/src/database/mongo/hash.js +++ b/src/database/mongo/hash.js @@ -46,7 +46,7 @@ module.exports = function (db, module) { if (data.hasOwnProperty('')) { delete data['']; } - db.collection('objects').update({ _key: key }, { $set: data }, { upsert: true, w: 1 }, function (err) { + db.collection('objects').updateOne({ _key: key }, { $set: data }, { upsert: true, w: 1 }, function (err) { if (err) { return callback(err); } @@ -208,8 +208,8 @@ module.exports = function (db, module) { } var data = {}; field = helpers.fieldToString(field); - data[field] = ''; - db.collection('objects').findOne({ _key: key }, { fields: data }, function (err, item) { + data[field] = 1; + db.collection('objects').findOne({ _key: key }, { projection: data }, function (err, item) { callback(err, !!item && item[field] !== undefined && item[field] !== null); }); }; @@ -222,10 +222,10 @@ module.exports = function (db, module) { var data = {}; fields.forEach(function (field) { field = helpers.fieldToString(field); - data[field] = ''; + data[field] = 1; }); - db.collection('objects').findOne({ _key: key }, { fields: data }, function (err, item) { + db.collection('objects').findOne({ _key: key }, { projection: data }, function (err, item) { if (err) { return callback(err); } @@ -259,7 +259,7 @@ module.exports = function (db, module) { data[field] = ''; }); - db.collection('objects').update({ _key: key }, { $unset: data }, function (err) { + db.collection('objects').updateOne({ _key: key }, { $unset: data }, function (err) { if (err) { return callback(err); } @@ -317,7 +317,7 @@ module.exports = function (db, module) { } - db.collection('objects').findAndModify({ _key: key }, {}, { $inc: data }, { new: true, upsert: true }, function (err, result) { + db.collection('objects').findOneAndUpdate({ _key: key }, { $inc: data }, { returnOriginal: false, upsert: true }, function (err, result) { if (err) { return callback(err); } diff --git a/src/database/mongo/list.js b/src/database/mongo/list.js index b0b87ae922..219cd53be5 100644 --- a/src/database/mongo/list.js +++ b/src/database/mongo/list.js @@ -18,7 +18,7 @@ module.exports = function (db, module) { } if (exists) { - db.collection('objects').update({ _key: key }, { $push: { array: { $each: [value], $position: 0 } } }, { upsert: true, w: 1 }, function (err) { + db.collection('objects').updateOne({ _key: key }, { $push: { array: { $each: [value], $position: 0 } } }, { upsert: true, w: 1 }, function (err) { callback(err); }); } else { @@ -33,7 +33,7 @@ module.exports = function (db, module) { return callback(); } value = helpers.valueToString(value); - db.collection('objects').update({ _key: key }, { $push: { array: value } }, { upsert: true, w: 1 }, function (err) { + db.collection('objects').updateOne({ _key: key }, { $push: { array: value } }, { upsert: true, w: 1 }, function (err) { callback(err); }); }; @@ -48,7 +48,7 @@ module.exports = function (db, module) { return callback(err); } - db.collection('objects').update({ _key: key }, { $pop: { array: 1 } }, function (err) { + db.collection('objects').updateOne({ _key: key }, { $pop: { array: 1 } }, function (err) { callback(err, (value && value.length) ? value[0] : null); }); }); @@ -61,7 +61,7 @@ module.exports = function (db, module) { } value = helpers.valueToString(value); - db.collection('objects').update({ _key: key }, { $pull: { array: value } }, function (err) { + db.collection('objects').updateOne({ _key: key }, { $pull: { array: value } }, function (err) { callback(err); }); }; @@ -76,7 +76,7 @@ module.exports = function (db, module) { return callback(err); } - db.collection('objects').update({ _key: key }, { $set: { array: value } }, function (err) { + db.collection('objects').updateOne({ _key: key }, { $set: { array: value } }, function (err) { callback(err); }); }); diff --git a/src/database/mongo/main.js b/src/database/mongo/main.js index 55a29e6ff9..1b23bc3409 100644 --- a/src/database/mongo/main.js +++ b/src/database/mongo/main.js @@ -12,7 +12,7 @@ module.exports = function (db, module) { module.emptydb = function (callback) { callback = callback || helpers.noop; - db.collection('objects').remove({}, function (err) { + db.collection('objects').deleteMany({}, function (err) { if (err) { return callback(err); } @@ -35,7 +35,7 @@ module.exports = function (db, module) { if (!key) { return callback(); } - db.collection('objects').remove({ _key: key }, function (err) { + db.collection('objects').deleteMany({ _key: key }, function (err) { if (err) { return callback(err); } @@ -49,7 +49,7 @@ module.exports = function (db, module) { if (!Array.isArray(keys) || !keys.length) { return callback(); } - db.collection('objects').remove({ _key: { $in: keys } }, function (err) { + db.collection('objects').deleteMany({ _key: { $in: keys } }, function (err) { if (err) { return callback(err); } @@ -97,14 +97,14 @@ module.exports = function (db, module) { if (!key) { return callback(); } - db.collection('objects').findAndModify({ _key: key }, {}, { $inc: { data: 1 } }, { new: true, upsert: true }, function (err, result) { + db.collection('objects').findOneAndUpdate({ _key: key }, { $inc: { data: 1 } }, { returnOriginal: false, upsert: true }, function (err, result) { callback(err, result && result.value ? result.value.data : null); }); }; module.rename = function (oldKey, newKey, callback) { callback = callback || helpers.noop; - db.collection('objects').update({ _key: oldKey }, { $set: { _key: newKey } }, { multi: true }, function (err) { + db.collection('objects').updateMany({ _key: oldKey }, { $set: { _key: newKey } }, function (err) { if (err) { return callback(err); } diff --git a/src/database/mongo/sets.js b/src/database/mongo/sets.js index 4f807b3922..b66772639c 100644 --- a/src/database/mongo/sets.js +++ b/src/database/mongo/sets.js @@ -13,7 +13,7 @@ module.exports = function (db, module) { array[index] = helpers.valueToString(element); }); - db.collection('objects').update({ + db.collection('objects').updateOne({ _key: key, }, { $addToSet: { @@ -74,7 +74,7 @@ module.exports = function (db, module) { callback(err); }); } else { - db.collection('objects').update({ _key: key }, { $pullAll: { members: value } }, function (err) { + db.collection('objects').updateOne({ _key: key }, { $pullAll: { members: value } }, function (err) { callback(err); }); } @@ -87,7 +87,7 @@ module.exports = function (db, module) { } value = helpers.valueToString(value); - db.collection('objects').update({ _key: { $in: keys } }, { $pull: { members: value } }, { multi: true }, function (err) { + db.collection('objects').updateMany({ _key: { $in: keys } }, { $pull: { members: value } }, function (err) { callback(err); }); }; @@ -98,7 +98,7 @@ module.exports = function (db, module) { } value = helpers.valueToString(value); - db.collection('objects').findOne({ _key: key, members: value }, { _id: 0, members: 0 }, function (err, item) { + db.collection('objects').findOne({ _key: key, members: value }, { projection: { _id: 0, members: 0 } }, function (err, item) { callback(err, item !== null && item !== undefined); }); }; @@ -112,7 +112,7 @@ module.exports = function (db, module) { values[i] = helpers.valueToString(values[i]); } - db.collection('objects').findOne({ _key: key }, { _id: 0, _key: 0 }, function (err, items) { + db.collection('objects').findOne({ _key: key }, { projection: { _id: 0, _key: 0 } }, function (err, items) { if (err) { return callback(err); } @@ -131,7 +131,7 @@ module.exports = function (db, module) { } value = helpers.valueToString(value); - db.collection('objects').find({ _key: { $in: sets }, members: value }, { _id: 0, members: 0 }).toArray(function (err, result) { + db.collection('objects').find({ _key: { $in: sets }, members: value }, { projection: { _id: 0, members: 0 } }).toArray(function (err, result) { if (err) { return callback(err); } @@ -184,7 +184,7 @@ module.exports = function (db, module) { if (!key) { return callback(null, 0); } - db.collection('objects').findOne({ _key: key }, { _id: 0 }, function (err, data) { + db.collection('objects').findOne({ _key: key }, { projection: { _id: 0 } }, function (err, data) { callback(err, data ? data.members.length : 0); }); }; diff --git a/src/database/mongo/sorted.js b/src/database/mongo/sorted.js index d4eb7a8522..9ec82f5a68 100644 --- a/src/database/mongo/sorted.js +++ b/src/database/mongo/sorted.js @@ -32,9 +32,9 @@ module.exports = function (db, module) { return callback(); } - var fields = { _id: 0, value: 1 }; - if (withScores) { - fields.score = 1; + var fields = { _id: 0, _key: 0 }; + if (!withScores) { + fields.score = 0; } if (Array.isArray(key)) { @@ -62,7 +62,7 @@ module.exports = function (db, module) { limit = 0; } - db.collection('objects').find({ _key: key }, { fields: fields }) + db.collection('objects').find({ _key: key }, { projection: fields }) .limit(limit) .skip(start) .sort({ score: sort }) @@ -70,6 +70,7 @@ module.exports = function (db, module) { if (err || !data) { return callback(err); } + if (reverse) { data.reverse(); } @@ -117,12 +118,12 @@ module.exports = function (db, module) { query.score.$lte = max; } - var fields = { _id: 0, value: 1 }; - if (withScores) { - fields.score = 1; + var fields = { _id: 0, _key: 0 }; + if (!withScores) { + fields.score = 0; } - db.collection('objects').find(query, { fields: fields }) + db.collection('objects').find(query, { projection: fields }) .limit(count) .skip(start) .sort({ score: sort }) @@ -155,7 +156,7 @@ module.exports = function (db, module) { query.score.$lte = max; } - db.collection('objects').count(query, function (err, count) { + db.collection('objects').countDocuments(query, function (err, count) { callback(err, count || 0); }); }; @@ -164,7 +165,7 @@ module.exports = function (db, module) { if (!key) { return callback(null, 0); } - db.collection('objects').count({ _key: key }, function (err, count) { + db.collection('objects').countDocuments({ _key: key }, function (err, count) { count = parseInt(count, 10); callback(err, count || 0); }); @@ -220,7 +221,7 @@ module.exports = function (db, module) { return callback(err, null); } - db.collection('objects').count({ + db.collection('objects').countDocuments({ $or: [ { _key: key, @@ -273,7 +274,7 @@ module.exports = function (db, module) { return callback(null, null); } value = helpers.valueToString(value); - db.collection('objects').findOne({ _key: key, value: value }, { fields: { _id: 0, score: 1 } }, function (err, result) { + db.collection('objects').findOne({ _key: key, value: value }, { projection: { _id: 0, _key: 0, value: 0 } }, function (err, result) { callback(err, result ? result.score : null); }); }; @@ -283,7 +284,7 @@ module.exports = function (db, module) { return callback(); } value = helpers.valueToString(value); - db.collection('objects').find({ _key: { $in: keys }, value: value }, { _id: 0, _key: 1, score: 1 }).toArray(function (err, result) { + db.collection('objects').find({ _key: { $in: keys }, value: value }, { projection: { _id: 0, value: 0 } }).toArray(function (err, result) { if (err) { return callback(err); } @@ -306,7 +307,7 @@ module.exports = function (db, module) { return callback(null, null); } values = values.map(helpers.valueToString); - db.collection('objects').find({ _key: key, value: { $in: values } }, { _id: 0, value: 1, score: 1 }).toArray(function (err, result) { + db.collection('objects').find({ _key: key, value: { $in: values } }, { projection: { _id: 0, _key: 0 } }).toArray(function (err, result) { if (err) { return callback(err); } @@ -333,7 +334,7 @@ module.exports = function (db, module) { return callback(); } value = helpers.valueToString(value); - db.collection('objects').findOne({ _key: key, value: value }, { _id: 0, value: 1 }, function (err, result) { + db.collection('objects').findOne({ _key: key, value: value }, { projection: { _id: 0, _key: 0, score: 0 } }, function (err, result) { callback(err, !!result); }); }; @@ -343,17 +344,19 @@ module.exports = function (db, module) { return callback(); } values = values.map(helpers.valueToString); - db.collection('objects').find({ _key: key, value: { $in: values } }, { fields: { _id: 0, value: 1 } }).toArray(function (err, results) { + db.collection('objects').find({ _key: key, value: { $in: values } }, { projection: { _id: 0, _key: 0, score: 0 } }).toArray(function (err, results) { if (err) { return callback(err); } - - results = results.map(function (item) { - return item.value; + var isMember = {}; + results.forEach(function (item) { + if (item) { + isMember[item.value] = true; + } }); values = values.map(function (value) { - return results.indexOf(value) !== -1; + return !!isMember[value]; }); callback(null, values); }); @@ -364,17 +367,19 @@ module.exports = function (db, module) { return callback(); } value = helpers.valueToString(value); - db.collection('objects').find({ _key: { $in: keys }, value: value }, { fields: { _id: 0, _key: 1, value: 1 } }).toArray(function (err, results) { + db.collection('objects').find({ _key: { $in: keys }, value: value }, { projection: { _id: 0, score: 0 } }).toArray(function (err, results) { if (err) { return callback(err); } - - results = results.map(function (item) { - return item._key; + var isMember = {}; + results.forEach(function (item) { + if (item) { + isMember[item._key] = true; + } }); results = keys.map(function (key) { - return results.indexOf(key) !== -1; + return !!isMember[key]; }); callback(null, results); }); @@ -384,7 +389,7 @@ module.exports = function (db, module) { if (!Array.isArray(keys) || !keys.length) { return callback(null, []); } - db.collection('objects').find({ _key: { $in: keys } }, { _id: 0, _key: 1, value: 1 }).sort({ score: 1 }).toArray(function (err, data) { + db.collection('objects').find({ _key: { $in: keys } }, { projection: { _id: 0, score: 0 } }).sort({ score: 1 }).toArray(function (err, data) { if (err) { return callback(err); } @@ -412,7 +417,7 @@ module.exports = function (db, module) { value = helpers.valueToString(value); data.score = parseFloat(increment); - db.collection('objects').findAndModify({ _key: key, value: value }, {}, { $inc: data }, { new: true, upsert: true }, function (err, result) { + db.collection('objects').findOneAndUpdate({ _key: key, value: value }, { $inc: data }, { returnOriginal: false, upsert: true }, function (err, result) { // if there is duplicate key error retry the upsert // https://github.com/NodeBB/NodeBB/issues/4467 // https://jira.mongodb.org/browse/SERVER-14322 @@ -448,7 +453,7 @@ module.exports = function (db, module) { var query = { _key: key }; buildLexQuery(query, min, max); - db.collection('objects').find(query, { _id: 0, value: 1 }) + db.collection('objects').find(query, { projection: { _id: 0, _key: 0, score: 0 } }) .sort({ value: sort }) .skip(start) .limit(count === -1 ? 0 : count) @@ -469,7 +474,7 @@ module.exports = function (db, module) { var query = { _key: key }; buildLexQuery(query, min, max); - db.collection('objects').remove(query, function (err) { + db.collection('objects').deleteMany(query, function (err) { callback(err); }); }; @@ -499,13 +504,12 @@ module.exports = function (db, module) { module.processSortedSet = function (setKey, processFn, options, callback) { var done = false; var ids = []; - var project = { _id: 0, value: 1 }; - if (options.withScores) { - project.score = 1; + var project = { _id: 0, _key: 0 }; + if (!options.withScores) { + project.score = 0; } - var cursor = db.collection('objects').find({ _key: setKey }) + var cursor = db.collection('objects').find({ _key: setKey }, { projection: project }) .sort({ score: 1 }) - .project(project) .batchSize(options.batch); async.whilst( diff --git a/src/database/mongo/sorted/add.js b/src/database/mongo/sorted/add.js index b90501feee..e71bb0568a 100644 --- a/src/database/mongo/sorted/add.js +++ b/src/database/mongo/sorted/add.js @@ -14,7 +14,7 @@ module.exports = function (db, module) { value = helpers.valueToString(value); - db.collection('objects').update({ _key: key, value: value }, { $set: { score: parseFloat(score) } }, { upsert: true, w: 1 }, function (err) { + db.collection('objects').updateOne({ _key: key, value: value }, { $set: { score: parseFloat(score) } }, { upsert: true, w: 1 }, function (err) { if (err && err.message.startsWith('E11000 duplicate key error')) { return process.nextTick(module.sortedSetAdd, key, score, value, callback); } diff --git a/src/database/mongo/sorted/remove.js b/src/database/mongo/sorted/remove.js index c9bf121e10..ab4fd90b33 100644 --- a/src/database/mongo/sorted/remove.js +++ b/src/database/mongo/sorted/remove.js @@ -11,17 +11,21 @@ module.exports = function (db, module) { if (!key) { return callback(); } - if (Array.isArray(key) && Array.isArray(value)) { - db.collection('objects').remove({ _key: { $in: key }, value: { $in: value } }, done); - } else if (Array.isArray(value)) { + + if (Array.isArray(value)) { value = value.map(helpers.valueToString); - db.collection('objects').remove({ _key: key, value: { $in: value } }, done); - } else if (Array.isArray(key)) { - value = helpers.valueToString(value); - db.collection('objects').remove({ _key: { $in: key }, value: value }, done); } else { value = helpers.valueToString(value); - db.collection('objects').remove({ _key: key, value: value }, done); + } + + if (Array.isArray(key) && Array.isArray(value)) { + db.collection('objects').deleteMany({ _key: { $in: key }, value: { $in: value } }, done); + } else if (Array.isArray(value)) { + db.collection('objects').deleteMany({ _key: key, value: { $in: value } }, done); + } else if (Array.isArray(key)) { + db.collection('objects').deleteMany({ _key: { $in: key }, value: value }, done); + } else { + db.collection('objects').deleteOne({ _key: key, value: value }, done); } }; @@ -32,7 +36,7 @@ module.exports = function (db, module) { } value = helpers.valueToString(value); - db.collection('objects').remove({ _key: { $in: keys }, value: value }, function (err) { + db.collection('objects').deleteMany({ _key: { $in: keys }, value: value }, function (err) { callback(err); }); }; @@ -52,7 +56,7 @@ module.exports = function (db, module) { query.score.$lte = parseFloat(max); } - db.collection('objects').remove(query, function (err) { + db.collection('objects').deleteMany(query, function (err) { callback(err); }); }; diff --git a/src/database/mongo/transaction.js b/src/database/mongo/transaction.js new file mode 100644 index 0000000000..75ea5fbaa2 --- /dev/null +++ b/src/database/mongo/transaction.js @@ -0,0 +1,8 @@ +'use strict'; + +module.exports = function (db, module) { + // TODO + module.transaction = function (perform, callback) { + perform(db, callback); + }; +}; diff --git a/src/database/postgres.js b/src/database/postgres.js new file mode 100644 index 0000000000..f06c962c22 --- /dev/null +++ b/src/database/postgres.js @@ -0,0 +1,465 @@ +'use strict'; + +var winston = require('winston'); +var async = require('async'); +var nconf = require('nconf'); +var session = require('express-session'); +var _ = require('lodash'); +var semver = require('semver'); +var dbNamespace = require('continuation-local-storage').createNamespace('postgres'); +var db; + +var postgresModule = module.exports; + +postgresModule.questions = [ + { + name: 'postgres:host', + description: 'Host IP or address of your PostgreSQL instance', + default: nconf.get('postgres:host') || '127.0.0.1', + }, + { + name: 'postgres:port', + description: 'Host port of your PostgreSQL instance', + default: nconf.get('postgres:port') || 5432, + }, + { + name: 'postgres:username', + description: 'PostgreSQL username', + default: nconf.get('postgres:username') || '', + }, + { + name: 'postgres:password', + description: 'Password of your PostgreSQL database', + hidden: true, + default: nconf.get('postgres:password') || '', + before: function (value) { value = value || nconf.get('postgres:password') || ''; return value; }, + }, + { + name: 'postgres:database', + description: 'PostgreSQL database name', + default: nconf.get('postgres:database') || 'nodebb', + }, +]; + +postgresModule.helpers = postgresModule.helpers || {}; +postgresModule.helpers.postgres = require('./postgres/helpers'); + +postgresModule.getConnectionOptions = function () { + // Sensible defaults for PostgreSQL, if not set + if (!nconf.get('postgres:host')) { + nconf.set('postgres:host', '127.0.0.1'); + } + if (!nconf.get('postgres:port')) { + nconf.set('postgres:port', 5432); + } + if (!nconf.get('postgres:database')) { + nconf.set('postgres:database', 'nodebb'); + } + + var connOptions = { + host: nconf.get('postgres:host'), + port: nconf.get('postgres:port'), + user: nconf.get('postgres:username'), + password: nconf.get('postgres:password'), + database: nconf.get('postgres:database'), + }; + + return _.merge(connOptions, nconf.get('postgres:options') || {}); +}; + +postgresModule.init = function (callback) { + callback = callback || function () { }; + + var Pool = require('pg').Pool; + + var connOptions = postgresModule.getConnectionOptions(); + + db = new Pool(connOptions); + + db.on('connect', function (client) { + var realQuery = client.query; + client.query = function () { + var args = Array.prototype.slice.call(arguments, 0); + if (dbNamespace.active && typeof args[args.length - 1] === 'function') { + args[args.length - 1] = dbNamespace.bind(args[args.length - 1]); + } + return realQuery.apply(client, args); + }; + }); + + db.connect(function (err, client, release) { + if (err) { + winston.error('NodeBB could not connect to your PostgreSQL database. PostgreSQL returned the following error: ' + err.message); + return callback(err); + } + + postgresModule.pool = db; + Object.defineProperty(postgresModule, 'client', { + get: function () { + return (dbNamespace.active && dbNamespace.get('db')) || db; + }, + configurable: true, + }); + + var wrappedDB = { + connect: function () { + return postgresModule.pool.connect.apply(postgresModule.pool, arguments); + }, + query: function () { + return postgresModule.client.query.apply(postgresModule.client, arguments); + }, + }; + + checkUpgrade(client, function (err) { + release(); + if (err) { + return callback(err); + } + + require('./postgres/main')(wrappedDB, postgresModule); + require('./postgres/hash')(wrappedDB, postgresModule); + require('./postgres/sets')(wrappedDB, postgresModule); + require('./postgres/sorted')(wrappedDB, postgresModule); + require('./postgres/list')(wrappedDB, postgresModule); + require('./postgres/transaction')(db, dbNamespace, postgresModule); + + postgresModule.async = require('../promisify')(postgresModule, ['client', 'sessionStore', 'pool']); + + callback(); + }); + }); +}; + +function checkUpgrade(client, callback) { + client.query(` +SELECT EXISTS(SELECT * + FROM "information_schema"."columns" + WHERE "table_schema" = 'public' + AND "table_name" = 'objects' + AND "column_name" = 'data') a, + EXISTS(SELECT * + FROM "information_schema"."columns" + WHERE "table_schema" = 'public' + AND "table_name" = 'legacy_hash' + AND "column_name" = '_key') b`, function (err, res) { + if (err) { + return callback(err); + } + + if (res.rows[0].b) { + return callback(null); + } + + var query = client.query.bind(client); + + async.series([ + async.apply(query, `BEGIN`), + async.apply(query, ` +CREATE TYPE LEGACY_OBJECT_TYPE AS ENUM ( + 'hash', 'zset', 'set', 'list', 'string' +)`), + async.apply(query, ` +CREATE TABLE "legacy_object" ( + "_key" TEXT NOT NULL + PRIMARY KEY, + "type" LEGACY_OBJECT_TYPE NOT NULL, + "expireAt" TIMESTAMPTZ DEFAULT NULL, + UNIQUE ( "_key", "type" ) +)`), + async.apply(query, ` +CREATE TABLE "legacy_hash" ( + "_key" TEXT NOT NULL + PRIMARY KEY, + "data" JSONB NOT NULL, + "type" LEGACY_OBJECT_TYPE NOT NULL + DEFAULT 'hash'::LEGACY_OBJECT_TYPE + CHECK ( "type" = 'hash' ), + CONSTRAINT "fk__legacy_hash__key" + FOREIGN KEY ("_key", "type") + REFERENCES "legacy_object"("_key", "type") + ON UPDATE CASCADE + ON DELETE CASCADE +)`), + async.apply(query, ` +CREATE TABLE "legacy_zset" ( + "_key" TEXT NOT NULL, + "value" TEXT NOT NULL, + "score" NUMERIC NOT NULL, + "type" LEGACY_OBJECT_TYPE NOT NULL + DEFAULT 'zset'::LEGACY_OBJECT_TYPE + CHECK ( "type" = 'zset' ), + PRIMARY KEY ("_key", "value"), + CONSTRAINT "fk__legacy_zset__key" + FOREIGN KEY ("_key", "type") + REFERENCES "legacy_object"("_key", "type") + ON UPDATE CASCADE + ON DELETE CASCADE +)`), + async.apply(query, ` +CREATE TABLE "legacy_set" ( + "_key" TEXT NOT NULL, + "member" TEXT NOT NULL, + "type" LEGACY_OBJECT_TYPE NOT NULL + DEFAULT 'set'::LEGACY_OBJECT_TYPE + CHECK ( "type" = 'set' ), + PRIMARY KEY ("_key", "member"), + CONSTRAINT "fk__legacy_set__key" + FOREIGN KEY ("_key", "type") + REFERENCES "legacy_object"("_key", "type") + ON UPDATE CASCADE + ON DELETE CASCADE +)`), + async.apply(query, ` +CREATE TABLE "legacy_list" ( + "_key" TEXT NOT NULL + PRIMARY KEY, + "array" TEXT[] NOT NULL, + "type" LEGACY_OBJECT_TYPE NOT NULL + DEFAULT 'list'::LEGACY_OBJECT_TYPE + CHECK ( "type" = 'list' ), + CONSTRAINT "fk__legacy_list__key" + FOREIGN KEY ("_key", "type") + REFERENCES "legacy_object"("_key", "type") + ON UPDATE CASCADE + ON DELETE CASCADE +)`), + async.apply(query, ` +CREATE TABLE "legacy_string" ( + "_key" TEXT NOT NULL + PRIMARY KEY, + "data" TEXT NOT NULL, + "type" LEGACY_OBJECT_TYPE NOT NULL + DEFAULT 'string'::LEGACY_OBJECT_TYPE + CHECK ( "type" = 'string' ), + CONSTRAINT "fk__legacy_string__key" + FOREIGN KEY ("_key", "type") + REFERENCES "legacy_object"("_key", "type") + ON UPDATE CASCADE + ON DELETE CASCADE +)`), + function (next) { + if (!res.rows[0].a) { + return next(); + } + async.series([ + async.apply(query, ` +INSERT INTO "legacy_object" ("_key", "type", "expireAt") +SELECT DISTINCT "data"->>'_key', + CASE WHEN (SELECT COUNT(*) + FROM jsonb_object_keys("data" - 'expireAt')) = 2 + THEN CASE WHEN ("data" ? 'value') + OR ("data" ? 'data') + THEN 'string' + WHEN "data" ? 'array' + THEN 'list' + WHEN "data" ? 'members' + THEN 'set' + ELSE 'hash' + END + WHEN (SELECT COUNT(*) + FROM jsonb_object_keys("data" - 'expireAt')) = 3 + THEN CASE WHEN ("data" ? 'value') + AND ("data" ? 'score') + THEN 'zset' + ELSE 'hash' + END + ELSE 'hash' + END::LEGACY_OBJECT_TYPE, + CASE WHEN ("data" ? 'expireAt') + THEN to_timestamp(("data"->>'expireAt')::double precision / 1000) + ELSE NULL + END + FROM "objects"`), + async.apply(query, ` +INSERT INTO "legacy_hash" ("_key", "data") +SELECT "data"->>'_key', + "data" - '_key' - 'expireAt' + FROM "objects" + WHERE CASE WHEN (SELECT COUNT(*) + FROM jsonb_object_keys("data" - 'expireAt')) = 2 + THEN NOT (("data" ? 'value') + OR ("data" ? 'data') + OR ("data" ? 'members') + OR ("data" ? 'array')) + WHEN (SELECT COUNT(*) + FROM jsonb_object_keys("data" - 'expireAt')) = 3 + THEN NOT (("data" ? 'value') + AND ("data" ? 'score')) + ELSE TRUE + END`), + async.apply(query, ` +INSERT INTO "legacy_zset" ("_key", "value", "score") +SELECT "data"->>'_key', + "data"->>'value', + ("data"->>'score')::NUMERIC + FROM "objects" + WHERE (SELECT COUNT(*) + FROM jsonb_object_keys("data" - 'expireAt')) = 3 + AND ("data" ? 'value') + AND ("data" ? 'score')`), + async.apply(query, ` +INSERT INTO "legacy_set" ("_key", "member") +SELECT "data"->>'_key', + jsonb_array_elements_text("data"->'members') + FROM "objects" + WHERE (SELECT COUNT(*) + FROM jsonb_object_keys("data" - 'expireAt')) = 2 + AND ("data" ? 'members')`), + async.apply(query, ` +INSERT INTO "legacy_list" ("_key", "array") +SELECT "data"->>'_key', + ARRAY(SELECT t + FROM jsonb_array_elements_text("data"->'list') WITH ORDINALITY l(t, i) + ORDER BY i ASC) + FROM "objects" + WHERE (SELECT COUNT(*) + FROM jsonb_object_keys("data" - 'expireAt')) = 2 + AND ("data" ? 'array')`), + async.apply(query, ` +INSERT INTO "legacy_string" ("_key", "data") +SELECT "data"->>'_key', + CASE WHEN "data" ? 'value' + THEN "data"->>'value' + ELSE "data"->>'data' + END + FROM "objects" + WHERE (SELECT COUNT(*) + FROM jsonb_object_keys("data" - 'expireAt')) = 2 + AND (("data" ? 'value') + OR ("data" ? 'data'))`), + async.apply(query, `DROP TABLE "objects" CASCADE`), + async.apply(query, `DROP FUNCTION "fun__objects__expireAt"() CASCADE`), + ], next); + }, + async.apply(query, ` +CREATE VIEW "legacy_object_live" AS +SELECT "_key", "type" + FROM "legacy_object" + WHERE "expireAt" IS NULL + OR "expireAt" > CURRENT_TIMESTAMP`), + ], function (err) { + query(err ? `ROLLBACK` : `COMMIT`, function (err1) { + callback(err1 || err); + }); + }); + }); +} + +postgresModule.initSessionStore = function (callback) { + var meta = require('../meta'); + var sessionStore; + + var ttl = meta.getSessionTTLSeconds(); + + if (nconf.get('redis')) { + sessionStore = require('connect-redis')(session); + var rdb = require('./redis'); + rdb.client = rdb.connect(); + + postgresModule.sessionStore = new sessionStore({ + client: rdb.client, + ttl: ttl, + }); + + return callback(); + } + + function done() { + sessionStore = require('connect-pg-simple')(session); + postgresModule.sessionStore = new sessionStore({ + pool: db, + ttl: ttl, + pruneSessionInterval: nconf.get('isPrimary') === 'true' ? 60 : false, + }); + + callback(); + } + + if (nconf.get('isPrimary') !== 'true') { + return done(); + } + + db.query(` +CREATE TABLE IF NOT EXISTS "session" ( + "sid" CHAR(32) NOT NULL + COLLATE "C" + PRIMARY KEY, + "sess" JSONB NOT NULL, + "expire" TIMESTAMPTZ NOT NULL +) WITHOUT OIDS; + +CREATE INDEX IF NOT EXISTS "session_expire_idx" ON "session"("expire"); + +ALTER TABLE "session" + ALTER "sid" SET STORAGE MAIN, + CLUSTER ON "session_expire_idx";`, function (err) { + if (err) { + return callback(err); + } + + done(); + }); +}; + +postgresModule.createIndices = function (callback) { + if (!postgresModule.pool) { + winston.warn('[database/createIndices] database not initialized'); + return callback(); + } + + var query = postgresModule.pool.query.bind(postgresModule.pool); + + winston.info('[database] Checking database indices.'); + async.series([ + async.apply(query, `CREATE INDEX IF NOT EXISTS "idx__legacy_zset__key__score" ON "legacy_zset"("_key" ASC, "score" DESC)`), + async.apply(query, `CREATE INDEX IF NOT EXISTS "idx__legacy_object__expireAt" ON "legacy_object"("expireAt" ASC)`), + ], function (err) { + if (err) { + winston.error('Error creating index ' + err.message); + return callback(err); + } + winston.info('[database] Checking database indices done!'); + callback(); + }); +}; + +postgresModule.checkCompatibility = function (callback) { + var postgresPkg = require('pg/package.json'); + postgresModule.checkCompatibilityVersion(postgresPkg.version, callback); +}; + +postgresModule.checkCompatibilityVersion = function (version, callback) { + if (semver.lt(version, '7.0.0')) { + return callback(new Error('The `pg` package is out-of-date, please run `./nodebb setup` again.')); + } + + callback(); +}; + +postgresModule.info = function (db, callback) { + if (!db) { + return callback(); + } + + db.query(` +SELECT true "postgres", + current_setting('server_version') "version", + EXTRACT(EPOCH FROM NOW() - pg_postmaster_start_time()) * 1000 "uptime"`, function (err, res) { + if (err) { + return callback(err); + } + callback(null, res.rows[0]); + }); +}; + +postgresModule.close = function (callback) { + callback = callback || function () {}; + db.end(callback); +}; + +postgresModule.socketAdapter = function () { + var postgresAdapter = require('socket.io-adapter-postgres'); + return postgresAdapter(postgresModule.getConnectionOptions(), { + pubClient: postgresModule.pool, + }); +}; diff --git a/src/database/postgres/hash.js b/src/database/postgres/hash.js new file mode 100644 index 0000000000..aeb794185b --- /dev/null +++ b/src/database/postgres/hash.js @@ -0,0 +1,391 @@ +'use strict'; + +var async = require('async'); + +module.exports = function (db, module) { + var helpers = module.helpers.postgres; + + module.setObject = function (key, data, callback) { + callback = callback || helpers.noop; + + if (!key || !data) { + return callback(); + } + + if (data.hasOwnProperty('')) { + delete data['']; + } + + module.transaction(function (tx, done) { + var query = tx.client.query.bind(tx.client); + + async.series([ + async.apply(helpers.ensureLegacyObjectType, tx.client, key, 'hash'), + async.apply(query, { + name: 'setObject', + text: ` +INSERT INTO "legacy_hash" ("_key", "data") +VALUES ($1::TEXT, $2::TEXT::JSONB) + ON CONFLICT ("_key") + DO UPDATE SET "data" = "legacy_hash"."data" || $2::TEXT::JSONB`, + values: [key, JSON.stringify(data)], + }), + ], function (err) { + done(err); + }); + }, callback); + }; + + module.setObjectField = function (key, field, value, callback) { + callback = callback || helpers.noop; + + if (!field) { + return callback(); + } + + module.transaction(function (tx, done) { + var query = tx.client.query.bind(tx.client); + + async.series([ + async.apply(helpers.ensureLegacyObjectType, tx.client, key, 'hash'), + async.apply(query, { + name: 'setObjectField', + text: ` +INSERT INTO "legacy_hash" ("_key", "data") +VALUES ($1::TEXT, jsonb_build_object($2::TEXT, $3::TEXT::JSONB)) + ON CONFLICT ("_key") + DO UPDATE SET "data" = jsonb_set("legacy_hash"."data", ARRAY[$2::TEXT], $3::TEXT::JSONB)`, + values: [key, field, JSON.stringify(value)], + }), + ], function (err) { + done(err); + }); + }, callback); + }; + + module.getObject = function (key, callback) { + if (!key) { + return callback(); + } + + db.query({ + name: 'getObject', + text: ` +SELECT h."data" + FROM "legacy_object_live" o + INNER JOIN "legacy_hash" h + ON o."_key" = h."_key" + AND o."type" = h."type" + WHERE o."_key" = $1::TEXT + LIMIT 1`, + values: [key], + }, function (err, res) { + if (err) { + return callback(err); + } + + if (res.rows.length) { + return callback(null, res.rows[0].data); + } + + callback(null, null); + }); + }; + + module.getObjects = function (keys, callback) { + if (!Array.isArray(keys) || !keys.length) { + return callback(null, []); + } + + db.query({ + name: 'getObjects', + text: ` +SELECT h."data" + FROM UNNEST($1::TEXT[]) WITH ORDINALITY k("_key", i) + LEFT OUTER JOIN "legacy_object_live" o + ON o."_key" = k."_key" + LEFT OUTER JOIN "legacy_hash" h + ON o."_key" = h."_key" + AND o."type" = h."type" + ORDER BY k.i ASC`, + values: [keys], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, res.rows.map(function (row) { + return row.data; + })); + }); + }; + + module.getObjectField = function (key, field, callback) { + if (!key) { + return callback(); + } + + db.query({ + name: 'getObjectField', + text: ` +SELECT h."data"->>$2::TEXT f + FROM "legacy_object_live" o + INNER JOIN "legacy_hash" h + ON o."_key" = h."_key" + AND o."type" = h."type" + WHERE o."_key" = $1::TEXT + LIMIT 1`, + values: [key, field], + }, function (err, res) { + if (err) { + return callback(err); + } + + if (res.rows.length) { + return callback(null, res.rows[0].f); + } + + callback(null, null); + }); + }; + + module.getObjectFields = function (key, fields, callback) { + if (!key) { + return callback(); + } + + db.query({ + name: 'getObjectFields', + text: ` +SELECT (SELECT jsonb_object_agg(f, d."value") + FROM UNNEST($2::TEXT[]) f + LEFT OUTER JOIN jsonb_each(h."data") d + ON d."key" = f) d + FROM "legacy_object_live" o + INNER JOIN "legacy_hash" h + ON o."_key" = h."_key" + AND o."type" = h."type" + WHERE o."_key" = $1::TEXT`, + values: [key, fields], + }, function (err, res) { + if (err) { + return callback(err); + } + + if (res.rows.length) { + return callback(null, res.rows[0].d); + } + + var obj = {}; + fields.forEach(function (f) { + obj[f] = null; + }); + + callback(null, obj); + }); + }; + + module.getObjectsFields = function (keys, fields, callback) { + if (!Array.isArray(keys) || !keys.length) { + return callback(null, []); + } + + db.query({ + name: 'getObjectsFields', + text: ` +SELECT (SELECT jsonb_object_agg(f, d."value") + FROM UNNEST($2::TEXT[]) f + LEFT OUTER JOIN jsonb_each(h."data") d + ON d."key" = f) d + FROM UNNEST($1::text[]) WITH ORDINALITY k("_key", i) + LEFT OUTER JOIN "legacy_object_live" o + ON o."_key" = k."_key" + LEFT OUTER JOIN "legacy_hash" h + ON o."_key" = h."_key" + AND o."type" = h."type" + ORDER BY k.i ASC`, + values: [keys, fields], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, res.rows.map(function (row) { + return row.d; + })); + }); + }; + + module.getObjectKeys = function (key, callback) { + if (!key) { + return callback(); + } + + db.query({ + name: 'getObjectKeys', + text: ` +SELECT ARRAY(SELECT jsonb_object_keys(h."data")) k + FROM "legacy_object_live" o + INNER JOIN "legacy_hash" h + ON o."_key" = h."_key" + AND o."type" = h."type" + WHERE o."_key" = $1::TEXT + LIMIT 1`, + values: [key], + }, function (err, res) { + if (err) { + return callback(err); + } + + if (res.rows.length) { + return callback(null, res.rows[0].k); + } + + callback(null, []); + }); + }; + + module.getObjectValues = function (key, callback) { + module.getObject(key, function (err, data) { + if (err) { + return callback(err); + } + + var values = []; + + if (data) { + for (var key in data) { + if (data.hasOwnProperty(key)) { + values.push(data[key]); + } + } + } + + callback(null, values); + }); + }; + + module.isObjectField = function (key, field, callback) { + if (!key) { + return callback(); + } + + db.query({ + name: 'isObjectField', + text: ` +SELECT (h."data" ? $2::TEXT AND h."data"->>$2::TEXT IS NOT NULL) b + FROM "legacy_object_live" o + INNER JOIN "legacy_hash" h + ON o."_key" = h."_key" + AND o."type" = h."type" + WHERE o."_key" = $1::TEXT + LIMIT 1`, + values: [key, field], + }, function (err, res) { + if (err) { + return callback(err); + } + + if (res.rows.length) { + return callback(null, res.rows[0].b); + } + + callback(null, false); + }); + }; + + module.isObjectFields = function (key, fields, callback) { + if (!key) { + return callback(); + } + + module.getObjectFields(key, fields, function (err, data) { + if (err) { + return callback(err); + } + + if (!data) { + return callback(null, fields.map(function () { + return false; + })); + } + + callback(null, fields.map(function (field) { + return data.hasOwnProperty(field) && data[field] !== null; + })); + }); + }; + + module.deleteObjectField = function (key, field, callback) { + module.deleteObjectFields(key, [field], callback); + }; + + module.deleteObjectFields = function (key, fields, callback) { + callback = callback || helpers.noop; + if (!key || !Array.isArray(fields) || !fields.length) { + return callback(); + } + + db.query({ + name: 'deleteObjectFields', + text: ` +UPDATE "legacy_hash" + SET "data" = COALESCE((SELECT jsonb_object_agg("key", "value") + FROM jsonb_each("data") + WHERE "key" <> ALL ($2::TEXT[])), '{}') + WHERE "_key" = $1::TEXT`, + values: [key, fields], + }, function (err) { + callback(err); + }); + }; + + module.incrObjectField = function (key, field, callback) { + module.incrObjectFieldBy(key, field, 1, callback); + }; + + module.decrObjectField = function (key, field, callback) { + module.incrObjectFieldBy(key, field, -1, callback); + }; + + module.incrObjectFieldBy = function (key, field, value, callback) { + callback = callback || helpers.noop; + value = parseInt(value, 10); + + if (!key || isNaN(value)) { + return callback(null, null); + } + + module.transaction(function (tx, done) { + var query = tx.client.query.bind(tx.client); + + async.waterfall([ + async.apply(Array.isArray(key) ? helpers.ensureLegacyObjectsType : helpers.ensureLegacyObjectType, tx.client, key, 'hash'), + async.apply(query, Array.isArray(key) ? { + name: 'incrObjectFieldByMulti', + text: ` +INSERT INTO "legacy_hash" ("_key", "data") +SELECT UNNEST($1::TEXT[]), jsonb_build_object($2::TEXT, $3::NUMERIC) + ON CONFLICT ("_key") + DO UPDATE SET "data" = jsonb_set("legacy_hash"."data", ARRAY[$2::TEXT], to_jsonb(COALESCE(("legacy_hash"."data"->>$2::TEXT)::NUMERIC, 0) + $3::NUMERIC)) +RETURNING ("data"->>$2::TEXT)::NUMERIC v`, + values: [key, field, value], + } : { + name: 'incrObjectFieldBy', + text: ` +INSERT INTO "legacy_hash" ("_key", "data") +VALUES ($1::TEXT, jsonb_build_object($2::TEXT, $3::NUMERIC)) + ON CONFLICT ("_key") + DO UPDATE SET "data" = jsonb_set("legacy_hash"."data", ARRAY[$2::TEXT], to_jsonb(COALESCE(("legacy_hash"."data"->>$2::TEXT)::NUMERIC, 0) + $3::NUMERIC)) +RETURNING ("data"->>$2::TEXT)::NUMERIC v`, + values: [key, field, value], + }), + function (res, next) { + next(null, Array.isArray(key) ? res.rows.map(function (r) { + return parseFloat(r.v); + }) : parseFloat(res.rows[0].v)); + }, + ], done); + }, callback); + }; +}; diff --git a/src/database/postgres/helpers.js b/src/database/postgres/helpers.js new file mode 100644 index 0000000000..1b9a4d2cf6 --- /dev/null +++ b/src/database/postgres/helpers.js @@ -0,0 +1,139 @@ +'use strict'; + +var helpers = {}; + +helpers.valueToString = function (value) { + if (value === null || value === undefined) { + return value; + } + + return value.toString(); +}; + +helpers.removeDuplicateValues = function (values) { + var others = Array.prototype.slice.call(arguments, 1); + for (var i = 0; i < values.length; i++) { + if (values.lastIndexOf(values[i]) !== i) { + values.splice(i, 1); + for (var j = 0; j < others.length; j++) { + others[j].splice(i, 1); + } + i -= 1; + } + } +}; + +helpers.ensureLegacyObjectType = function (db, key, type, callback) { + db.query({ + name: 'ensureLegacyObjectTypeBefore', + text: ` +DELETE FROM "legacy_object" + WHERE "expireAt" IS NOT NULL + AND "expireAt" <= CURRENT_TIMESTAMP`, + }, function (err) { + if (err) { + return callback(err); + } + + db.query({ + name: 'ensureLegacyObjectType1', + text: ` +INSERT INTO "legacy_object" ("_key", "type") +VALUES ($1::TEXT, $2::TEXT::LEGACY_OBJECT_TYPE) + ON CONFLICT + DO NOTHING`, + values: [key, type], + }, function (err) { + if (err) { + return callback(err); + } + + db.query({ + name: 'ensureLegacyObjectType2', + text: ` +SELECT "type" + FROM "legacy_object_live" + WHERE "_key" = $1::TEXT`, + values: [key], + }, function (err, res) { + if (err) { + return callback(err); + } + + if (res.rows[0].type !== type) { + return callback(new Error('database: cannot insert ' + JSON.stringify(key) + ' as ' + type + ' because it already exists as ' + res.rows[0].type)); + } + + callback(null); + }); + }); + }); +}; + +helpers.ensureLegacyObjectsType = function (db, keys, type, callback) { + db.query({ + name: 'ensureLegacyObjectTypeBefore', + text: ` +DELETE FROM "legacy_object" + WHERE "expireAt" IS NOT NULL + AND "expireAt" <= CURRENT_TIMESTAMP`, + }, function (err) { + if (err) { + return callback(err); + } + + db.query({ + name: 'ensureLegacyObjectsType1', + text: ` +INSERT INTO "legacy_object" ("_key", "type") +SELECT k, $2::TEXT::LEGACY_OBJECT_TYPE + FROM UNNEST($1::TEXT[]) k + ON CONFLICT + DO NOTHING`, + values: [keys, type], + }, function (err) { + if (err) { + return callback(err); + } + + db.query({ + name: 'ensureLegacyObjectsType2', + text: ` +SELECT "_key", "type" + FROM "legacy_object_live" + WHERE "_key" = ANY($1::TEXT[])`, + values: [keys], + }, function (err, res) { + if (err) { + return callback(err); + } + + var invalid = res.rows.filter(function (r) { + return r.type !== type; + }); + + if (invalid.length) { + return callback(new Error('database: cannot insert multiple objects as ' + type + ' because they already exist: ' + invalid.map(function (r) { + return JSON.stringify(r._key) + ' is ' + r.type; + }).join(', '))); + } + + var missing = keys.filter(function (k) { + return !res.rows.some(function (r) { + return r._key === k; + }); + }); + + if (missing.length) { + return callback(new Error('database: failed to insert keys for objects: ' + JSON.stringify(missing))); + } + + callback(null); + }); + }); + }); +}; + +helpers.noop = function () {}; + +module.exports = helpers; diff --git a/src/database/postgres/list.js b/src/database/postgres/list.js new file mode 100644 index 0000000000..9d11e16dff --- /dev/null +++ b/src/database/postgres/list.js @@ -0,0 +1,234 @@ +'use strict'; + +var async = require('async'); + +module.exports = function (db, module) { + var helpers = module.helpers.postgres; + + module.listPrepend = function (key, value, callback) { + callback = callback || helpers.noop; + + if (!key) { + return callback(); + } + + module.transaction(function (tx, done) { + var query = tx.client.query.bind(tx.client); + + async.series([ + async.apply(helpers.ensureLegacyObjectType, tx.client, key, 'list'), + async.apply(query, { + name: 'listPrepend', + text: ` +INSERT INTO "legacy_list" ("_key", "array") +VALUES ($1::TEXT, ARRAY[$2::TEXT]) + ON CONFLICT ("_key") + DO UPDATE SET "array" = ARRAY[$2::TEXT] || "legacy_list"."array"`, + values: [key, value], + }), + ], function (err) { + done(err); + }); + }, callback); + }; + + module.listAppend = function (key, value, callback) { + callback = callback || helpers.noop; + + if (!key) { + return callback(); + } + + module.transaction(function (tx, done) { + var query = tx.client.query.bind(tx.client); + + async.series([ + async.apply(helpers.ensureLegacyObjectType, tx.client, key, 'list'), + async.apply(query, { + name: 'listAppend', + text: ` +INSERT INTO "legacy_list" ("_key", "array") +VALUES ($1::TEXT, ARRAY[$2::TEXT]) + ON CONFLICT ("_key") + DO UPDATE SET "array" = "legacy_list"."array" || ARRAY[$2::TEXT]`, + values: [key, value], + }), + ], function (err) { + done(err); + }); + }, callback || helpers.noop); + }; + + module.listRemoveLast = function (key, callback) { + callback = callback || helpers.noop; + + if (!key) { + return callback(); + } + + db.query({ + name: 'listRemoveLast', + text: ` +WITH A AS ( + SELECT l.* + FROM "legacy_object_live" o + INNER JOIN "legacy_list" l + ON o."_key" = l."_key" + AND o."type" = l."type" + WHERE o."_key" = $1::TEXT + FOR UPDATE) +UPDATE "legacy_list" l + SET "array" = A."array"[1 : array_length(A."array", 1) - 1] + FROM A + WHERE A."_key" = l."_key" +RETURNING A."array"[array_length(A."array", 1)] v`, + values: [key], + }, function (err, res) { + if (err) { + return callback(err); + } + + if (res.rows.length) { + return callback(null, res.rows[0].v); + } + + callback(null, null); + }); + }; + + module.listRemoveAll = function (key, value, callback) { + callback = callback || helpers.noop; + + if (!key) { + return callback(); + } + + db.query({ + name: 'listRemoveAll', + text: ` +UPDATE "legacy_list" l + SET "array" = array_remove(l."array", $2::TEXT) + FROM "legacy_object_live" o + WHERE o."_key" = l."_key" + AND o."type" = l."type" + AND o."_key" = $1::TEXT`, + values: [key, value], + }, function (err) { + callback(err); + }); + }; + + module.listTrim = function (key, start, stop, callback) { + callback = callback || helpers.noop; + + if (!key) { + return callback(); + } + + stop += 1; + + db.query(stop > 0 ? { + name: 'listTrim', + text: ` +UPDATE "legacy_list" l + SET "array" = ARRAY(SELECT m.m + FROM UNNEST(l."array") WITH ORDINALITY m(m, i) + ORDER BY m.i ASC + LIMIT ($3::INTEGER - $2::INTEGER) + OFFSET $2::INTEGER) + FROM "legacy_object_live" o + WHERE o."_key" = l."_key" + AND o."type" = l."type" + AND o."_key" = $1::TEXT`, + values: [key, start, stop], + } : { + name: 'listTrimBack', + text: ` +UPDATE "legacy_list" l + SET "array" = ARRAY(SELECT m.m + FROM UNNEST(l."array") WITH ORDINALITY m(m, i) + ORDER BY m.i ASC + LIMIT ($3::INTEGER - $2::INTEGER + array_length(l."array", 1)) + OFFSET $2::INTEGER) + FROM "legacy_object_live" o + WHERE o."_key" = l."_key" + AND o."type" = l."type" + AND o."_key" = $1::TEXT`, + values: [key, start, stop], + }, function (err) { + callback(err); + }); + }; + + module.getListRange = function (key, start, stop, callback) { + if (!key) { + return callback(); + } + + stop += 1; + + db.query(stop > 0 ? { + name: 'getListRange', + text: ` +SELECT ARRAY(SELECT m.m + FROM UNNEST(l."array") WITH ORDINALITY m(m, i) + ORDER BY m.i ASC + LIMIT ($3::INTEGER - $2::INTEGER) + OFFSET $2::INTEGER) l + FROM "legacy_object_live" o + INNER JOIN "legacy_list" l + ON o."_key" = l."_key" + AND o."type" = l."type" + WHERE o."_key" = $1::TEXT`, + values: [key, start, stop], + } : { + name: 'getListRangeBack', + text: ` +SELECT ARRAY(SELECT m.m + FROM UNNEST(l."array") WITH ORDINALITY m(m, i) + ORDER BY m.i ASC + LIMIT ($3::INTEGER - $2::INTEGER + array_length(l."array", 1)) + OFFSET $2::INTEGER) l + FROM "legacy_object_live" o + INNER JOIN "legacy_list" l + ON o."_key" = l."_key" + AND o."type" = l."type" + WHERE o."_key" = $1::TEXT`, + values: [key, start, stop], + }, function (err, res) { + if (err) { + return callback(err); + } + + if (res.rows.length) { + return callback(null, res.rows[0].l); + } + + callback(null, []); + }); + }; + + module.listLength = function (key, callback) { + db.query({ + name: 'listLength', + text: ` +SELECT array_length(l."array", 1) l + FROM "legacy_object_live" o + INNER JOIN "legacy_list" l + ON o."_key" = l."_key" + AND o."type" = l."type" + WHERE o."_key" = $1::TEXT`, + values: [key], + }, function (err, res) { + if (err) { + return callback(err); + } + + if (res.rows.length) { + return callback(null, res.rows[0].l); + } + + callback(null, 0); + }); + }; +}; diff --git a/src/database/postgres/main.js b/src/database/postgres/main.js new file mode 100644 index 0000000000..2dcc6ecd5e --- /dev/null +++ b/src/database/postgres/main.js @@ -0,0 +1,239 @@ +'use strict'; + +var async = require('async'); + +module.exports = function (db, module) { + var helpers = module.helpers.postgres; + + var query = db.query.bind(db); + + module.flushdb = function (callback) { + callback = callback || helpers.noop; + + async.series([ + async.apply(query, `DROP SCHEMA "public" CASCADE`), + async.apply(query, `CREATE SCHEMA "public"`), + ], function (err) { + callback(err); + }); + }; + + module.emptydb = function (callback) { + callback = callback || helpers.noop; + query(`DELETE FROM "legacy_object"`, function (err) { + callback(err); + }); + }; + + module.exists = function (key, callback) { + if (!key) { + return callback(); + } + + query({ + name: 'exists', + text: ` +SELECT EXISTS(SELECT * + FROM "legacy_object_live" + WHERE "_key" = $1::TEXT + LIMIT 1) e`, + values: [key], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, res.rows[0].e); + }); + }; + + module.delete = function (key, callback) { + callback = callback || helpers.noop; + if (!key) { + return callback(); + } + + query({ + name: 'delete', + text: ` +DELETE FROM "legacy_object" + WHERE "_key" = $1::TEXT`, + values: [key], + }, function (err) { + callback(err); + }); + }; + + module.deleteAll = function (keys, callback) { + callback = callback || helpers.noop; + + if (!Array.isArray(keys) || !keys.length) { + return callback(); + } + + query({ + name: 'deleteAll', + text: ` +DELETE FROM "legacy_object" + WHERE "_key" = ANY($1::TEXT[])`, + values: [keys], + }, function (err) { + callback(err); + }); + }; + + module.get = function (key, callback) { + if (!key) { + return callback(); + } + + query({ + name: 'get', + text: ` +SELECT s."data" t + FROM "legacy_object_live" o + INNER JOIN "legacy_string" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = $1::TEXT + LIMIT 1`, + values: [key], + }, function (err, res) { + if (err) { + return callback(err); + } + + if (res.rows.length) { + return callback(null, res.rows[0].t); + } + + callback(null, null); + }); + }; + + module.set = function (key, value, callback) { + callback = callback || helpers.noop; + + if (!key) { + return callback(); + } + + module.transaction(function (tx, done) { + async.series([ + async.apply(helpers.ensureLegacyObjectType, tx.client, key, 'string'), + async.apply(tx.client.query.bind(tx.client), { + name: 'set', + text: ` +INSERT INTO "legacy_string" ("_key", "data") +VALUES ($1::TEXT, $2::TEXT) + ON CONFLICT ("_key") + DO UPDATE SET "data" = $2::TEXT`, + values: [key, value], + }), + ], function (err) { + done(err); + }); + }, callback); + }; + + module.increment = function (key, callback) { + callback = callback || helpers.noop; + + if (!key) { + return callback(); + } + + module.transaction(function (tx, done) { + async.waterfall([ + async.apply(helpers.ensureLegacyObjectType, tx.client, key, 'string'), + async.apply(tx.client.query.bind(tx.client), { + name: 'increment', + text: ` +INSERT INTO "legacy_string" ("_key", "data") +VALUES ($1::TEXT, '1') + ON CONFLICT ("_key") + DO UPDATE SET "data" = ("legacy_string"."data"::NUMERIC + 1)::TEXT +RETURNING "data" d`, + values: [key], + }), + ], function (err, res) { + if (err) { + return done(err); + } + + done(null, parseFloat(res.rows[0].d)); + }); + }, callback); + }; + + module.rename = function (oldKey, newKey, callback) { + module.transaction(function (tx, done) { + async.series([ + async.apply(tx.delete, newKey), + async.apply(tx.client.query.bind(tx.client), { + name: 'rename', + text: ` +UPDATE "legacy_object" + SET "_key" = $2::TEXT + WHERE "_key" = $1::TEXT`, + values: [oldKey, newKey], + }), + ], function (err) { + done(err); + }); + }, callback || helpers.noop); + }; + + module.type = function (key, callback) { + query({ + name: 'type', + text: ` +SELECT "type"::TEXT t + FROM "legacy_object_live" + WHERE "_key" = $1::TEXT + LIMIT 1`, + values: [key], + }, function (err, res) { + if (err) { + return callback(err); + } + + if (res.rows.length) { + return callback(null, res.rows[0].t); + } + + callback(null, null); + }); + }; + + function doExpire(key, date, callback) { + query({ + name: 'expire', + text: ` +UPDATE "legacy_object" + SET "expireAt" = $2::TIMESTAMPTZ + WHERE "_key" = $1::TEXT`, + values: [key, date], + }, function (err) { + if (callback) { + callback(err); + } + }); + } + + module.expire = function (key, seconds, callback) { + doExpire(key, new Date(((Date.now() / 1000) + seconds) * 1000), callback); + }; + + module.expireAt = function (key, timestamp, callback) { + doExpire(key, new Date(timestamp * 1000), callback); + }; + + module.pexpire = function (key, ms, callback) { + doExpire(key, new Date(Date.now() + parseInt(ms, 10)), callback); + }; + + module.pexpireAt = function (key, timestamp, callback) { + doExpire(key, new Date(timestamp), callback); + }; +}; diff --git a/src/database/postgres/pubsub.js b/src/database/postgres/pubsub.js new file mode 100644 index 0000000000..969ef62cb8 --- /dev/null +++ b/src/database/postgres/pubsub.js @@ -0,0 +1,51 @@ +'use strict'; + +var util = require('util'); +var winston = require('winston'); +var EventEmitter = require('events').EventEmitter; +var pg = require('pg'); +var db = require('../postgres'); + +var PubSub = function () { + var self = this; + + var subClient = new pg.Client(db.getConnectionOptions()); + + subClient.connect(function (err) { + if (err) { + winston.error(err); + return; + } + + subClient.query('LISTEN pubsub', function (err) { + if (err) { + winston.error(err); + } + }); + + subClient.on('notification', function (message) { + if (message.channel !== 'pubsub') { + return; + } + + try { + var msg = JSON.parse(message.payload); + self.emit(msg.event, msg.data); + } catch (err) { + winston.error(err.stack); + } + }); + }); +}; + +util.inherits(PubSub, EventEmitter); + +PubSub.prototype.publish = function (event, data) { + db.pool.query({ + name: 'pubSubPublish', + text: `SELECT pg_notify('pubsub', $1::TEXT)`, + values: [JSON.stringify({ event: event, data: data })], + }); +}; + +module.exports = new PubSub(); diff --git a/src/database/postgres/sets.js b/src/database/postgres/sets.js new file mode 100644 index 0000000000..615cb6ad42 --- /dev/null +++ b/src/database/postgres/sets.js @@ -0,0 +1,342 @@ +'use strict'; + +var async = require('async'); + +module.exports = function (db, module) { + var helpers = module.helpers.postgres; + + module.setAdd = function (key, value, callback) { + callback = callback || helpers.noop; + + if (!Array.isArray(value)) { + value = [value]; + } + + module.transaction(function (tx, done) { + var query = tx.client.query.bind(tx.client); + + async.series([ + async.apply(helpers.ensureLegacyObjectType, tx.client, key, 'set'), + async.apply(query, { + name: 'setAdd', + text: ` +INSERT INTO "legacy_set" ("_key", "member") +SELECT $1::TEXT, m + FROM UNNEST($2::TEXT[]) m + ON CONFLICT ("_key", "member") + DO NOTHING`, + values: [key, value], + }), + ], function (err) { + done(err); + }); + }, callback); + }; + + module.setsAdd = function (keys, value, callback) { + callback = callback || helpers.noop; + + if (!Array.isArray(keys) || !keys.length) { + return callback(); + } + + if (!Array.isArray(value)) { + value = [value]; + } + + keys = keys.filter(function (k, i, a) { + return a.indexOf(k) === i; + }); + + module.transaction(function (tx, done) { + var query = tx.client.query.bind(tx.client); + + async.series([ + async.apply(helpers.ensureLegacyObjectsType, tx.client, keys, 'set'), + async.apply(query, { + name: 'setsAdd', + text: ` +INSERT INTO "legacy_set" ("_key", "member") +SELECT k, m + FROM UNNEST($1::TEXT[]) k + CROSS JOIN UNNEST($2::TEXT[]) m + ON CONFLICT ("_key", "member") + DO NOTHING`, + values: [keys, value], + }), + ], function (err) { + done(err); + }); + }, callback); + }; + + module.setRemove = function (key, value, callback) { + callback = callback || helpers.noop; + + if (!Array.isArray(key)) { + key = [key]; + } + + if (!Array.isArray(value)) { + value = [value]; + } + + db.query({ + name: 'setRemove', + text: ` +DELETE FROM "legacy_set" + WHERE "_key" = ANY($1::TEXT[]) + AND "member" = ANY($2::TEXT[])`, + values: [key, value], + }, function (err) { + callback(err); + }); + }; + + module.setsRemove = function (keys, value, callback) { + callback = callback || helpers.noop; + + if (!Array.isArray(keys) || !keys.length) { + return callback(); + } + + db.query({ + name: 'setsRemove', + text: ` +DELETE FROM "legacy_set" + WHERE "_key" = ANY($1::TEXT[]) + AND "member" = $2::TEXT`, + values: [keys, value], + }, function (err) { + callback(err); + }); + }; + + module.isSetMember = function (key, value, callback) { + if (!key) { + return callback(null, false); + } + + db.query({ + name: 'isSetMember', + text: ` +SELECT 1 + FROM "legacy_object_live" o + INNER JOIN "legacy_set" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = $1::TEXT + AND s."member" = $2::TEXT`, + values: [key, value], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, !!res.rows.length); + }); + }; + + module.isSetMembers = function (key, values, callback) { + if (!key || !Array.isArray(values) || !values.length) { + return callback(null, []); + } + + values = values.map(helpers.valueToString); + + db.query({ + name: 'isSetMembers', + text: ` +SELECT s."member" m + FROM "legacy_object_live" o + INNER JOIN "legacy_set" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = $1::TEXT + AND s."member" = ANY($2::TEXT[])`, + values: [key, values], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, values.map(function (v) { + return res.rows.some(function (r) { + return r.m === v; + }); + })); + }); + }; + + module.isMemberOfSets = function (sets, value, callback) { + if (!Array.isArray(sets) || !sets.length) { + return callback(null, []); + } + + value = helpers.valueToString(value); + + db.query({ + name: 'isMemberOfSets', + text: ` +SELECT o."_key" k + FROM "legacy_object_live" o + INNER JOIN "legacy_set" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = ANY($1::TEXT[]) + AND s."member" = $2::TEXT`, + values: [sets, value], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, sets.map(function (s) { + return res.rows.some(function (r) { + return r.k === s; + }); + })); + }); + }; + + module.getSetMembers = function (key, callback) { + if (!key) { + return callback(null, []); + } + + db.query({ + name: 'getSetMembers', + text: ` +SELECT s."member" m + FROM "legacy_object_live" o + INNER JOIN "legacy_set" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = $1::TEXT`, + values: [key], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, res.rows.map(function (r) { + return r.m; + })); + }); + }; + + module.getSetsMembers = function (keys, callback) { + if (!Array.isArray(keys) || !keys.length) { + return callback(null, []); + } + + db.query({ + name: 'getSetsMembers', + text: ` +SELECT o."_key" k, + array_agg(s."member") m + FROM "legacy_object_live" o + INNER JOIN "legacy_set" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = ANY($1::TEXT[]) + GROUP BY o."_key"`, + values: [keys], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, keys.map(function (k) { + return (res.rows.find(function (r) { + return r.k === k; + }) || { m: [] }).m; + })); + }); + }; + + module.setCount = function (key, callback) { + if (!key) { + return callback(null, 0); + } + + db.query({ + name: 'setCount', + text: ` +SELECT COUNT(*) c + FROM "legacy_object_live" o + INNER JOIN "legacy_set" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = $1::TEXT`, + values: [key], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, parseInt(res.rows[0].c, 10)); + }); + }; + + module.setsCount = function (keys, callback) { + db.query({ + name: 'setsCount', + text: ` +SELECT o."_key" k, + COUNT(*) c + FROM "legacy_object_live" o + INNER JOIN "legacy_set" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = ANY($1::TEXT[]) + GROUP BY o."_key"`, + values: [keys], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, keys.map(function (k) { + return (res.rows.find(function (r) { + return r.k === k; + }) || { c: 0 }).c; + })); + }); + }; + + module.setRemoveRandom = function (key, callback) { + callback = callback || helpers.noop; + + db.query({ + name: 'setRemoveRandom', + text: ` +WITH A AS ( + SELECT s."member" + FROM "legacy_object_live" o + INNER JOIN "legacy_set" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = $1::TEXT + ORDER BY RANDOM() + LIMIT 1 + FOR UPDATE) +DELETE FROM "legacy_set" s + USING A + WHERE s."_key" = $1::TEXT + AND s."member" = A."member" +RETURNING A."member" m`, + values: [key], + }, function (err, res) { + if (err) { + return callback(err); + } + + if (res.rows.length) { + return callback(null, res.rows[0].m); + } + + callback(null, null); + }); + }; +}; diff --git a/src/database/postgres/sorted.js b/src/database/postgres/sorted.js new file mode 100644 index 0000000000..d540f6ca9a --- /dev/null +++ b/src/database/postgres/sorted.js @@ -0,0 +1,744 @@ +'use strict'; + +var async = require('async'); + +module.exports = function (db, module) { + var helpers = module.helpers.postgres; + + var query = db.query.bind(db); + + require('./sorted/add')(db, module); + require('./sorted/remove')(db, module); + require('./sorted/union')(db, module); + require('./sorted/intersect')(db, module); + + module.getSortedSetRange = function (key, start, stop, callback) { + getSortedSetRange(key, start, stop, 1, false, callback); + }; + + module.getSortedSetRevRange = function (key, start, stop, callback) { + getSortedSetRange(key, start, stop, -1, false, callback); + }; + + module.getSortedSetRangeWithScores = function (key, start, stop, callback) { + getSortedSetRange(key, start, stop, 1, true, callback); + }; + + module.getSortedSetRevRangeWithScores = function (key, start, stop, callback) { + getSortedSetRange(key, start, stop, -1, true, callback); + }; + + function getSortedSetRange(key, start, stop, sort, withScores, callback) { + if (!key) { + return callback(); + } + + if (!Array.isArray(key)) { + key = [key]; + } + + if (start < 0 && start > stop) { + return callback(null, []); + } + + var reverse = false; + if (start === 0 && stop < -1) { + reverse = true; + sort *= -1; + start = Math.abs(stop + 1); + stop = -1; + } else if (start < 0 && stop > start) { + var tmp1 = Math.abs(stop + 1); + stop = Math.abs(start + 1); + start = tmp1; + } + + var limit = stop - start + 1; + if (limit <= 0) { + limit = null; + } + + query({ + name: 'getSortedSetRangeWithScores' + (sort > 0 ? 'Asc' : 'Desc'), + text: ` +SELECT z."value", + z."score" + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = ANY($1::TEXT[]) + ORDER BY z."score" ` + (sort > 0 ? 'ASC' : 'DESC') + ` + LIMIT $3::INTEGER +OFFSET $2::INTEGER`, + values: [key, start, limit], + }, function (err, res) { + if (err) { + return callback(err); + } + + if (reverse) { + res.rows.reverse(); + } + + if (withScores) { + res.rows = res.rows.map(function (r) { + return { + value: r.value, + score: parseFloat(r.score), + }; + }); + } else { + res.rows = res.rows.map(function (r) { + return r.value; + }); + } + + callback(null, res.rows); + }); + } + + module.getSortedSetRangeByScore = function (key, start, count, min, max, callback) { + getSortedSetRangeByScore(key, start, count, min, max, 1, false, callback); + }; + + module.getSortedSetRevRangeByScore = function (key, start, count, max, min, callback) { + getSortedSetRangeByScore(key, start, count, min, max, -1, false, callback); + }; + + module.getSortedSetRangeByScoreWithScores = function (key, start, count, min, max, callback) { + getSortedSetRangeByScore(key, start, count, min, max, 1, true, callback); + }; + + module.getSortedSetRevRangeByScoreWithScores = function (key, start, count, max, min, callback) { + getSortedSetRangeByScore(key, start, count, min, max, -1, true, callback); + }; + + function getSortedSetRangeByScore(key, start, count, min, max, sort, withScores, callback) { + if (!key) { + return callback(); + } + + if (!Array.isArray(key)) { + key = [key]; + } + + if (parseInt(count, 10) === -1) { + count = null; + } + + if (min === '-inf') { + min = null; + } + if (max === '+inf') { + max = null; + } + + query({ + name: 'getSortedSetRangeByScoreWithScores' + (sort > 0 ? 'Asc' : 'Desc'), + text: ` +SELECT z."value", + z."score" + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = ANY($1::TEXT[]) + AND (z."score" >= $4::NUMERIC OR $4::NUMERIC IS NULL) + AND (z."score" <= $5::NUMERIC OR $5::NUMERIC IS NULL) + ORDER BY z."score" ` + (sort > 0 ? 'ASC' : 'DESC') + ` + LIMIT $3::INTEGER +OFFSET $2::INTEGER`, + values: [key, start, count, min, max], + }, function (err, res) { + if (err) { + return callback(err); + } + + if (withScores) { + res.rows = res.rows.map(function (r) { + return { + value: r.value, + score: parseFloat(r.score), + }; + }); + } else { + res.rows = res.rows.map(function (r) { + return r.value; + }); + } + + return callback(null, res.rows); + }); + } + + module.sortedSetCount = function (key, min, max, callback) { + if (!key) { + return callback(); + } + + if (min === '-inf') { + min = null; + } + if (max === '+inf') { + max = null; + } + + query({ + name: 'sortedSetCount', + text: ` +SELECT COUNT(*) c + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = $1::TEXT + AND (z."score" >= $2::NUMERIC OR $2::NUMERIC IS NULL) + AND (z."score" <= $3::NUMERIC OR $3::NUMERIC IS NULL)`, + values: [key, min, max], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, parseInt(res.rows[0].c, 10)); + }); + }; + + module.sortedSetCard = function (key, callback) { + if (!key) { + return callback(null, 0); + } + + query({ + name: 'sortedSetCard', + text: ` +SELECT COUNT(*) c + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = $1::TEXT`, + values: [key], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, parseInt(res.rows[0].c, 10)); + }); + }; + + module.sortedSetsCard = function (keys, callback) { + if (!Array.isArray(keys) || !keys.length) { + return callback(); + } + + query({ + name: 'sortedSetsCard', + text: ` +SELECT o."_key" k, + COUNT(*) c + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = ANY($1::TEXT[]) + GROUP BY o."_key"`, + values: [keys], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, keys.map(function (k) { + return parseInt((res.rows.find(function (r) { + return r.k === k; + }) || { c: 0 }).c, 10); + })); + }); + }; + + module.sortedSetRank = function (key, value, callback) { + getSortedSetRank('ASC', [key], [value], function (err, result) { + callback(err, result ? result[0] : null); + }); + }; + + module.sortedSetRevRank = function (key, value, callback) { + getSortedSetRank('DESC', [key], [value], function (err, result) { + callback(err, result ? result[0] : null); + }); + }; + + function getSortedSetRank(sort, keys, values, callback) { + values = values.map(helpers.valueToString); + query({ + name: 'getSortedSetRank' + sort, + text: ` +SELECT (SELECT r + FROM (SELECT z."value" v, + RANK() OVER (PARTITION BY o."_key" + ORDER BY z."score" ` + sort + `, + z."value" ` + sort + `) - 1 r + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = kvi.k) r + WHERE v = kvi.v) r + FROM UNNEST($1::TEXT[], $2::TEXT[]) WITH ORDINALITY kvi(k, v, i) + ORDER BY kvi.i ASC`, + values: [keys, values], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, res.rows.map(function (r) { return r.r === null ? null : parseFloat(r.r); })); + }); + } + + module.sortedSetsRanks = function (keys, values, callback) { + if (!Array.isArray(keys) || !keys.length) { + return callback(null, []); + } + + getSortedSetRank('ASC', keys, values, callback); + }; + + module.sortedSetRanks = function (key, values, callback) { + if (!Array.isArray(values) || !values.length) { + return callback(null, []); + } + + getSortedSetRank('ASC', new Array(values.length).fill(key), values, callback); + }; + + module.sortedSetScore = function (key, value, callback) { + if (!key) { + return callback(null, null); + } + + value = helpers.valueToString(value); + + query({ + name: 'sortedSetScore', + text: ` +SELECT z."score" s + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = $1::TEXT + AND z."value" = $2::TEXT`, + values: [key, value], + }, function (err, res) { + if (err) { + return callback(err); + } + + if (res.rows.length) { + return callback(null, parseFloat(res.rows[0].s)); + } + + callback(null, null); + }); + }; + + module.sortedSetsScore = function (keys, value, callback) { + if (!Array.isArray(keys) || !keys.length) { + return callback(); + } + + value = helpers.valueToString(value); + + query({ + name: 'sortedSetsScore', + text: ` +SELECT o."_key" k, + z."score" s + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = ANY($1::TEXT[]) + AND z."value" = $2::TEXT`, + values: [keys, value], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, keys.map(function (k) { + var s = res.rows.find(function (r) { + return r.k === k; + }); + + return s ? parseFloat(s.s) : null; + })); + }); + }; + + module.sortedSetScores = function (key, values, callback) { + if (!key) { + return callback(null, null); + } + + values = values.map(helpers.valueToString); + + query({ + name: 'sortedSetScores', + text: ` +SELECT z."value" v, + z."score" s + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = $1::TEXT + AND z."value" = ANY($2::TEXT[])`, + values: [key, values], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, values.map(function (v) { + var s = res.rows.find(function (r) { + return r.v === v; + }); + + return s ? parseFloat(s.s) : null; + })); + }); + }; + + module.isSortedSetMember = function (key, value, callback) { + if (!key) { + return callback(); + } + + value = helpers.valueToString(value); + + query({ + name: 'isSortedSetMember', + text: ` +SELECT 1 + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = $1::TEXT + AND z."value" = $2::TEXT`, + values: [key, value], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, !!res.rows.length); + }); + }; + + module.isSortedSetMembers = function (key, values, callback) { + if (!key) { + return callback(); + } + + values = values.map(helpers.valueToString); + + query({ + name: 'isSortedSetMembers', + text: ` +SELECT z."value" v + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = $1::TEXT + AND z."value" = ANY($2::TEXT[])`, + values: [key, values], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, values.map(function (v) { + return res.rows.some(function (r) { + return r.v === v; + }); + })); + }); + }; + + module.isMemberOfSortedSets = function (keys, value, callback) { + if (!Array.isArray(keys)) { + return callback(); + } + + value = helpers.valueToString(value); + + query({ + name: 'isMemberOfSortedSets', + text: ` +SELECT o."_key" k + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = ANY($1::TEXT[]) + AND z."value" = $2::TEXT`, + values: [keys, value], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, keys.map(function (k) { + return res.rows.some(function (r) { + return r.k === k; + }); + })); + }); + }; + + module.getSortedSetsMembers = function (keys, callback) { + if (!Array.isArray(keys) || !keys.length) { + return callback(null, []); + } + + query({ + name: 'getSortedSetsMembers', + text: ` +SELECT o."_key" k, + array_agg(z."value" ORDER BY z."score" ASC) m + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = ANY($1::TEXT[]) + GROUP BY o."_key"`, + values: [keys], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, keys.map(function (k) { + return (res.rows.find(function (r) { + return r.k === k; + }) || { m: [] }).m; + })); + }); + }; + + module.sortedSetIncrBy = function (key, increment, value, callback) { + callback = callback || helpers.noop; + + if (!key) { + return callback(); + } + + value = helpers.valueToString(value); + increment = parseFloat(increment); + + module.transaction(function (tx, done) { + async.waterfall([ + async.apply(helpers.ensureLegacyObjectType, tx.client, key, 'zset'), + async.apply(tx.client.query.bind(tx.client), { + name: 'sortedSetIncrBy', + text: ` +INSERT INTO "legacy_zset" ("_key", "value", "score") +VALUES ($1::TEXT, $2::TEXT, $3::NUMERIC) + ON CONFLICT ("_key", "value") + DO UPDATE SET "score" = "legacy_zset"."score" + $3::NUMERIC +RETURNING "score" s`, + values: [key, value, increment], + }), + function (res, next) { + next(null, parseFloat(res.rows[0].s)); + }, + ], done); + }, callback); + }; + + module.getSortedSetRangeByLex = function (key, min, max, start, count, callback) { + sortedSetLex(key, min, max, 1, start, count, callback); + }; + + module.getSortedSetRevRangeByLex = function (key, max, min, start, count, callback) { + sortedSetLex(key, min, max, -1, start, count, callback); + }; + + module.sortedSetLexCount = function (key, min, max, callback) { + var q = buildLexQuery(key, min, max); + + query({ + name: 'sortedSetLexCount' + q.suffix, + text: ` +SELECT COUNT(*) c + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE ` + q.where, + values: q.values, + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, parseInt(res.rows[0].c, 10)); + }); + }; + + function sortedSetLex(key, min, max, sort, start, count, callback) { + if (!callback) { + callback = start; + start = 0; + count = 0; + } + + var q = buildLexQuery(key, min, max); + q.values.push(start); + q.values.push(count <= 0 ? null : count); + query({ + name: 'sortedSetLex' + (sort > 0 ? 'Asc' : 'Desc') + q.suffix, + text: ` +SELECT z."value" v + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE ` + q.where + ` + ORDER BY z."value" ` + (sort > 0 ? 'ASC' : 'DESC') + ` + LIMIT $` + q.values.length + `::INTEGER +OFFSET $` + (q.values.length - 1) + `::INTEGER`, + values: q.values, + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, res.rows.map(function (r) { + return r.v; + })); + }); + } + + module.sortedSetRemoveRangeByLex = function (key, min, max, callback) { + callback = callback || helpers.noop; + + var q = buildLexQuery(key, min, max); + query({ + name: 'sortedSetRemoveRangeByLex' + q.suffix, + text: ` +DELETE FROM "legacy_zset" z + USING "legacy_object_live" o + WHERE o."_key" = z."_key" + AND o."type" = z."type" + AND ` + q.where, + values: q.values, + }, function (err) { + callback(err); + }); + }; + + function buildLexQuery(key, min, max) { + var q = { + suffix: '', + where: `o."_key" = $1::TEXT`, + values: [key], + }; + + if (min !== '-') { + if (min.match(/^\(/)) { + q.values.push(min.substr(1)); + q.suffix += 'GT'; + q.where += ` AND z."value" > $` + q.values.length + `::TEXT`; + } else if (min.match(/^\[/)) { + q.values.push(min.substr(1)); + q.suffix += 'GE'; + q.where += ` AND z."value" >= $` + q.values.length + `::TEXT`; + } else { + q.values.push(min); + q.suffix += 'GE'; + q.where += ` AND z."value" >= $` + q.values.length + `::TEXT`; + } + } + + if (max !== '+') { + if (max.match(/^\(/)) { + q.values.push(max.substr(1)); + q.suffix += 'LT'; + q.where += ` AND z."value" < $` + q.values.length + `::TEXT`; + } else if (max.match(/^\[/)) { + q.values.push(max.substr(1)); + q.suffix += 'LE'; + q.where += ` AND z."value" <= $` + q.values.length + `::TEXT`; + } else { + q.values.push(max); + q.suffix += 'LE'; + q.where += ` AND z."value" <= $` + q.values.length + `::TEXT`; + } + } + + return q; + } + + module.processSortedSet = function (setKey, process, options, callback) { + var Cursor = require('pg-cursor'); + + db.connect(function (err, client, done) { + if (err) { + return callback(err); + } + + var batchSize = (options || {}).batch || 100; + var query = client.query(new Cursor(` +SELECT z."value", z."score" + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = $1::TEXT + ORDER BY z."score" ASC, z."value" ASC`, [setKey])); + + async.doUntil(function (next) { + query.read(batchSize, function (err, rows) { + if (err) { + return next(err); + } + + if (!rows.length) { + return next(null, true); + } + + rows = rows.map(function (row) { + return options.withScores ? row : row.value; + }); + + process(rows, function (err) { + if (err) { + return query.close(function () { + next(err); + }); + } + + if (options.interval) { + setTimeout(next, options.interval); + } else { + next(); + } + }); + }); + }, function (stop) { + return stop; + }, function (err) { + done(); + callback(err); + }); + }); + }; +}; diff --git a/src/database/postgres/sorted/add.js b/src/database/postgres/sorted/add.js new file mode 100644 index 0000000000..a187091746 --- /dev/null +++ b/src/database/postgres/sorted/add.js @@ -0,0 +1,108 @@ +'use strict'; + +var async = require('async'); + +module.exports = function (db, module) { + var helpers = module.helpers.postgres; + + module.sortedSetAdd = function (key, score, value, callback) { + callback = callback || helpers.noop; + + if (!key) { + return callback(); + } + + if (Array.isArray(score) && Array.isArray(value)) { + return sortedSetAddBulk(key, score, value, callback); + } + + value = helpers.valueToString(value); + score = parseFloat(score); + + module.transaction(function (tx, done) { + var query = tx.client.query.bind(tx.client); + + async.series([ + async.apply(helpers.ensureLegacyObjectType, tx.client, key, 'zset'), + async.apply(query, { + name: 'sortedSetAdd', + text: ` +INSERT INTO "legacy_zset" ("_key", "value", "score") +VALUES ($1::TEXT, $2::TEXT, $3::NUMERIC) + ON CONFLICT ("_key", "value") + DO UPDATE SET "score" = $3::NUMERIC`, + values: [key, value, score], + }), + ], function (err) { + done(err); + }); + }, callback); + }; + + function sortedSetAddBulk(key, scores, values, callback) { + if (!scores.length || !values.length) { + return callback(); + } + if (scores.length !== values.length) { + return callback(new Error('[[error:invalid-data]]')); + } + + values = values.map(helpers.valueToString); + scores = scores.map(function (score) { + return parseFloat(score); + }); + + helpers.removeDuplicateValues(values, scores); + + module.transaction(function (tx, done) { + var query = tx.client.query.bind(tx.client); + + async.series([ + async.apply(helpers.ensureLegacyObjectType, tx.client, key, 'zset'), + async.apply(query, { + name: 'sortedSetAddBulk', + text: ` +INSERT INTO "legacy_zset" ("_key", "value", "score") +SELECT $1::TEXT, v, s + FROM UNNEST($2::TEXT[], $3::NUMERIC[]) vs(v, s) + ON CONFLICT ("_key", "value") + DO UPDATE SET "score" = EXCLUDED."score"`, + values: [key, values, scores], + }), + ], function (err) { + done(err); + }); + }, callback); + } + + module.sortedSetsAdd = function (keys, score, value, callback) { + callback = callback || helpers.noop; + + if (!Array.isArray(keys) || !keys.length) { + return callback(); + } + + value = helpers.valueToString(value); + score = parseFloat(score); + + module.transaction(function (tx, done) { + var query = tx.client.query.bind(tx.client); + + async.series([ + async.apply(helpers.ensureLegacyObjectsType, tx.client, keys, 'zset'), + async.apply(query, { + name: 'sortedSetsAdd', + text: ` +INSERT INTO "legacy_zset" ("_key", "value", "score") +SELECT k, $2::TEXT, $3::NUMERIC + FROM UNNEST($1::TEXT[]) k + ON CONFLICT ("_key", "value") + DO UPDATE SET "score" = $3::NUMERIC`, + values: [keys, value, score], + }), + ], function (err) { + done(err); + }); + }, callback); + }; +}; diff --git a/src/database/postgres/sorted/intersect.js b/src/database/postgres/sorted/intersect.js new file mode 100644 index 0000000000..47fe5b9b16 --- /dev/null +++ b/src/database/postgres/sorted/intersect.js @@ -0,0 +1,105 @@ +'use strict'; + +module.exports = function (db, module) { + module.sortedSetIntersectCard = function (keys, callback) { + if (!Array.isArray(keys) || !keys.length) { + return callback(null, 0); + } + + db.query({ + name: 'sortedSetIntersectCard', + text: ` +WITH A AS (SELECT z."value" v, + COUNT(*) c + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = ANY($1::TEXT[]) + GROUP BY z."value") +SELECT COUNT(*) c + FROM A + WHERE A.c = array_length($1::TEXT[], 1)`, + values: [keys], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, parseInt(res.rows[0].c, 10)); + }); + }; + + + module.getSortedSetIntersect = function (params, callback) { + params.sort = 1; + getSortedSetIntersect(params, callback); + }; + + module.getSortedSetRevIntersect = function (params, callback) { + params.sort = -1; + getSortedSetIntersect(params, callback); + }; + + function getSortedSetIntersect(params, callback) { + var sets = params.sets; + var start = params.hasOwnProperty('start') ? params.start : 0; + var stop = params.hasOwnProperty('stop') ? params.stop : -1; + var weights = params.weights || []; + var aggregate = params.aggregate || 'SUM'; + + if (sets.length < weights.length) { + weights = weights.slice(0, sets.length); + } + while (sets.length > weights.length) { + weights.push(1); + } + + var limit = stop - start + 1; + if (limit <= 0) { + limit = null; + } + + db.query({ + name: 'getSortedSetIntersect' + aggregate + (params.sort > 0 ? 'Asc' : 'Desc') + 'WithScores', + text: ` +WITH A AS (SELECT z."value", + ` + aggregate + `(z."score" * k."weight") "score", + COUNT(*) c + FROM UNNEST($1::TEXT[], $2::NUMERIC[]) k("_key", "weight") + INNER JOIN "legacy_object_live" o + ON o."_key" = k."_key" + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + GROUP BY z."value") +SELECT A."value", + A."score" + FROM A + WHERE c = array_length($1::TEXT[], 1) + ORDER BY A."score" ` + (params.sort > 0 ? 'ASC' : 'DESC') + ` + LIMIT $4::INTEGER +OFFSET $3::INTEGER`, + values: [sets, weights, start, limit], + }, function (err, res) { + if (err) { + return callback(err); + } + + if (params.withScores) { + res.rows = res.rows.map(function (r) { + return { + value: r.value, + score: parseFloat(r.score), + }; + }); + } else { + res.rows = res.rows.map(function (r) { + return r.value; + }); + } + + callback(null, res.rows); + }); + } +}; diff --git a/src/database/postgres/sorted/remove.js b/src/database/postgres/sorted/remove.js new file mode 100644 index 0000000000..6118b22981 --- /dev/null +++ b/src/database/postgres/sorted/remove.js @@ -0,0 +1,83 @@ +'use strict'; + +module.exports = function (db, module) { + var helpers = module.helpers.postgres; + + module.sortedSetRemove = function (key, value, callback) { + function done(err) { + if (callback) { + callback(err); + } + } + + if (!key) { + return done(); + } + + if (!Array.isArray(key)) { + key = [key]; + } + + if (!Array.isArray(value)) { + value = [value]; + } + value = value.map(helpers.valueToString); + + db.query({ + name: 'sortedSetRemove', + text: ` +DELETE FROM "legacy_zset" + WHERE "_key" = ANY($1::TEXT[]) + AND "value" = ANY($2::TEXT[])`, + values: [key, value], + }, done); + }; + + module.sortedSetsRemove = function (keys, value, callback) { + callback = callback || helpers.noop; + + if (!Array.isArray(keys) || !keys.length) { + return callback(); + } + + value = helpers.valueToString(value); + + db.query({ + name: 'sortedSetsRemove', + text: ` +DELETE FROM "legacy_zset" + WHERE "_key" = ANY($1::TEXT[]) + AND "value" = $2::TEXT`, + values: [keys, value], + }, function (err) { + callback(err); + }); + }; + + module.sortedSetsRemoveRangeByScore = function (keys, min, max, callback) { + callback = callback || helpers.noop; + + if (!Array.isArray(keys) || !keys.length) { + return callback(); + } + + if (min === '-inf') { + min = null; + } + if (max === '+inf') { + max = null; + } + + db.query({ + name: 'sortedSetsRemoveRangeByScore', + text: ` +DELETE FROM "legacy_zset" + WHERE "_key" = ANY($1::TEXT[]) + AND ("score" >= $2::NUMERIC OR $2::NUMERIC IS NULL) + AND ("score" <= $3::NUMERIC OR $3::NUMERIC IS NULL)`, + values: [keys, min, max], + }, function (err) { + callback(err); + }); + }; +}; diff --git a/src/database/postgres/sorted/union.js b/src/database/postgres/sorted/union.js new file mode 100644 index 0000000000..2f991dc761 --- /dev/null +++ b/src/database/postgres/sorted/union.js @@ -0,0 +1,97 @@ +'use strict'; + +module.exports = function (db, module) { + module.sortedSetUnionCard = function (keys, callback) { + if (!Array.isArray(keys) || !keys.length) { + return callback(null, 0); + } + + db.query({ + name: 'sortedSetUnionCard', + text: ` +SELECT COUNT(DISTINCT z."value") c + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = ANY($1::TEXT[])`, + values: [keys], + }, function (err, res) { + if (err) { + return callback(err); + } + + callback(null, parseInt(res.rows[0].c, 10)); + }); + }; + + module.getSortedSetUnion = function (params, callback) { + params.sort = 1; + getSortedSetUnion(params, callback); + }; + + module.getSortedSetRevUnion = function (params, callback) { + params.sort = -1; + getSortedSetUnion(params, callback); + }; + + function getSortedSetUnion(params, callback) { + var sets = params.sets; + var start = params.hasOwnProperty('start') ? params.start : 0; + var stop = params.hasOwnProperty('stop') ? params.stop : -1; + var weights = params.weights || []; + var aggregate = params.aggregate || 'SUM'; + + if (sets.length < weights.length) { + weights = weights.slice(0, sets.length); + } + while (sets.length > weights.length) { + weights.push(1); + } + + var limit = stop - start + 1; + if (limit <= 0) { + limit = null; + } + + db.query({ + name: 'getSortedSetUnion' + aggregate + (params.sort > 0 ? 'Asc' : 'Desc') + 'WithScores', + text: ` +WITH A AS (SELECT z."value", + ` + aggregate + `(z."score" * k."weight") "score" + FROM UNNEST($1::TEXT[], $2::NUMERIC[]) k("_key", "weight") + INNER JOIN "legacy_object_live" o + ON o."_key" = k."_key" + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + GROUP BY z."value") +SELECT A."value", + A."score" + FROM A + ORDER BY A."score" ` + (params.sort > 0 ? 'ASC' : 'DESC') + ` + LIMIT $4::INTEGER +OFFSET $3::INTEGER`, + values: [sets, weights, start, limit], + }, function (err, res) { + if (err) { + return callback(err); + } + + if (params.withScores) { + res.rows = res.rows.map(function (r) { + return { + value: r.value, + score: parseFloat(r.score), + }; + }); + } else { + res.rows = res.rows.map(function (r) { + return r.value; + }); + } + + callback(null, res.rows); + }); + } +}; diff --git a/src/database/postgres/transaction.js b/src/database/postgres/transaction.js new file mode 100644 index 0000000000..ed13d7a537 --- /dev/null +++ b/src/database/postgres/transaction.js @@ -0,0 +1,50 @@ +'use strict'; + +module.exports = function (db, dbNamespace, module) { + module.transaction = function (perform, callback) { + if (dbNamespace.active && dbNamespace.get('db')) { + var client = dbNamespace.get('db'); + return client.query(`SAVEPOINT nodebb_subtx`, function (err) { + if (err) { + return callback(err); + } + + perform(module, function (err) { + var args = Array.prototype.slice.call(arguments, 1); + + client.query(err ? `ROLLBACK TO SAVEPOINT nodebb_subtx` : `RELEASE SAVEPOINT nodebb_subtx`, function (err1) { + callback.apply(this, [err || err1].concat(args)); + }); + }); + }); + } + + db.connect(function (err, client, done) { + if (err) { + return callback(err); + } + + dbNamespace.run(function () { + dbNamespace.set('db', client); + + client.query(`BEGIN`, function (err) { + if (err) { + done(); + dbNamespace.set('db', null); + return callback(err); + } + + perform(module, function (err) { + var args = Array.prototype.slice.call(arguments, 1); + + client.query(err ? `ROLLBACK` : `COMMIT`, function (err1) { + done(); + dbNamespace.set('db', null); + callback.apply(this, [err || err1].concat(args)); + }); + }); + }); + }); + }); + }; +}; diff --git a/src/database/redis.js b/src/database/redis.js index 4bbd6ed0da..6d780d0853 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -50,6 +50,9 @@ redisModule.init = function (callback) { require('./redis/sets')(redisClient, redisModule); require('./redis/sorted')(redisClient, redisModule); require('./redis/list')(redisClient, redisModule); + require('./redis/transaction')(redisClient, redisModule); + + redisModule.async = require('../promisify')(redisModule, ['client', 'sessionStore']); callback(); }); diff --git a/src/database/redis/hash.js b/src/database/redis/hash.js index 9dd6276f88..3055143375 100644 --- a/src/database/redis/hash.js +++ b/src/database/redis/hash.js @@ -14,7 +14,7 @@ module.exports = function (redisClient, module) { } Object.keys(data).forEach(function (key) { - if (data[key] === undefined) { + if (data[key] === undefined || data[key] === null) { delete data[key]; } }); @@ -26,6 +26,9 @@ module.exports = function (redisClient, module) { module.setObjectField = function (key, field, value, callback) { callback = callback || function () {}; + if (!field) { + return callback(); + } redisClient.hset(key, field, value, function (err) { callback(err); }); diff --git a/src/database/redis/transaction.js b/src/database/redis/transaction.js new file mode 100644 index 0000000000..75ea5fbaa2 --- /dev/null +++ b/src/database/redis/transaction.js @@ -0,0 +1,8 @@ +'use strict'; + +module.exports = function (db, module) { + // TODO + module.transaction = function (perform, callback) { + perform(db, callback); + }; +}; diff --git a/src/emailer.js b/src/emailer.js index 184eb5d95b..dbc56b9175 100644 --- a/src/emailer.js +++ b/src/emailer.js @@ -210,6 +210,12 @@ Emailer.sendToEmail = function (template, email, language, params, callback) { var lang = language || meta.config.defaultLang || 'en-GB'; + // Add some default email headers based on local configuration + params.headers = Object.assign({ + 'List-Id': '<' + [template, params.uid, getHostname()].join('.') + '>', + 'List-Unsubscribe': '<' + [nconf.get('url'), 'uid', params.uid, 'settings'].join('/') + '>', + }, params.headers); + async.waterfall([ function (next) { Plugins.fireHook('filter:email.params', { @@ -249,6 +255,7 @@ Emailer.sendToEmail = function (template, email, language, params, callback) { uid: params.uid, pid: params.pid, fromUid: params.fromUid, + headers: params.headers, }; Plugins.fireHook('filter:email.modify', data, next); }, @@ -289,22 +296,26 @@ Emailer.sendViaFallback = function (data, callback) { function buildCustomTemplates(config) { async.waterfall([ function (next) { - Emailer.getTemplates(config, next); + async.parallel({ + templates: function (cb) { + Emailer.getTemplates(config, cb); + }, + paths: function (cb) { + file.walk(viewsDir, cb); + }, + }, next); }, - function (templates, next) { - templates = templates.filter(function (template) { + function (result, next) { + var templates = result.templates.filter(function (template) { return template.isCustom && template.text !== prevConfig['email:custom:' + path]; }); + var paths = _.fromPairs(result.paths.map(function (p) { + var relative = path.relative(viewsDir, p).replace(/\\/g, '/'); + return [relative, p]; + })); async.each(templates, function (template, next) { async.waterfall([ function (next) { - file.walk(viewsDir, next); - }, - function (paths, next) { - paths = _.fromPairs(paths.map(function (p) { - var relative = path.relative(viewsDir, p).replace(/\\/g, '/'); - return [relative, p]; - })); meta.templates.processImports(paths, template.path, template.text, next); }, function (source, next) { diff --git a/src/events.js b/src/events.js index cb8798ed70..34dcf15ec4 100644 --- a/src/events.js +++ b/src/events.js @@ -4,6 +4,7 @@ var async = require('async'); var validator = require('validator'); var winston = require('winston'); +var _ = require('lodash'); var db = require('./database'); var batch = require('./batch'); @@ -12,6 +13,41 @@ var utils = require('./utils'); var events = module.exports; +events.types = [ + 'plugin-activate', + 'plugin-deactivate', + 'restart', + 'build', + 'config-change', + 'settings-change', + 'category-purge', + 'privilege-change', + 'post-delete', + 'post-restore', + 'post-purge', + 'topic-delete', + 'topic-restore', + 'topic-purge', + 'topic-rename', + 'password-reset', + 'user-ban', + 'user-unban', + 'user-delete', + 'password-change', + 'email-change', + 'username-change', + 'ip-blacklist-save', + 'ip-blacklist-addRule', + 'registration-approved', + 'registration-rejected', + 'accept-membership', + 'reject-membership', + 'theme-set', + 'export:uploads', + 'account-locked', + 'getUsersCSV', +]; + /** * Useful options in data: type, uid, ip, targetUid * Everything else gets stringified and shown as pretty JSON string @@ -31,6 +67,9 @@ events.log = function (data, callback) { function (next) { db.sortedSetAdd('events:time', data.timestamp, eid, next); }, + function (next) { + db.sortedSetAdd('events:time:' + data.type, data.timestamp, eid, next); + }, function (next) { db.setObject('event:' + eid, data, next); }, @@ -41,10 +80,10 @@ events.log = function (data, callback) { }); }; -events.getEvents = function (start, stop, callback) { +events.getEvents = function (filter, start, stop, callback) { async.waterfall([ function (next) { - db.getSortedSetRevRange('events:time', start, stop, next); + db.getSortedSetRevRange('events:time' + (filter ? ':' + filter : ''), start, stop, next); }, function (eids, next) { var keys = eids.map(function (eid) { @@ -123,15 +162,24 @@ function addUserData(eventsData, field, objectName, callback) { events.deleteEvents = function (eids, callback) { callback = callback || function () {}; - async.parallel([ + var keys; + async.waterfall([ function (next) { - var keys = eids.map(function (eid) { + keys = eids.map(function (eid) { return 'event:' + eid; }); - db.deleteAll(keys, next); + db.getObjectsFields(keys, ['type'], next); }, - function (next) { - db.sortedSetRemove('events:time', eids, next); + function (eventData, next) { + var sets = _.uniq(['events:time'].concat(eventData.map(e => 'events:time:' + e.type))); + async.parallel([ + function (next) { + db.deleteAll(keys, next); + }, + function (next) { + db.sortedSetRemove(sets, eids, next); + }, + ], next); }, ], callback); }; @@ -146,7 +194,7 @@ events.deleteAll = function (callback) { events.output = function () { console.log('\nDisplaying last ten administrative events...'.bold); - events.getEvents(0, 9, function (err, events) { + events.getEvents('', 0, 9, function (err, events) { if (err) { winston.error('Error fetching events', err); throw err; diff --git a/src/file.js b/src/file.js index 3f0aaabcf0..a1f5f0580c 100644 --- a/src/file.js +++ b/src/file.js @@ -4,7 +4,6 @@ var fs = require('fs'); var nconf = require('nconf'); var path = require('path'); var winston = require('winston'); -var jimp = require('jimp'); var mkdirp = require('mkdirp'); var mime = require('mime'); var graceful = require('graceful-fs'); @@ -107,12 +106,22 @@ file.isFileTypeAllowed = function (path, callback) { }); } - // Attempt to read the file, if it passes, file type is allowed - jimp.read(path, function (err) { + require('sharp')(path, { + failOnError: true, + }).metadata(function (err) { callback(err); }); }; +// https://stackoverflow.com/a/31205878/583363 +file.appendToFileName = function (filename, string) { + var dotIndex = filename.lastIndexOf('.'); + if (dotIndex === -1) { + return filename + string; + } + return filename.substring(0, dotIndex) + string + filename.substring(dotIndex); +}; + file.allowedExtensions = function () { var meta = require('./meta'); var allowedExtensions = (meta.config.allowedFileExtensions || '').trim(); @@ -163,7 +172,7 @@ file.existsSync = function (path) { file.delete = function (path, callback) { callback = callback || function () {}; if (!path) { - return callback(); + return setImmediate(callback); } fs.unlink(path, function (err) { if (err) { diff --git a/src/flags.js b/src/flags.js index 238bd7f204..c0c49c4a0a 100644 --- a/src/flags.js +++ b/src/flags.js @@ -320,6 +320,7 @@ Flags.getNotes = function (flagId, callback) { next(null, notes.map(function (note, idx) { note.user = users[idx]; + note.content = validator.escape(note.content); return note; })); }); @@ -496,7 +497,7 @@ Flags.update = function (flagId, uid, changeset, callback) { var tasks = []; var now = changeset.datetime || Date.now(); var notifyAssignee = function (assigneeId, next) { - if (assigneeId === '') { + if (assigneeId === '' || parseInt(uid, 10) === parseInt(assigneeId, 10)) { // Do nothing return next(); } @@ -688,7 +689,7 @@ Flags.notify = function (flagObj, uid, callback) { bodyShort: '[[notifications:user_flagged_post_in, ' + flagObj.reporter.username + ', ' + titleEscaped + ']]', bodyLong: flagObj.description, pid: flagObj.targetId, - path: '/post/' + flagObj.targetId, + path: '/flags/' + flagObj.flagId, nid: 'flag:post:' + flagObj.targetId + ':uid:' + uid, from: uid, mergeId: 'notifications:user_flagged_post_in|' + flagObj.targetId, @@ -725,7 +726,7 @@ Flags.notify = function (flagObj, uid, callback) { type: 'new-user-flag', bodyShort: '[[notifications:user_flagged_user, ' + flagObj.reporter.username + ', ' + flagObj.target.username + ']]', bodyLong: flagObj.description, - path: '/uid/' + flagObj.targetId, + path: '/flags/' + flagObj.flagId, nid: 'flag:user:' + flagObj.targetId + ':uid:' + uid, from: uid, mergeId: 'notifications:user_flagged_user|' + flagObj.targetId, diff --git a/src/groups.js b/src/groups.js index 780eda46c4..9ac5cc8b85 100644 --- a/src/groups.js +++ b/src/groups.js @@ -301,3 +301,5 @@ Groups.existsBySlug = function (slug, callback) { db.isObjectField('groupslug:groupname', slug, callback); } }; + +Groups.async = require('./promisify')(Groups); diff --git a/src/groups/cover.js b/src/groups/cover.js index 289ee16302..ae2f86c90f 100644 --- a/src/groups/cover.js +++ b/src/groups/cover.js @@ -2,7 +2,6 @@ var async = require('async'); var path = require('path'); -var Jimp = require('jimp'); var mime = require('mime'); var db = require('../database'); @@ -27,7 +26,6 @@ module.exports = function (Groups) { var tempPath = data.file ? data.file : ''; var url; var type = data.file ? mime.getType(data.file) : 'image/png'; - async.waterfall([ function (next) { if (tempPath) { @@ -49,7 +47,10 @@ module.exports = function (Groups) { Groups.setGroupField(data.groupName, 'cover:url', url, next); }, function (next) { - resizeCover(tempPath, next); + image.resizeImage({ + path: tempPath, + width: 358, + }, next); }, function (next) { uploadsController.uploadGroupCover(uid, { @@ -74,22 +75,6 @@ module.exports = function (Groups) { }); }; - function resizeCover(path, callback) { - async.waterfall([ - function (next) { - new Jimp(path, next); - }, - function (image, next) { - image.resize(358, Jimp.AUTO, next); - }, - function (image, next) { - image.write(path, next); - }, - ], function (err) { - callback(err); - }); - } - Groups.removeCover = function (data, callback) { db.deleteObjectFields('group:' + data.groupName, ['cover:url', 'cover:thumb:url', 'cover:position'], callback); }; diff --git a/src/groups/membership.js b/src/groups/membership.js index 09e86e12f6..1f6cc7af14 100644 --- a/src/groups/membership.js +++ b/src/groups/membership.js @@ -105,7 +105,7 @@ module.exports = function (Groups) { return callback(err); } - user.setUserField(uid, 'groupTitle', groupName, callback); + user.setUserField(uid, 'groupTitle', JSON.stringify([groupName]), callback); }); } @@ -294,13 +294,17 @@ module.exports = function (Groups) { } async.waterfall([ function (next) { - db.getObjectField('user:' + uid, 'groupTitle', next); + user.getUserData(uid, next); }, - function (groupTitle, next) { - if (groupNames.includes(groupTitle)) { - db.deleteObjectField('user:' + uid, 'groupTitle', next); + function (userData, next) { + var newTitleArray = userData.groupTitleArray.filter(function (groupTitle) { + return !groupNames.includes(groupTitle); + }); + + if (newTitleArray.length) { + db.setObjectField('user:' + uid, 'groupTitle', JSON.stringify(newTitleArray), next); } else { - next(); + db.deleteObjectField('user:' + uid, 'groupTitle', next); } }, ], callback); diff --git a/src/image.js b/src/image.js index f99a73e3bc..9e65654507 100644 --- a/src/image.js +++ b/src/image.js @@ -3,21 +3,28 @@ var os = require('os'); var fs = require('fs'); var path = require('path'); -var Jimp = require('jimp'); -var async = require('async'); var crypto = require('crypto'); +var async = require('async'); var file = require('./file'); var plugins = require('./plugins'); var image = module.exports; +function requireSharp() { + var sharp = require('sharp'); + if (os.platform() === 'win32') { + // https://github.com/lovell/sharp/issues/1259 + sharp.cache(false); + } + return sharp; +} + image.resizeImage = function (data, callback) { if (plugins.hasListeners('filter:image.resize')) { plugins.fireHook('filter:image.resize', { path: data.path, target: data.target, - extension: data.extension, width: data.width, height: data.height, quality: data.quality, @@ -25,64 +32,26 @@ image.resizeImage = function (data, callback) { callback(err); }); } else { - new Jimp(data.path, function (err, image) { - if (err) { - return callback(err); - } + async.waterfall([ + function (next) { + fs.readFile(data.path, next); + }, + function (buffer, next) { + var sharp = requireSharp(); + var sharpImage = sharp(buffer, { + failOnError: true, + }); + sharpImage.rotate(); // auto-orients based on exif data + sharpImage.resize(data.hasOwnProperty('width') ? data.width : null, data.hasOwnProperty('height') ? data.height : null); - var w = image.bitmap.width; - var h = image.bitmap.height; - var origRatio = w / h; - var desiredRatio = data.width && data.height ? data.width / data.height : origRatio; - var x = 0; - var y = 0; - var crop; - - if (image._exif && image._exif.tags && image._exif.tags.Orientation) { - image.exifRotate(); - } - - if (origRatio !== desiredRatio) { - if (desiredRatio > origRatio) { - desiredRatio = 1 / desiredRatio; + if (data.quality) { + sharpImage.jpeg({ quality: data.quality }); } - if (origRatio >= 1) { - y = 0; // height is the smaller dimension here - x = Math.floor((w / 2) - (h * desiredRatio / 2)); - crop = async.apply(image.crop.bind(image), x, y, h * desiredRatio, h); - } else { - x = 0; // width is the smaller dimension here - y = Math.floor((h / 2) - (w * desiredRatio / 2)); - crop = async.apply(image.crop.bind(image), x, y, w, w * desiredRatio); - } - } else { - // Simple resize given either width, height, or both - crop = async.apply(setImmediate); - } - async.waterfall([ - crop, - function (_image, next) { - if (typeof _image === 'function' && !next) { - next = _image; - _image = image; - } - - if ((data.width && data.height) || (w > data.width) || (h > data.height)) { - _image.resize(data.width || Jimp.AUTO, data.height || Jimp.AUTO, next); - } else { - next(null, image); - } - }, - function (image, next) { - if (data.quality) { - image.quality(data.quality); - } - image.write(data.target || data.path, next); - }, - ], function (err) { - callback(err); - }); + sharpImage.toFile(data.target || data.path, next); + }, + ], function (err) { + callback(err); }); } }; @@ -91,21 +60,14 @@ image.normalise = function (path, extension, callback) { if (plugins.hasListeners('filter:image.normalise')) { plugins.fireHook('filter:image.normalise', { path: path, - extension: extension, }, function (err) { callback(err, path + '.png'); }); } else { - async.waterfall([ - function (next) { - new Jimp(path, next); - }, - function (image, next) { - image.write(path + '.png', function (err) { - next(err, path + '.png'); - }); - }, - ], callback); + var sharp = requireSharp(); + sharp(path, { failOnError: true }).png().toFile(path + '.png', function (err) { + callback(err, path + '.png'); + }); } }; @@ -114,15 +76,33 @@ image.size = function (path, callback) { plugins.fireHook('filter:image.size', { path: path, }, function (err, image) { - callback(err, image); + callback(err, image ? { width: image.width, height: image.height } : undefined); }); } else { - new Jimp(path, function (err, data) { - callback(err, data ? data.bitmap : null); + var sharp = requireSharp(); + sharp(path, { failOnError: true }).metadata(function (err, metadata) { + callback(err, metadata ? { width: metadata.width, height: metadata.height } : undefined); }); } }; +image.checkDimensions = function (path, callback) { + const meta = require('./meta'); + image.size(path, function (err, result) { + if (err) { + return callback(err); + } + + const maxWidth = parseInt(meta.config.rejectImageWidth, 10) || 5000; + const maxHeight = parseInt(meta.config.rejectImageHeight, 10) || 5000; + if (result.width > maxWidth || result.height > maxHeight) { + return callback(new Error('[[error:invalid-image-dimensions]]')); + } + + callback(); + }); +}; + image.convertImageToBase64 = function (path, callback) { fs.readFile(path, 'base64', callback); }; @@ -151,3 +131,7 @@ image.writeImageDataToTempFile = function (imageData, callback) { callback(err, filepath); }); }; + +image.sizeFromBase64 = function (imageData) { + return Buffer.from(imageData.slice(imageData.indexOf('base64') + 7), 'base64').length; +}; diff --git a/src/install.js b/src/install.js index 25cb213488..5ef191cf44 100644 --- a/src/install.js +++ b/src/install.js @@ -126,7 +126,8 @@ function setupConfig(next) { var config = {}; var redisQuestions = require('./database/redis').questions; var mongoQuestions = require('./database/mongo').questions; - var allQuestions = questions.main.concat(questions.optional).concat(redisQuestions).concat(mongoQuestions); + var postgresQuestions = require('./database/postgres').questions; + var allQuestions = questions.main.concat(questions.optional).concat(redisQuestions).concat(mongoQuestions).concat(postgresQuestions); allQuestions.forEach(function (question) { config[question.name] = install.values[question.name] || question.default || undefined; @@ -380,7 +381,10 @@ function createGlobalModeratorsGroup(next) { function giveGlobalPrivileges(next) { var privileges = require('./privileges'); - var defaultPrivileges = ['chat', 'upload:post:image', 'signature', 'search:content', 'search:users', 'search:tags']; + var defaultPrivileges = [ + 'chat', 'upload:post:image', 'signature', 'search:content', + 'search:users', 'search:tags', 'local:login', + ]; privileges.global.give(defaultPrivileges, 'registered-users', next); } diff --git a/src/messaging.js b/src/messaging.js index ba9e794b30..543d6d03a7 100644 --- a/src/messaging.js +++ b/src/messaging.js @@ -383,3 +383,5 @@ Messaging.hasPrivateChat = function (uid, withUid, callback) { }, ], callback); }; + +Messaging.async = require('./promisify')(Messaging); diff --git a/src/messaging/create.js b/src/messaging/create.js index bcee2f4f5d..c173528c65 100644 --- a/src/messaging/create.js +++ b/src/messaging/create.js @@ -30,13 +30,20 @@ module.exports = function (Messaging) { if (!content) { return callback(new Error('[[error:invalid-chat-message]]')); } - content = String(content); - var maximumChatMessageLength = (meta.config.maximumChatMessageLength || 1000); - if (content.length > maximumChatMessageLength) { - return callback(new Error('[[error:chat-message-too-long, ' + maximumChatMessageLength + ']]')); - } - callback(); + plugins.fireHook('filter:messaging.checkContent', { content: content }, function (err, data) { + if (err) { + return callback(err); + } + + content = String(data.content); + + var maximumChatMessageLength = (meta.config.maximumChatMessageLength || 1000); + if (content.length > maximumChatMessageLength) { + return callback(new Error('[[error:chat-message-too-long, ' + maximumChatMessageLength + ']]')); + } + callback(); + }); }; Messaging.addMessage = function (data, callback) { @@ -84,7 +91,6 @@ module.exports = function (Messaging) { async.apply(Messaging.addRoomToUsers, data.roomId, uids, data.timestamp), async.apply(Messaging.addMessageToUsers, data.roomId, uids, mid, data.timestamp), async.apply(Messaging.markUnread, uids, data.roomId), - async.apply(Messaging.addUsersToRoom, data.uid, [data.uid], data.roomId), ], next); }, function (results, next) { diff --git a/src/messaging/rooms.js b/src/messaging/rooms.js index 219f19513c..6e85e594d0 100644 --- a/src/messaging/rooms.js +++ b/src/messaging/rooms.js @@ -229,6 +229,14 @@ module.exports = function (Messaging) { function (uids, next) { user.getUsersFields(uids, ['uid', 'username', 'picture', 'status'], next); }, + function (users, next) { + db.getObjectField('chat:room:' + roomId, 'owner', function (err, ownerId) { + next(err, users.map(function (user) { + user.isOwner = parseInt(user.uid, 10) === parseInt(ownerId, 10); + return user; + })); + }); + }, ], callback); }; diff --git a/src/meta.js b/src/meta.js index cb2a381d6d..9a03c89ee6 100644 --- a/src/meta.js +++ b/src/meta.js @@ -39,26 +39,6 @@ Meta.userOrGroupExists = function (slug, callback) { }); }; -/** - * Reload deprecated as of v1.1.2+, remove in v2.x - */ -Meta.reload = function (callback) { - restart(); - callback(); -}; - -Meta.restart = function () { - pubsub.publish('meta:restart', { hostname: os.hostname() }); - restart(); -}; - -Meta.getSessionTTLSeconds = function () { - var ttlDays = 60 * 60 * 24 * (parseInt(Meta.config.loginDays, 10) || 0); - var ttlSeconds = (parseInt(Meta.config.loginSeconds, 10) || 0); - var ttl = ttlSeconds || ttlDays || 1209600; // Default to 14 days - return ttl; -}; - if (nconf.get('isPrimary') === 'true') { pubsub.on('meta:restart', function (data) { if (data.hostname !== os.hostname()) { @@ -67,6 +47,11 @@ if (nconf.get('isPrimary') === 'true') { }); } +Meta.restart = function () { + pubsub.publish('meta:restart', { hostname: os.hostname() }); + restart(); +}; + function restart() { if (process.send) { process.send({ @@ -76,3 +61,10 @@ function restart() { winston.error('[meta.restart] Could not restart, are you sure NodeBB was started with `./nodebb start`?'); } } + +Meta.getSessionTTLSeconds = function () { + var ttlDays = 60 * 60 * 24 * (parseInt(Meta.config.loginDays, 10) || 0); + var ttlSeconds = (parseInt(Meta.config.loginSeconds, 10) || 0); + var ttl = ttlSeconds || ttlDays || 1209600; // Default to 14 days + return ttl; +}; diff --git a/src/meta/build.js b/src/meta/build.js index 62399ec2e7..62288dad3a 100644 --- a/src/meta/build.js +++ b/src/meta/build.js @@ -134,13 +134,22 @@ function buildTargets(targets, parallel, callback) { }, callback); } -function build(targets, callback) { +function build(targets, options, callback) { + if (!callback && typeof options === 'function') { + callback = options; + options = {}; + } else if (!options) { + options = {}; + } + if (targets === true) { targets = allTargets; } else if (!Array.isArray(targets)) { targets = targets.split(','); } + var parallel = !nconf.get('series') && !options.series; + targets = targets // get full target name .map(function (target) { @@ -200,7 +209,6 @@ function build(targets, callback) { require('./minifier').maxThreads = threads - 1; } - var parallel = !nconf.get('series'); if (parallel) { winston.info('[build] Building in parallel mode'); } else { diff --git a/src/meta/css.js b/src/meta/css.js index cf39d5f8f6..5762c2cf1c 100644 --- a/src/meta/css.js +++ b/src/meta/css.js @@ -89,6 +89,7 @@ function getImports(files, prefix, extension, callback) { function getBundleMetadata(target, callback) { var paths = [ path.join(__dirname, '../../node_modules'), + path.join(__dirname, '../../public/less'), path.join(__dirname, '../../public/vendor/fontawesome/less'), ]; diff --git a/src/meta/tags.js b/src/meta/tags.js index 09019884c3..fe95bab6cf 100644 --- a/src/meta/tags.js +++ b/src/meta/tags.js @@ -56,7 +56,7 @@ Tags.parse = function (req, data, meta, link, callback) { var defaultLinks = [{ rel: 'icon', type: 'image/x-icon', - href: nconf.get('relative_path') + '/favicon.ico' + (Meta.config['cache-buster'] ? '?' + Meta.config['cache-buster'] : ''), + href: nconf.get('relative_path') + nconf.get('upload_url') + '/system/favicon.ico' + (Meta.config['cache-buster'] ? '?' + Meta.config['cache-buster'] : ''), }, { rel: 'manifest', href: nconf.get('relative_path') + '/manifest.json', @@ -75,31 +75,31 @@ Tags.parse = function (req, data, meta, link, callback) { if (Meta.config['brand:touchIcon']) { defaultLinks.push({ rel: 'apple-touch-icon', - href: nconf.get('relative_path') + '/apple-touch-icon', + href: nconf.get('relative_path') + nconf.get('upload_url') + '/system/touchicon-orig.png', }, { rel: 'icon', sizes: '36x36', - href: nconf.get('relative_path') + '/assets/uploads/system/touchicon-36.png', + href: nconf.get('relative_path') + nconf.get('upload_url') + '/system/touchicon-36.png', }, { rel: 'icon', sizes: '48x48', - href: nconf.get('relative_path') + '/assets/uploads/system/touchicon-48.png', + href: nconf.get('relative_path') + nconf.get('upload_url') + '/system/touchicon-48.png', }, { rel: 'icon', sizes: '72x72', - href: nconf.get('relative_path') + '/assets/uploads/system/touchicon-72.png', + href: nconf.get('relative_path') + nconf.get('upload_url') + '/system/touchicon-72.png', }, { rel: 'icon', sizes: '96x96', - href: nconf.get('relative_path') + '/assets/uploads/system/touchicon-96.png', + href: nconf.get('relative_path') + nconf.get('upload_url') + '/system/touchicon-96.png', }, { rel: 'icon', sizes: '144x144', - href: nconf.get('relative_path') + '/assets/uploads/system/touchicon-144.png', + href: nconf.get('relative_path') + nconf.get('upload_url') + '/system/touchicon-144.png', }, { rel: 'icon', sizes: '192x192', - href: nconf.get('relative_path') + '/assets/uploads/system/touchicon-192.png', + href: nconf.get('relative_path') + nconf.get('upload_url') + '/system/touchicon-192.png', }); } plugins.fireHook('filter:meta.getLinkTags', { req: req, data: data, links: defaultLinks }, next); diff --git a/src/meta/templates.js b/src/meta/templates.js index 1ccf14c8e3..11635dcb50 100644 --- a/src/meta/templates.js +++ b/src/meta/templates.js @@ -8,6 +8,7 @@ var path = require('path'); var fs = require('fs'); var nconf = require('nconf'); var _ = require('lodash'); +var Benchpress = require('benchpressjs'); var plugins = require('../plugins'); var file = require('../file'); @@ -113,6 +114,34 @@ function getTemplateFiles(dirs, callback) { ], callback); } +function compileTemplate(filename, source, callback) { + async.waterfall([ + function (next) { + file.walk(viewsPath, next); + }, + function (paths, next) { + paths = _.fromPairs(paths.map(function (p) { + var relative = path.relative(viewsPath, p).replace(/\\/g, '/'); + return [relative, p]; + })); + async.waterfall([ + function (next) { + processImports(paths, filename, source, next); + }, + function (source, next) { + Benchpress.precompile(source, { + minify: global.env !== 'development', + }, next); + }, + function (compiled, next) { + fs.writeFile(path.join(viewsPath, filename.replace(/\.tpl$/, '.js')), compiled, next); + }, + ], next); + }, + ], callback); +} +Templates.compileTemplate = compileTemplate; + function compile(callback) { callback = callback || function () {}; @@ -144,8 +173,22 @@ function compile(callback) { next(err, source); }); }, - function (compiled, next) { - fs.writeFile(path.join(viewsPath, name), compiled, next); + function (imported, next) { + async.parallel([ + function (cb) { + fs.writeFile(path.join(viewsPath, name), imported, cb); + }, + function (cb) { + Benchpress.precompile(imported, { minify: global.env !== 'development' }, function (err, compiled) { + if (err) { + cb(err); + return; + } + + fs.writeFile(path.join(viewsPath, name.replace(/\.tpl$/, '.js')), compiled, cb); + }); + }, + ], next); }, ], next); }, next); diff --git a/src/middleware/header.js b/src/middleware/header.js index f845d8b4bb..cb29604f95 100644 --- a/src/middleware/header.js +++ b/src/middleware/header.js @@ -120,9 +120,7 @@ module.exports = function (middleware) { banned: async.apply(user.isBanned, req.uid), banReason: async.apply(user.getBannedReason, req.uid), - unreadTopicCount: async.apply(topics.getTotalUnread, req.uid), - unreadNewTopicCount: async.apply(topics.getTotalUnread, req.uid, 'new'), - unreadWatchedTopicCount: async.apply(topics.getTotalUnread, req.uid, 'watched'), + unreadCounts: async.apply(topics.getUnreadTids, { uid: req.uid, count: true }), unreadChatCount: async.apply(messaging.getUnreadCount, req.uid), unreadNotificationCount: async.apply(user.notifications.getUnreadCount, req.uid), }, next); @@ -137,6 +135,7 @@ module.exports = function (middleware) { results.user.isGlobalMod = results.isGlobalMod; results.user.isMod = !!results.isModerator; results.user.privileges = results.privileges; + results.user[results.user.status] = true; results.user.uid = parseInt(results.user.uid, 10); results.user.email = String(results.user.email); @@ -146,12 +145,14 @@ module.exports = function (middleware) { setBootswatchCSS(templateValues, res.locals.config); var unreadCount = { - topic: results.unreadTopicCount || 0, - newTopic: results.unreadNewTopicCount || 0, - watchedTopic: results.unreadWatchedTopicCount || 0, + topic: results.unreadCounts[''] || 0, + newTopic: results.unreadCounts.new || 0, + watchedTopic: results.unreadCounts.watched || 0, + unrepliedTopic: results.unreadCounts.unreplied || 0, chat: results.unreadChatCount || 0, notification: results.unreadNotificationCount || 0, }; + Object.keys(unreadCount).forEach(function (key) { if (unreadCount[key] > 99) { unreadCount[key] = '99+'; @@ -159,25 +160,18 @@ module.exports = function (middleware) { }); results.navigation = results.navigation.map(function (item) { - if (item.originalRoute === '/unread' && results.unreadTopicCount > 0) { - return Object.assign({}, item, { - content: unreadCount.topic, - iconClass: item.iconClass + ' unread-count', - }); + function modifyNavItem(item, route, count, content) { + if (item && item.originalRoute === route) { + item.content = content; + if (count > 0) { + item.iconClass += ' unread-count'; + } + } } - if (item.originalRoute === '/unread/new' && results.unreadNewTopicCount > 0) { - return Object.assign({}, item, { - content: unreadCount.newTopic, - iconClass: item.iconClass + ' unread-count', - }); - } - if (item.originalRoute === '/unread/watched' && results.unreadWatchedTopicCount > 0) { - return Object.assign({}, item, { - content: unreadCount.watchedTopic, - iconClass: item.iconClass + ' unread-count', - }); - } - + modifyNavItem(item, '/unread', results.unreadCounts[''], unreadCount.topic); + modifyNavItem(item, '/unread?filter=new', results.unreadCounts.new, unreadCount.newTopic); + modifyNavItem(item, '/unread?filter=watched', results.unreadCounts.watched, unreadCount.watchedTopic); + modifyNavItem(item, '/unread?filter=unreplied', results.unreadCounts.unreplied, unreadCount.unrepliedTopic); return item; }); diff --git a/src/middleware/index.js b/src/middleware/index.js index 0173ecb3c6..5eba1ed3dc 100644 --- a/src/middleware/index.js +++ b/src/middleware/index.js @@ -2,13 +2,11 @@ var async = require('async'); var path = require('path'); -var fs = require('fs'); var csrf = require('csurf'); var validator = require('validator'); var nconf = require('nconf'); var ensureLoggedIn = require('connect-ensure-login'); var toobusy = require('toobusy-js'); -var Benchpress = require('benchpressjs'); var LRU = require('lru-cache'); var plugins = require('../plugins'); @@ -145,8 +143,14 @@ middleware.privateUploads = function (req, res, next) { if (req.loggedIn || parseInt(meta.config.privateUploads, 10) !== 1) { return next(); } + if (req.path.startsWith(nconf.get('relative_path') + '/assets/uploads/files')) { - return res.status(403).json('not-allowed'); + var extensions = (meta.config.privateUploadsExtensions || '').split(',').filter(Boolean); + var ext = path.extname(req.path); + ext = ext ? ext.replace(/^\./, '') : ext; + if (!extensions.length || extensions.includes(ext)) { + return res.status(403).json('not-allowed'); + } } next(); }; @@ -201,58 +205,3 @@ middleware.delayLoading = function (req, res, next) { setTimeout(next, 1000); }; - -var viewsDir = nconf.get('views_dir'); -var workingCache = {}; - -middleware.templatesOnDemand = function (req, res, next) { - var filePath = req.filePath || path.join(viewsDir, req.path); - if (!filePath.endsWith('.js')) { - return next(); - } - var tplPath = filePath.replace(/\.js$/, '.tpl'); - if (workingCache[filePath]) { - workingCache[filePath].push(next); - return; - } - - async.waterfall([ - function (cb) { - file.exists(filePath, cb); - }, - function (exists, cb) { - if (exists) { - return next(); - } - - // need to check here again - // because compilation could have started since last check - if (workingCache[filePath]) { - workingCache[filePath].push(next); - return; - } - - workingCache[filePath] = [next]; - fs.readFile(tplPath, 'utf8', cb); - }, - function (source, cb) { - Benchpress.precompile({ - source: source, - minify: global.env !== 'development', - }, cb); - }, - function (compiled, cb) { - if (!compiled) { - return cb(new Error('[[error:templatesOnDemand.compiled-template-empty, ' + tplPath + ']]')); - } - fs.writeFile(filePath, compiled, cb); - }, - ], function (err) { - var arr = workingCache[filePath]; - workingCache[filePath] = null; - - arr.forEach(function (callback) { - callback(err); - }); - }); -}; diff --git a/src/middleware/user.js b/src/middleware/user.js index 17c52e7ca6..3cc69a750d 100644 --- a/src/middleware/user.js +++ b/src/middleware/user.js @@ -8,6 +8,8 @@ var user = require('../user'); var privileges = require('../privileges'); var plugins = require('../plugins'); +var auth = require('../routes/authentication'); + var controllers = { helpers: require('../controllers/helpers'), }; @@ -22,7 +24,19 @@ module.exports = function (middleware) { return plugins.fireHook('action:middleware.authenticate', { req: req, res: res, - next: next, + next: function (err) { + if (err) { + return next(err); + } + + auth.setAuthVars(req, res, function () { + if (req.loggedIn && req.user && req.user.uid) { + return next(); + } + + controllers.helpers.notAllowed(req, res); + }); + }, }); } diff --git a/src/navigation/admin.js b/src/navigation/admin.js index 8bab6d4219..308445db4c 100644 --- a/src/navigation/admin.js +++ b/src/navigation/admin.js @@ -15,12 +15,13 @@ pubsub.on('admin:navigation:save', function () { admin.save = function (data, callback) { var order = Object.keys(data); - var items = data.map(function (item) { + var items = data.map(function (item, index) { for (var i in item) { if (item.hasOwnProperty(i) && typeof item[i] === 'string' && (i === 'title' || i === 'text')) { item[i] = translator.escape(item[i]); } } + item.order = order[index]; return JSON.stringify(item); }); diff --git a/src/notifications.js b/src/notifications.js index 32adb67fe3..e45625e6e4 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -608,3 +608,5 @@ Notifications.merge = function (notifications, callback) { callback(err, data.notifications); }); }; + +Notifications.async = require('./promisify')(Notifications); diff --git a/src/posts.js b/src/posts.js index cd58d81527..080a143f94 100644 --- a/src/posts.js +++ b/src/posts.js @@ -321,3 +321,5 @@ Posts.modifyPostByPrivilege = function (post, privileges) { } } }; + +Posts.async = require('./promisify')(Posts); diff --git a/src/posts/delete.js b/src/posts/delete.js index 44ffb12d87..14cd9dd3b1 100644 --- a/src/posts/delete.js +++ b/src/posts/delete.js @@ -31,7 +31,7 @@ module.exports = function (Posts) { postData.cid = topicData.cid; async.parallel([ function (next) { - updateTopicTimestamp(topicData, next); + topics.updateLastPostTimeFromLastPid(postData.tid, next); }, function (next) { db.sortedSetRemove('cid:' + topicData.cid + ':pids', pid, next); @@ -68,7 +68,7 @@ module.exports = function (Posts) { postData.cid = topicData.cid; async.parallel([ function (next) { - updateTopicTimestamp(topicData, next); + topics.updateLastPostTimeFromLastPid(topicData.tid, next); }, function (next) { db.sortedSetAdd('cid:' + topicData.cid + ':pids', postData.timestamp, pid, next); @@ -85,35 +85,6 @@ module.exports = function (Posts) { ], callback); }; - function updateTopicTimestamp(topicData, callback) { - var timestamp; - async.waterfall([ - function (next) { - topics.getLatestUndeletedPid(topicData.tid, next); - }, - function (pid, next) { - if (!parseInt(pid, 10)) { - return callback(); - } - Posts.getPostField(pid, 'timestamp', next); - }, - function (_timestamp, next) { - timestamp = _timestamp; - if (!parseInt(timestamp, 10)) { - return callback(); - } - topics.updateTimestamp(topicData.tid, timestamp, next); - }, - function (next) { - if (parseInt(topicData.pinned, 10) !== 1) { - db.sortedSetAdd('cid:' + topicData.cid + ':tids', timestamp, topicData.tid, next); - } else { - next(); - } - }, - ], callback); - } - Posts.purge = function (pid, uid, callback) { async.waterfall([ function (next) { @@ -194,10 +165,14 @@ module.exports = function (Posts) { topics.updateTeaser(postData.tid, next); }, function (next) { - updateTopicTimestamp(topicData, next); + topics.updateLastPostTimeFromLastPid(postData.tid, next); }, function (next) { - db.sortedSetIncrBy('cid:' + topicData.cid + ':tids:posts', -1, postData.tid, next); + if (parseInt(topicData.pinned, 10) !== 1) { + db.sortedSetIncrBy('cid:' + topicData.cid + ':tids:posts', -1, postData.tid, next); + } else { + next(); + } }, function (next) { db.sortedSetIncrBy('tid:' + postData.tid + ':posters', -1, postData.uid, next); diff --git a/src/posts/queue.js b/src/posts/queue.js index 6d8001f2f7..c1fb332450 100644 --- a/src/posts/queue.js +++ b/src/posts/queue.js @@ -134,7 +134,7 @@ module.exports = function (Posts) { db.delete('post:queue:' + id, next); }, function (next) { - notifications.rescind('post-queued-' + id, next); + notifications.rescind('post-queue-' + id, next); }, ], callback); }; diff --git a/src/privileges.js b/src/privileges.js index 9ed1664ced..add204c029 100644 --- a/src/privileges.js +++ b/src/privileges.js @@ -49,3 +49,5 @@ require('./privileges/categories')(privileges); require('./privileges/topics')(privileges); require('./privileges/posts')(privileges); require('./privileges/users')(privileges); + +privileges.async = require('./promisify')(privileges); diff --git a/src/privileges/global.js b/src/privileges/global.js index 554adf0bed..5d0cb88213 100644 --- a/src/privileges/global.js +++ b/src/privileges/global.js @@ -21,6 +21,7 @@ module.exports = function (privileges) { { name: '[[admin/manage/privileges:search-content]]' }, { name: '[[admin/manage/privileges:search-users]]' }, { name: '[[admin/manage/privileges:search-tags]]' }, + { name: '[[admin/manage/privileges:allow-local-login]]' }, ]; privileges.global.userPrivilegeList = [ @@ -32,6 +33,7 @@ module.exports = function (privileges) { 'search:content', 'search:users', 'search:tags', + 'local:login', ]; privileges.global.groupPrivilegeList = privileges.global.userPrivilegeList.map(function (privilege) { @@ -111,6 +113,10 @@ module.exports = function (privileges) { ], callback); }; + privileges.global.canGroup = function (privilege, groupName, callback) { + groups.isMember(groupName, 'cid:0:privileges:groups:' + privilege, callback); + }; + privileges.global.give = function (privileges, groupName, callback) { helpers.giveOrRescind(groups.join, privileges, 0, groupName, callback); }; diff --git a/src/privileges/topics.js b/src/privileges/topics.js index 0cbf49e7ae..9e387acc77 100644 --- a/src/privileges/topics.js +++ b/src/privileges/topics.js @@ -16,7 +16,7 @@ module.exports = function (privileges) { privileges.topics.get = function (tid, uid, callback) { var topic; - var privs = ['topics:reply', 'topics:read', 'topics:tag', 'topics:delete', 'posts:edit', 'posts:history', 'posts:delete', 'posts:view_deleted', 'read']; + var privs = ['topics:reply', 'topics:read', 'topics:tag', 'topics:delete', 'posts:edit', 'posts:history', 'posts:delete', 'posts:view_deleted', 'read', 'purge']; async.waterfall([ async.apply(topics.getTopicFields, tid, ['cid', 'uid', 'locked', 'deleted']), function (_topic, next) { @@ -37,6 +37,7 @@ module.exports = function (privileges) { var isAdminOrMod = results.isAdministrator || results.isModerator; var editable = isAdminOrMod; var deletable = isAdminOrMod || (isOwner && privData['topics:delete']); + var purge = results.isAdministrator || privData.purge; plugins.fireHook('filter:privileges.topics.get', { 'topics:reply': (privData['topics:reply'] && !locked && !deleted) || isAdminOrMod, @@ -51,6 +52,7 @@ module.exports = function (privileges) { view_thread_tools: editable || deletable, editable: editable, deletable: deletable, + purge: purge, view_deleted: isAdminOrMod || isOwner, isAdminOrMod: isAdminOrMod, disabled: disabled, diff --git a/src/promisify.js b/src/promisify.js new file mode 100644 index 0000000000..9882cf2753 --- /dev/null +++ b/src/promisify.js @@ -0,0 +1,37 @@ +'use strict'; + +// remove once node 6 support is removed +require('util.promisify/shim')(); + +var util = require('util'); +var _ = require('lodash'); + +module.exports = function (theModule, ignoreKeys) { + ignoreKeys = ignoreKeys || []; + function isCallbackedFunction(func) { + if (typeof func !== 'function') { + return false; + } + var str = func.toString().split('\n')[0]; + return str.includes('callback)'); + } + function promisifyRecursive(module) { + if (!module) { + return; + } + var keys = Object.keys(module); + keys.forEach(function (key) { + if (ignoreKeys.includes(key)) { + return; + } + if (isCallbackedFunction(module[key])) { + module[key] = util.promisify(module[key]); + } else if (typeof module[key] === 'object') { + promisifyRecursive(module[key]); + } + }); + } + const asyncModule = _.cloneDeep(theModule); + promisifyRecursive(asyncModule); + return asyncModule; +}; diff --git a/src/pubsub.js b/src/pubsub.js index 6b6f12d171..ee6185be20 100644 --- a/src/pubsub.js +++ b/src/pubsub.js @@ -1,5 +1,6 @@ 'use strict'; +var EventEmitter = require('events'); var nconf = require('nconf'); var real; @@ -12,17 +13,35 @@ function get() { var pubsub; if (nconf.get('isCluster') === 'false') { - var EventEmitter = require('events'); pubsub = new EventEmitter(); pubsub.publish = pubsub.emit.bind(pubsub); + } else if (nconf.get('singleHostCluster')) { + pubsub = new EventEmitter(); + if (!process.send) { + pubsub.publish = pubsub.emit.bind(pubsub); + } else { + pubsub.publish = function (event, data) { + process.send({ + action: 'pubsub', + event: event, + data: data, + }); + }; + process.on('message', function (message) { + if (message && typeof message === 'object' && message.action === 'pubsub') { + pubsub.emit(message.event, message.data); + } + }); + } } else if (nconf.get('redis')) { pubsub = require('./database/redis/pubsub'); } else if (nconf.get('mongo')) { pubsub = require('./database/mongo/pubsub'); + } else if (nconf.get('postgres')) { + pubsub = require('./database/postgres/pubsub'); } real = pubsub; - return pubsub; } diff --git a/src/routes/accounts.js b/src/routes/accounts.js index 9febb67391..e5b6198ed1 100644 --- a/src/routes/accounts.js +++ b/src/routes/accounts.js @@ -14,6 +14,7 @@ module.exports = function (app, middleware, controllers) { setupPageRoute(app, '/user/:userslug/following', middleware, middlewares, controllers.accounts.follow.getFollowing); setupPageRoute(app, '/user/:userslug/followers', middleware, middlewares, controllers.accounts.follow.getFollowers); + setupPageRoute(app, '/user/:userslug/categories', middleware, middlewares, controllers.accounts.categories.get); setupPageRoute(app, '/user/:userslug/posts', middleware, middlewares, controllers.accounts.posts.getPosts); setupPageRoute(app, '/user/:userslug/topics', middleware, middlewares, controllers.accounts.posts.getTopics); setupPageRoute(app, '/user/:userslug/best', middleware, middlewares, controllers.accounts.posts.getBestPosts); @@ -33,8 +34,8 @@ module.exports = function (app, middleware, controllers) { setupPageRoute(app, '/user/:userslug/uploads', middleware, accountMiddlewares, controllers.accounts.uploads.get); setupPageRoute(app, '/user/:userslug/consent', middleware, accountMiddlewares, controllers.accounts.consent.get); setupPageRoute(app, '/user/:userslug/blocks', middleware, accountMiddlewares, controllers.accounts.blocks.getBlocks); - - app.delete('/api/user/:userslug/session/:uuid', [middleware.exposeUid, middleware.ensureSelfOrGlobalPrivilege], controllers.accounts.session.revoke); + setupPageRoute(app, '/user/:userslug/sessions', middleware, accountMiddlewares, controllers.accounts.sessions.get); + app.delete('/api/user/:userslug/session/:uuid', [middleware.exposeUid, middleware.ensureSelfOrGlobalPrivilege], controllers.accounts.sessions.revoke); setupPageRoute(app, '/notifications', middleware, [middleware.authenticate], controllers.accounts.notifications.get); setupPageRoute(app, '/user/:userslug/chats/:roomid?', middleware, middlewares, controllers.accounts.chats.get); diff --git a/src/routes/authentication.js b/src/routes/authentication.js index 986cc31ed7..8e743cc988 100644 --- a/src/routes/authentication.js +++ b/src/routes/authentication.js @@ -19,23 +19,25 @@ Auth.initialize = function (app, middleware) { app.use(passport.initialize()); app.use(passport.session()); - app.use(function (req, res, next) { - var isSpider = req.isSpider(); - req.loggedIn = !isSpider && !!req.user; - if (isSpider) { - req.uid = -1; - } else if (req.user) { - req.uid = parseInt(req.user.uid, 10); - } else { - req.uid = 0; - } - next(); - }); + app.use(Auth.setAuthVars); Auth.app = app; Auth.middleware = middleware; }; +Auth.setAuthVars = function (req, res, next) { + var isSpider = req.isSpider(); + req.loggedIn = !isSpider && !!req.user; + if (isSpider) { + req.uid = -1; + } else if (req.user) { + req.uid = parseInt(req.user.uid, 10); + } else { + req.uid = 0; + } + next(); +}; + Auth.getLoginStrategies = function () { return loginStrategies; }; @@ -85,7 +87,8 @@ Auth.reloadRoutes = function (callback) { router.post('/register', Auth.middleware.applyCSRF, Auth.middleware.applyBlacklist, controllers.authentication.register); router.post('/register/complete', Auth.middleware.applyCSRF, Auth.middleware.applyBlacklist, controllers.authentication.registerComplete); - router.get('/register/abort', controllers.authentication.registerAbort); + // router.get('/register/abort', controllers.authentication.registerAbort); + router.post('/register/abort', controllers.authentication.registerAbort); router.post('/login', Auth.middleware.applyCSRF, Auth.middleware.applyBlacklist, controllers.authentication.login); router.post('/logout', Auth.middleware.applyCSRF, controllers.authentication.logout); diff --git a/src/routes/feeds.js b/src/routes/feeds.js index f843e08e7a..07e1cf2379 100644 --- a/src/routes/feeds.js +++ b/src/routes/feeds.js @@ -26,6 +26,7 @@ var terms = { module.exports = function (app, middleware) { app.get('/topic/:topic_id.rss', middleware.maintenanceMode, generateForTopic); app.get('/category/:category_id.rss', middleware.maintenanceMode, generateForCategory); + app.get('/topics.rss', middleware.maintenanceMode, generateForTopics); app.get('/recent.rss', middleware.maintenanceMode, generateForRecent); app.get('/top.rss', middleware.maintenanceMode, generateForTop); app.get('/top/:term.rss', middleware.maintenanceMode, generateForTop); @@ -153,7 +154,9 @@ function generateForCategory(req, res, next) { } var cid = req.params.category_id; var category; - + if (!parseInt(cid, 10)) { + return next(); + } async.waterfall([ function (next) { async.parallel({ @@ -191,6 +194,31 @@ function generateForCategory(req, res, next) { ], next); } +function generateForTopics(req, res, next) { + if (parseInt(meta.config['feeds:disableRSS'], 10) === 1) { + return controllers404.send404(req, res); + } + + async.waterfall([ + function (next) { + if (req.query.token && req.query.uid) { + db.getObjectField('user:' + req.query.uid, 'rss_token', next); + } else { + next(null, null); + } + }, + function (token, next) { + sendTopicsFeed({ + uid: token && token === req.query.token ? req.query.uid : req.uid, + title: 'Most recently created topics', + description: 'A list of topics that have been created recently', + feed_url: '/topics.rss', + useMainPost: true, + }, 'topics:tid', req, res, next); + }, + ], next); +} + function generateForRecent(req, res, next) { if (parseInt(meta.config['feeds:disableRSS'], 10) === 1) { return controllers404.send404(req, res); @@ -205,7 +233,7 @@ function generateForRecent(req, res, next) { } }, function (token, next) { - generateForTopics({ + sendTopicsFeed({ uid: token && token === req.query.token ? req.query.uid : req.uid, title: 'Recently Active Topics', description: 'A list of topics that have been active within the past 24 hours', @@ -297,7 +325,7 @@ function generateForPopular(req, res, next) { ], next); } -function generateForTopics(options, set, req, res, next) { +function sendTopicsFeed(options, set, req, res, next) { var start = options.hasOwnProperty('start') ? options.start : 0; var stop = options.hasOwnProperty('stop') ? options.stop : 19; async.waterfall([ @@ -326,14 +354,14 @@ function generateTopicsFeed(feedOptions, feedTopics, callback) { feed.pubDate = new Date(parseInt(feedTopics[0].lastposttime, 10)).toUTCString(); } - async.each(feedTopics, function (topicData, next) { + async.eachSeries(feedTopics, function (topicData, next) { var feedItem = { title: utils.stripHTMLTags(topicData.title, utils.tags), url: nconf.get('url') + '/topic/' + topicData.slug, date: new Date(parseInt(topicData.lastposttime, 10)).toUTCString(), }; - if (topicData.teaser && topicData.teaser.user) { + if (topicData.teaser && topicData.teaser.user && !feedOptions.useMainPost) { feedItem.description = topicData.teaser.content; feedItem.author = topicData.teaser.user.username; feed.item(feedItem); @@ -464,7 +492,7 @@ function generateForUserTopics(req, res, callback) { user.getUserFields(uid, ['uid', 'username'], next); }, function (userData, next) { - generateForTopics({ + sendTopicsFeed({ uid: req.uid, title: 'Topics by ' + userData.username, description: 'A list of topics that are posted by ' + userData.username, @@ -484,7 +512,7 @@ function generateForTag(req, res, next) { var topicsPerPage = meta.config.topicsPerPage || 20; var start = Math.max(0, (page - 1) * topicsPerPage); var stop = start + topicsPerPage - 1; - generateForTopics({ + sendTopicsFeed({ uid: req.uid, title: 'Topics tagged with ' + tag, description: 'A list of topics that have been tagged with ' + tag, diff --git a/src/routes/index.js b/src/routes/index.js index 0b5b37c7b2..1d5d4fe1ee 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -152,7 +152,6 @@ module.exports = function (app, middleware, hotswapIds, callback) { } app.use(middleware.privateUploads); - app.use(relativePath + '/assets/templates', middleware.templatesOnDemand); var statics = [ { route: '/assets', path: path.join(__dirname, '../../build/public') }, diff --git a/src/search.js b/src/search.js index 89f8cb66ab..5cb8263bc1 100644 --- a/src/search.js +++ b/src/search.js @@ -315,7 +315,7 @@ function filterByTags(posts, hasTags) { var hasAllTags = false; if (post && post.topic && Array.isArray(post.topic.tags) && post.topic.tags.length) { hasAllTags = hasTags.every(function (tag) { - return post.topic.tags.indexOf(tag) !== -1; + return post.topic.tags.includes(tag); }); } return hasAllTags; @@ -370,23 +370,15 @@ function getSearchCids(data, callback) { return callback(null, []); } - if (data.categories.indexOf('all') !== -1) { - async.waterfall([ - function (next) { - db.getSortedSetRange('categories:cid', 0, -1, next); - }, - function (cids, next) { - privileges.categories.filterCids('read', cids, data.uid, next); - }, - ], callback); - return; + if (data.categories.includes('all')) { + return categories.getCidsByPrivilege('categories:cid', data.uid, 'read', callback); } async.waterfall([ function (next) { async.parallel({ watchedCids: function (next) { - if (data.categories.indexOf('watched') !== -1) { + if (data.categories.includes('watched')) { user.getWatchedCategories(data.uid, next); } else { next(null, []); diff --git a/src/socket.io/admin.js b/src/socket.io/admin.js index 3f9cd91666..c13132c973 100644 --- a/src/socket.io/admin.js +++ b/src/socket.io/admin.js @@ -178,6 +178,14 @@ SocketAdmin.config.setMultiple = function (socket, data, callback) { return callback(new Error('[[error:invalid-data]]')); } + var changes = {}; + Object.keys(data).forEach(function (key) { + if (data[key] !== meta.config[key]) { + changes[key] = data[key]; + changes[key + '_old'] = meta.config[key]; + } + }); + async.waterfall([ function (next) { meta.configs.setMultiple(data, next); @@ -194,10 +202,15 @@ SocketAdmin.config.setMultiple = function (socket, data, callback) { logger.monitorConfig({ io: index.server }, setting); } } - data.type = 'config-change'; - data.uid = socket.uid; - data.ip = socket.ip; - events.log(data, next); + + if (Object.keys(changes).length) { + changes.type = 'config-change'; + changes.uid = socket.uid; + changes.ip = socket.ip; + events.log(changes, next); + } else { + next(); + } }, ], callback); }; diff --git a/src/socket.io/admin/categories.js b/src/socket.io/admin/categories.js index 6ccc20c873..4bab1330d5 100644 --- a/src/socket.io/admin/categories.js +++ b/src/socket.io/admin/categories.js @@ -84,8 +84,10 @@ Categories.setPrivilege = function (socket, data, callback) { function onSetComplete() { events.log({ uid: socket.uid, + type: 'privilege-change', ip: socket.ip, - privilege: data.privilege, + privilege: data.privilege.toString(), + cid: data.cid, action: data.set ? 'grant' : 'rescind', target: data.member, }, callback); diff --git a/src/socket.io/admin/user.js b/src/socket.io/admin/user.js index d8d9a0f282..c15e41fec6 100644 --- a/src/socket.io/admin/user.js +++ b/src/socket.io/admin/user.js @@ -2,6 +2,7 @@ var async = require('async'); var validator = require('validator'); +var winston = require('winston'); var db = require('../../database'); var groups = require('../../groups'); @@ -146,37 +147,48 @@ function deleteUsers(socket, uids, method, callback) { if (!Array.isArray(uids)) { return callback(new Error('[[error:invalid-data]]')); } + async.waterfall([ + function (next) { + groups.isMembers(uids, 'administrators', next); + }, + function (isMembers, next) { + if (isMembers.includes(true)) { + return callback(new Error('[[error:cant-delete-other-admins]]')); + } - async.each(uids, function (uid, next) { - async.waterfall([ - function (next) { - user.isAdministrator(uid, next); - }, - function (isAdmin, next) { - if (isAdmin) { - return next(new Error('[[error:cant-delete-other-admins]]')); - } + callback(); - method(uid, next); - }, - function (next) { - events.log({ - type: 'user-delete', - uid: socket.uid, - targetUid: uid, - ip: socket.ip, - }, next); - }, - function (next) { - plugins.fireHook('action:user.delete', { - callerUid: socket.uid, - uid: uid, - ip: socket.ip, - }); - next(); - }, - ], next); - }, callback); + async.each(uids, function (uid, next) { + async.waterfall([ + function (next) { + method(uid, next); + }, + function (userData, next) { + events.log({ + type: 'user-delete', + uid: socket.uid, + targetUid: uid, + ip: socket.ip, + username: userData.username, + email: userData.email, + }, next); + }, + function (next) { + plugins.fireHook('action:user.delete', { + callerUid: socket.uid, + uid: uid, + ip: socket.ip, + }); + next(); + }, + ], next); + }, next); + }, + ], function (err) { + if (err) { + winston.error(err); + } + }); } User.search = function (socket, data, callback) { diff --git a/src/socket.io/blacklist.js b/src/socket.io/blacklist.js index d4f1481508..75b11ce30a 100644 --- a/src/socket.io/blacklist.js +++ b/src/socket.io/blacklist.js @@ -5,6 +5,7 @@ var async = require('async'); var user = require('../user'); var meta = require('../meta'); +var events = require('../events'); var SocketBlacklist = module.exports; @@ -13,21 +14,14 @@ SocketBlacklist.validate = function (socket, data, callback) { }; SocketBlacklist.save = function (socket, rules, callback) { - async.waterfall([ - function (next) { - user.isAdminOrGlobalMod(socket.uid, next); - }, - function (isAdminOrGlobalMod, next) { - if (!isAdminOrGlobalMod) { - return callback(new Error('[[error:no-privileges]]')); - } - - meta.blacklist.save(rules, next); - }, - ], callback); + blacklist(socket, 'save', rules, callback); }; SocketBlacklist.addRule = function (socket, rule, callback) { + blacklist(socket, 'addRule', rule, callback); +}; + +function blacklist(socket, method, rule, callback) { async.waterfall([ function (next) { user.isAdminOrGlobalMod(socket.uid, next); @@ -37,7 +31,15 @@ SocketBlacklist.addRule = function (socket, rule, callback) { return callback(new Error('[[error:no-privileges]]')); } - meta.blacklist.addRule(rule, next); + meta.blacklist[method](rule, next); + }, + function (next) { + events.log({ + type: 'ip-blacklist-' + method, + uid: socket.uid, + ip: socket.ip, + rule: rule, + }, next); }, ], callback); -}; +} diff --git a/src/socket.io/categories.js b/src/socket.io/categories.js index 9dfb285d82..97bdc7d44c 100644 --- a/src/socket.io/categories.js +++ b/src/socket.io/categories.js @@ -174,7 +174,17 @@ SocketCategories.ignore = function (socket, cid, callback) { }; function ignoreOrWatch(fn, socket, cid, callback) { + var targetUid = socket.uid; + var cids = [parseInt(cid, 10)]; + if (typeof cid === 'object') { + targetUid = cid.uid; + cids = [parseInt(cid.cid, 10)]; + } + async.waterfall([ + function (next) { + user.isAdminOrGlobalModOrSelf(socket.uid, targetUid, next); + }, function (next) { db.getSortedSetRange('categories:cid', 0, -1, next); }, @@ -187,10 +197,7 @@ function ignoreOrWatch(fn, socket, cid, callback) { c.parentCid = parseInt(c.parentCid, 10); }); - var cids = [parseInt(cid, 10)]; - // filter to subcategories of cid - var cat; do { cat = categoryData.find(function (c) { @@ -202,11 +209,14 @@ function ignoreOrWatch(fn, socket, cid, callback) { } while (cat); async.each(cids, function (cid, next) { - fn(socket.uid, cid, next); + fn(targetUid, cid, next); }, next); }, function (next) { - topics.pushUnreadCount(socket.uid, next); + topics.pushUnreadCount(targetUid, next); + }, + function (next) { + next(null, cids); }, ], callback); } diff --git a/src/socket.io/helpers.js b/src/socket.io/helpers.js index 4679b73598..1688d88252 100644 --- a/src/socket.io/helpers.js +++ b/src/socket.io/helpers.js @@ -31,6 +31,12 @@ SocketHelpers.notifyNew = function (uid, type, result) { function (uids, next) { filterTidCidIgnorers(uids, result.posts[0].topic.tid, result.posts[0].topic.cid, next); }, + function (uids, next) { + user.blocks.filterUids(uid, uids, next); + }, + function (uids, next) { + user.blocks.filterUids(result.posts[0].topic.uid, uids, next); + }, function (uids, next) { plugins.fireHook('filter:sockets.sendNewPostToUids', { uidsTo: uids, uidFrom: uid, type: type }, next); }, @@ -186,9 +192,15 @@ SocketHelpers.upvote = function (data, notification) { all: function () { return votes > 0; }, + first: function () { + return votes === 1; + }, everyTen: function () { return votes > 0 && votes % 10 === 0; }, + threshold: function () { + return [1, 5, 10, 25].indexOf(votes) !== -1 || (votes >= 50 && votes % 50 === 0); + }, logarithmic: function () { return votes > 1 && Math.log10(votes) % 1 === 0; }, diff --git a/src/socket.io/index.js b/src/socket.io/index.js index 884a7e432c..4c772db131 100644 --- a/src/socket.io/index.js +++ b/src/socket.io/index.js @@ -27,7 +27,13 @@ Sockets.init = function (server) { path: nconf.get('relative_path') + '/socket.io', }); - io.adapter(nconf.get('redis') ? require('../database/redis').socketAdapter() : db.socketAdapter()); + if (nconf.get('singleHostCluster')) { + io.adapter(require('./single-host-cluster')); + } else if (nconf.get('redis')) { + io.adapter(require('../database/redis').socketAdapter()); + } else { + io.adapter(db.socketAdapter()); + } io.use(socketioWildcard); io.use(authorize); diff --git a/src/socket.io/posts/edit.js b/src/socket.io/posts/edit.js index 48f750c2ce..296e3e2cb4 100644 --- a/src/socket.io/posts/edit.js +++ b/src/socket.io/posts/edit.js @@ -51,6 +51,7 @@ module.exports = function (SocketPosts) { type: 'topic-rename', uid: socket.uid, ip: socket.ip, + tid: result.topic.tid, oldTitle: validator.escape(String(result.topic.oldTitle)), newTitle: validator.escape(String(result.topic.title)), }); diff --git a/src/socket.io/posts/tools.js b/src/socket.io/posts/tools.js index 9b2da8ee01..c50bed33ca 100644 --- a/src/socket.io/posts/tools.js +++ b/src/socket.io/posts/tools.js @@ -39,6 +39,9 @@ module.exports = function (SocketPosts) { canDelete: function (next) { privileges.posts.canDelete(data.pid, socket.uid, next); }, + canPurge: function (next) { + privileges.posts.canPurge(data.pid, socket.uid, next); + }, canFlag: function (next) { privileges.posts.canFlag(data.pid, socket.uid, next); }, @@ -62,6 +65,7 @@ module.exports = function (SocketPosts) { posts.selfPost = socket.uid && socket.uid === parseInt(posts.uid, 10); posts.display_edit_tools = results.canEdit.flag; posts.display_delete_tools = results.canDelete.flag; + posts.display_purge_tools = results.canPurge; posts.display_flag_tools = socket.uid && !posts.selfPost && results.canFlag.flag; posts.display_moderator_tools = posts.display_edit_tools || posts.display_delete_tools; posts.display_move_tools = results.isAdmin || results.isModerator; @@ -104,6 +108,7 @@ module.exports = function (SocketPosts) { type: 'post-delete', uid: socket.uid, pid: data.pid, + tid: postData.tid, ip: socket.ip, }); @@ -139,6 +144,7 @@ module.exports = function (SocketPosts) { type: 'post-restore', uid: socket.uid, pid: data.pid, + tid: postData.tid, ip: socket.ip, }); @@ -200,6 +206,7 @@ module.exports = function (SocketPosts) { uid: socket.uid, pid: data.pid, ip: socket.ip, + tid: postData.tid, title: String(topicData.title), }, next); }, diff --git a/src/socket.io/single-host-cluster.js b/src/socket.io/single-host-cluster.js new file mode 100644 index 0000000000..b2645d6eb9 --- /dev/null +++ b/src/socket.io/single-host-cluster.js @@ -0,0 +1,67 @@ +'use strict'; + +var Client = { + sendMessage: function (channel, message) { + process.send({ + action: 'socket.io', + channel: channel, + message: message, + }); + }, + trigger: function (channel, message) { + Client.message.concat(Client.pmessage).forEach(function (callback) { + setImmediate(function () { + callback.call(Client, channel, message); + }); + }); + }, + publish: function (channel, message) { + Client.sendMessage(channel, message); + }, + // we don't actually care about which channels we're subscribed to + subscribe: function () {}, + psubscribe: function () {}, + unsubscribe: function () {}, + unpsubscribe: function () {}, + message: [], + pmessage: [], + on: function (event, callback) { + if (event !== 'message' && event !== 'pmessage') { + return; + } + Client[event].push(callback); + }, + off: function (event, callback) { + if (event !== 'message' && event !== 'pmessage') { + return; + } + if (callback) { + Client[event] = Client[event].filter(function (c) { + return c !== callback; + }); + } else { + Client[event] = []; + } + }, +}; + +process.on('message', function (message) { + if (message && typeof message === 'object' && message.action === 'socket.io') { + Client.trigger(message.channel, message.message); + } +}); + +var adapter = require('socket.io-adapter-cluster')({ + client: Client, +}); +// Otherwise, every node thinks it is the master node and ignores messages +// because they are from "itself". +Object.defineProperty(adapter.prototype, 'id', { + get: function () { + return process.pid; + }, + set: function () { + // ignore + }, +}); +module.exports = adapter; diff --git a/src/socket.io/topics.js b/src/socket.io/topics.js index 89fca006c5..bbeda1bee5 100644 --- a/src/socket.io/topics.js +++ b/src/socket.io/topics.js @@ -8,6 +8,7 @@ var websockets = require('./index'); var user = require('../user'); var meta = require('../meta'); var apiController = require('../controllers/api'); +var privileges = require('../privileges'); var socketHelpers = require('./helpers'); var SocketTopics = module.exports; @@ -62,7 +63,18 @@ function postTopic(socket, data, callback) { } SocketTopics.postcount = function (socket, tid, callback) { - topics.getTopicField(tid, 'postcount', callback); + async.waterfall([ + function (next) { + privileges.topics.can('read', tid, socket.uid, next); + }, + function (canRead, next) { + if (!canRead) { + return next(new Error('[[no-privileges]]')); + } + + topics.getTopicField(tid, 'postcount', next); + }, + ], callback); }; SocketTopics.bookmark = function (socket, data, callback) { diff --git a/src/socket.io/user.js b/src/socket.io/user.js index 5401ccca44..ca21d0a38b 100644 --- a/src/socket.io/user.js +++ b/src/socket.io/user.js @@ -37,8 +37,14 @@ SocketUser.deleteAccount = function (socket, data, callback) { async.waterfall([ function (next) { - user.isPasswordCorrect(socket.uid, data.password, function (err, ok) { - next(err || !ok ? new Error('[[error:invalid-password]]') : undefined); + user.hasPassword(socket.uid, next); + }, + function (hasPassword, next) { + if (!hasPassword) { + return next(); + } + user.isPasswordCorrect(socket.uid, data.password, socket.ip, function (err, ok) { + next(err || (!ok ? new Error('[[error:invalid-password]]') : undefined)); }); }, function (next) { @@ -50,7 +56,7 @@ SocketUser.deleteAccount = function (socket, data, callback) { } user.deleteAccount(socket.uid, next); }, - function (next) { + function (userData, next) { require('./index').server.sockets.emit('event:user_status_change', { uid: socket.uid, status: 'offline' }); events.log({ @@ -58,18 +64,12 @@ SocketUser.deleteAccount = function (socket, data, callback) { uid: socket.uid, targetUid: socket.uid, ip: socket.ip, + username: userData.username, + email: userData.email, }); next(); }, - ], function (err) { - if (err) { - return setTimeout(function () { - callback(err); - }, 2500); - } - - callback(); - }); + ], callback); }; SocketUser.emailExists = function (socket, data, callback) { @@ -263,12 +263,19 @@ SocketUser.getUnreadCounts = function (socket, data, callback) { return callback(null, {}); } async.parallel({ - unreadTopicCount: async.apply(topics.getTotalUnread, socket.uid), - unreadNewTopicCount: async.apply(topics.getTotalUnread, socket.uid, 'new'), - unreadWatchedTopicCount: async.apply(topics.getTotalUnread, socket.uid, 'watched'), + unreadCounts: async.apply(topics.getUnreadTids, { uid: socket.uid, count: true }), unreadChatCount: async.apply(messaging.getUnreadCount, socket.uid), unreadNotificationCount: async.apply(user.notifications.getUnreadCount, socket.uid), - }, callback); + }, function (err, results) { + if (err) { + return callback(err); + } + results.unreadTopicCount = results.unreadCounts['']; + results.unreadNewTopicCount = results.unreadCounts.new; + results.unreadWatchedTopicCount = results.unreadCounts.watched; + results.unreadUnrepliedTopicCount = results.unreadCounts.unreplied; + callback(null, results); + }); }; SocketUser.invite = function (socket, email, callback) { @@ -290,24 +297,26 @@ SocketUser.invite = function (socket, email, callback) { if (registrationType === 'admin-invite-only' && !isAdmin) { return next(new Error('[[error:no-privileges]]')); } - var max = parseInt(meta.config.maximumInvites, 10); - if (!max) { - return user.sendInvitationEmail(socket.uid, email, callback); - } + email = email.split(',').map(email => email.trim()).filter(Boolean); + async.eachSeries(email, function (email, next) { + async.waterfall([ + function (next) { + if (max) { + user.getInvitesNumber(socket.uid, next); + } else { + next(null, 0); + } + }, + function (invites, next) { + if (!isAdmin && max && invites >= max) { + return next(new Error('[[error:invite-maximum-met, ' + invites + ', ' + max + ']]')); + } - async.waterfall([ - function (next) { - user.getInvitesNumber(socket.uid, next); - }, - function (invites, next) { - if (!isAdmin && invites >= max) { - return next(new Error('[[error:invite-maximum-met, ' + invites + ', ' + max + ']]')); - } - - user.sendInvitationEmail(socket.uid, email, next); - }, - ], next); + user.sendInvitationEmail(socket.uid, email, next); + }, + ], next); + }, next); }, ], callback); }; diff --git a/src/socket.io/user/ban.js b/src/socket.io/user/ban.js index 66f98eb061..b53398791f 100644 --- a/src/socket.io/user/ban.js +++ b/src/socket.io/user/ban.js @@ -3,6 +3,7 @@ var async = require('async'); var winston = require('winston'); +var db = require('../../database'); var user = require('../../user'); var meta = require('../../meta'); var websockets = require('../index'); @@ -22,7 +23,7 @@ module.exports = function (SocketUser) { toggleBan(socket.uid, data.uids, function (uid, next) { async.waterfall([ function (next) { - banUser(uid, data.until || 0, data.reason || '', next); + banUser(socket.uid, uid, data.until || 0, data.reason || '', next); }, function (next) { events.log({ @@ -30,6 +31,7 @@ module.exports = function (SocketUser) { uid: socket.uid, targetUid: uid, ip: socket.ip, + reason: data.reason || undefined, }, next); }, function (next) { @@ -38,6 +40,7 @@ module.exports = function (SocketUser) { ip: socket.ip, uid: uid, until: data.until > 0 ? data.until : undefined, + reason: data.reason || undefined, }); next(); }, @@ -92,7 +95,7 @@ module.exports = function (SocketUser) { ], callback); } - function banUser(uid, until, reason, callback) { + function banUser(callerUid, uid, until, reason, callback) { async.waterfall([ function (next) { user.isAdministrator(uid, next); @@ -123,6 +126,9 @@ module.exports = function (SocketUser) { function (next) { user.ban(uid, until, reason, next); }, + function (banData, next) { + db.setObjectField('uid:' + uid + ':ban:' + banData.timestamp, 'fromUid', callerUid, next); + }, function (next) { if (!reason) { return translator.translate('[[user:info.banned-no-reason]]', function (translated) { diff --git a/src/socket.io/user/profile.js b/src/socket.io/user/profile.js index 6cf76be9ee..857ac4ddf8 100644 --- a/src/socket.io/user/profile.js +++ b/src/socket.io/user/profile.js @@ -15,7 +15,7 @@ module.exports = function (SocketUser) { async.waterfall([ function (next) { - isPrivilegedOrSelfAndPasswordMatch(socket.uid, data, next); + isPrivilegedOrSelfAndPasswordMatch(socket, data, next); }, function (next) { SocketUser.updateProfile(socket, data, next); @@ -72,26 +72,19 @@ module.exports = function (SocketUser) { ], callback); }; - function isPrivilegedOrSelfAndPasswordMatch(uid, data, callback) { + function isPrivilegedOrSelfAndPasswordMatch(socket, data, callback) { + const uid = socket.uid; + const isSelf = parseInt(uid, 10) === parseInt(data.uid, 10); + async.waterfall([ function (next) { async.parallel({ isAdmin: async.apply(user.isAdministrator, uid), isTargetAdmin: async.apply(user.isAdministrator, data.uid), isGlobalMod: async.apply(user.isGlobalModerator, uid), - hasPassword: async.apply(user.hasPassword, data.uid), - passwordMatch: function (next) { - if (data.password) { - user.isPasswordCorrect(data.uid, data.password, next); - } else { - next(null, false); - } - }, }, next); }, function (results, next) { - var isSelf = parseInt(uid, 10) === parseInt(data.uid, 10); - if (results.isTargetAdmin && !results.isAdmin) { return next(new Error('[[error:no-privileges]]')); } @@ -100,6 +93,17 @@ module.exports = function (SocketUser) { return next(new Error('[[error:no-privileges]]')); } + async.parallel({ + hasPassword: async.apply(user.hasPassword, data.uid), + passwordMatch: function (next) { + if (data.password) { + user.isPasswordCorrect(data.uid, data.password, socket.ip, next); + } else { + next(null, false); + } + }, + }, next); + }, function (results, next) { if (isSelf && results.hasPassword && !results.passwordMatch) { return next(new Error('[[error:invalid-password]]')); } @@ -119,7 +123,7 @@ module.exports = function (SocketUser) { } async.waterfall([ function (next) { - user.changePassword(socket.uid, data, next); + user.changePassword(socket.uid, Object.assign(data, { ip: socket.ip }), next); }, function (next) { events.log({ @@ -201,24 +205,29 @@ module.exports = function (SocketUser) { }; SocketUser.toggleBlock = function (socket, data, callback) { - let current; + let isBlocked; async.waterfall([ function (next) { - user.blocks.can(socket.uid, data.blockerUid, data.blockeeUid, next); + async.parallel({ + can: function (next) { + user.blocks.can(socket.uid, data.blockerUid, data.blockeeUid, next); + }, + is: function (next) { + user.blocks.is(data.blockeeUid, data.blockerUid, next); + }, + }, next); }, - function (can, next) { - if (!can) { + function (results, next) { + isBlocked = results.is; + if (!results.can && !isBlocked) { return next(new Error('[[error:cannot-block-privileged]]')); } - user.blocks.is(data.blockeeUid, data.blockerUid, next); - }, - function (is, next) { - current = is; - user.blocks[is ? 'remove' : 'add'](data.blockeeUid, data.blockerUid, next); + + user.blocks[isBlocked ? 'remove' : 'add'](data.blockeeUid, data.blockerUid, next); }, ], function (err) { - callback(err, !current); + callback(err, !isBlocked); }); }; }; diff --git a/src/start.js b/src/start.js index 427a58830b..d16e4d3a34 100644 --- a/src/start.js +++ b/src/start.js @@ -118,19 +118,6 @@ function addProcessHandlers() { process.on('SIGTERM', shutdown); process.on('SIGINT', shutdown); process.on('SIGHUP', restart); - process.on('message', function (message) { - if (typeof message !== 'object') { - return; - } - var meta = require('./meta'); - - switch (message.action) { - case 'reload': - meta.reload(); - break; - } - }); - process.on('uncaughtException', function (err) { winston.error(err); diff --git a/src/topics.js b/src/topics.js index 5214b6a716..daf3db8790 100644 --- a/src/topics.js +++ b/src/topics.js @@ -114,7 +114,7 @@ Topics.getTopicsByTids = function (tids, uid, callback) { user.getMultipleUserSettings(uids, next); }, categories: function (next) { - categories.getCategoriesFields(cids, ['cid', 'name', 'slug', 'icon', 'image', 'bgColor', 'color', 'disabled'], next); + categories.getCategoriesFields(cids, ['cid', 'name', 'slug', 'icon', 'image', 'imageClass', 'bgColor', 'color', 'disabled'], next); }, hasRead: function (next) { Topics.hasReadTopics(tids, uid, next); @@ -368,3 +368,5 @@ Topics.search = function (tid, term, callback) { callback(err, Array.isArray(pids) ? pids : []); }); }; + +Topics.async = require('./promisify')(Topics); diff --git a/src/topics/create.js b/src/topics/create.js index fcd166dfde..4a0d1be942 100644 --- a/src/topics/create.js +++ b/src/topics/create.js @@ -298,7 +298,7 @@ module.exports = function (Topics) { posts.getUserInfoForPosts([postData.uid], uid, next); }, topicInfo: function (next) { - Topics.getTopicFields(tid, ['tid', 'title', 'slug', 'cid', 'postcount', 'mainPid'], next); + Topics.getTopicFields(tid, ['tid', 'uid', 'title', 'slug', 'cid', 'postcount', 'mainPid'], next); }, parents: function (next) { Topics.addParentPosts([postData], next); diff --git a/src/topics/data.js b/src/topics/data.js index 36a0d36206..80f4969103 100644 --- a/src/topics/data.js +++ b/src/topics/data.js @@ -92,12 +92,19 @@ module.exports = function (Topics) { if (!topic) { return; } + if (topic.hasOwnProperty('title')) { + topic.titleRaw = topic.title; + topic.title = String(topic.title); + } - topic.titleRaw = topic.title; - topic.title = String(topic.title); escapeTitle(topic); - topic.timestampISO = utils.toISOString(topic.timestamp); - topic.lastposttimeISO = utils.toISOString(topic.lastposttime); + if (topic.hasOwnProperty('timestamp')) { + topic.timestampISO = utils.toISOString(topic.timestamp); + } + if (topic.hasOwnProperty('lastposttime')) { + topic.lastposttimeISO = utils.toISOString(topic.lastposttime); + } + if (topic.hasOwnProperty('upvotes')) { topic.upvotes = parseInt(topic.upvotes, 10) || 0; } diff --git a/src/topics/fork.js b/src/topics/fork.js index 75591207a3..4ac4dfb113 100644 --- a/src/topics/fork.js +++ b/src/topics/fork.js @@ -68,7 +68,7 @@ module.exports = function (Topics) { }, next); }, function (next) { - Topics.updateTimestamp(tid, Date.now(), next); + Topics.updateLastPostTime(tid, Date.now(), next); }, function (next) { plugins.fireHook('action:topic.fork', { tid: tid, fromTid: fromTid, uid: uid }); @@ -106,13 +106,7 @@ module.exports = function (Topics) { function (next) { async.parallel([ function (next) { - updateCategoryPostCount(postData.tid, tid, next); - }, - function (next) { - Topics.decreasePostCount(postData.tid, next); - }, - function (next) { - Topics.increasePostCount(tid, next); + updateCategory(postData, tid, next); }, function (next) { posts.setPostField(pid, 'tid', tid, next); @@ -124,8 +118,8 @@ module.exports = function (Topics) { }, function (results, next) { async.parallel([ - async.apply(updateRecentTopic, tid), - async.apply(updateRecentTopic, postData.tid), + async.apply(Topics.updateLastPostTimeFromLastPid, tid), + async.apply(Topics.updateLastPostTimeFromLastPid, postData.tid), ], function (err) { next(err); }); @@ -137,40 +131,42 @@ module.exports = function (Topics) { ], callback); }; - function updateCategoryPostCount(oldTid, tid, callback) { + function updateCategory(postData, toTid, callback) { + var topicData; async.waterfall([ function (next) { - Topics.getTopicsFields([oldTid, tid], ['cid'], next); + Topics.getTopicsFields([postData.tid, toTid], ['cid', 'pinned'], next); }, - function (topicData, next) { + function (_topicData, next) { + topicData = _topicData; if (!topicData[0].cid || !topicData[1].cid) { return callback(); } + var tasks = []; + if (parseInt(topicData[0].pinned, 10) !== 1) { + tasks.push(async.apply(db.sortedSetIncrBy, 'cid:' + topicData[0].cid + ':tids:posts', -1, postData.tid)); + } + if (parseInt(topicData[1].pinned, 10) !== 1) { + tasks.push(async.apply(db.sortedSetIncrBy, 'cid:' + topicData[1].cid + ':tids:posts', 1, toTid)); + } else { + next(); + } + async.series(tasks, function (err) { + next(err); + }); + }, + function (next) { if (parseInt(topicData[0].cid, 10) === parseInt(topicData[1].cid, 10)) { return callback(); } + async.parallel([ async.apply(db.incrObjectFieldBy, 'category:' + topicData[0].cid, 'post_count', -1), async.apply(db.incrObjectFieldBy, 'category:' + topicData[1].cid, 'post_count', 1), + async.apply(db.sortedSetRemove, 'cid:' + topicData[0].cid + ':pids', postData.pid), + async.apply(db.sortedSetAdd, 'cid:' + topicData[1].cid + ':pids', postData.timestamp, postData.pid), ], next); }, ], callback); } - - function updateRecentTopic(tid, callback) { - async.waterfall([ - function (next) { - Topics.getLatestUndeletedPid(tid, next); - }, - function (pid, next) { - if (!pid) { - return callback(); - } - posts.getPostField(pid, 'timestamp', next); - }, - function (timestamp, next) { - Topics.updateTimestamp(tid, timestamp, next); - }, - ], callback); - } }; diff --git a/src/topics/posts.js b/src/topics/posts.js index a3bc448d75..7e1140e5fa 100644 --- a/src/topics/posts.js +++ b/src/topics/posts.js @@ -16,10 +16,7 @@ module.exports = function (Topics) { Topics.onNewPostMade = function (postData, callback) { async.series([ function (next) { - Topics.increasePostCount(postData.tid, next); - }, - function (next) { - Topics.updateTimestamp(postData.tid, postData.timestamp, next); + Topics.updateLastPostTime(postData.tid, postData.timestamp, next); }, function (next) { Topics.addPostToTopic(postData.tid, postData, next); @@ -287,6 +284,9 @@ module.exports = function (Topics) { }); } }, + function (next) { + Topics.increasePostCount(tid, next); + }, function (next) { db.sortedSetIncrBy('tid:' + tid + ':posters', 1, postData.uid, next); }, @@ -304,6 +304,9 @@ module.exports = function (Topics) { 'tid:' + tid + ':posts:votes', ], postData.pid, next); }, + function (next) { + Topics.decreasePostCount(tid, next); + }, function (next) { db.sortedSetIncrBy('tid:' + tid + ':posters', -1, postData.uid, next); }, diff --git a/src/topics/recent.js b/src/topics/recent.js index 1556e081bc..eb964d259a 100644 --- a/src/topics/recent.js +++ b/src/topics/recent.js @@ -3,12 +3,14 @@ 'use strict'; var async = require('async'); +var winston = require('winston'); var db = require('../database'); var plugins = require('../plugins'); var privileges = require('../privileges'); var user = require('../user'); var meta = require('../meta'); +var posts = require('../posts'); module.exports = function (Topics) { var terms = { @@ -26,6 +28,7 @@ module.exports = function (Topics) { }; params.term = params.term || 'alltime'; + params.sort = params.sort || 'recent'; if (params.hasOwnProperty('cids') && params.cids && !Array.isArray(params.cids)) { params.cids = [params.cids]; } @@ -226,34 +229,58 @@ module.exports = function (Topics) { db.getSortedSetRevRangeByScore(set, start, count, '+inf', Date.now() - since, callback); }; - Topics.updateTimestamp = function (tid, timestamp, callback) { - async.parallel([ + Topics.updateLastPostTimeFromLastPid = function (tid, callback) { + async.waterfall([ function (next) { - var topicData; - async.waterfall([ - function (next) { - Topics.getTopicFields(tid, ['cid', 'deleted'], next); - }, - function (_topicData, next) { - topicData = _topicData; - db.sortedSetAdd('cid:' + topicData.cid + ':tids:lastposttime', timestamp, tid, next); - }, - function (next) { - if (parseInt(topicData.deleted, 10) === 1) { - return next(); - } - Topics.updateRecent(tid, timestamp, next); - }, - ], next); + Topics.getLatestUndeletedPid(tid, next); + }, + function (pid, next) { + if (!parseInt(pid, 10)) { + return callback(); + } + posts.getPostField(pid, 'timestamp', next); + }, + function (timestamp, next) { + if (!parseInt(timestamp, 10)) { + return callback(); + } + Topics.updateLastPostTime(tid, timestamp, next); + }, + ], callback); + }; + + Topics.updateLastPostTime = function (tid, lastposttime, callback) { + async.waterfall([ + function (next) { + Topics.setTopicField(tid, 'lastposttime', lastposttime, next); }, function (next) { - Topics.setTopicField(tid, 'lastposttime', timestamp, next); + Topics.getTopicFields(tid, ['cid', 'deleted', 'pinned'], next); + }, + function (topicData, next) { + var tasks = [ + async.apply(db.sortedSetAdd, 'cid:' + topicData.cid + ':tids:lastposttime', lastposttime, tid), + ]; + + if (parseInt(topicData.deleted, 10) !== 1) { + tasks.push(async.apply(Topics.updateRecent, tid, lastposttime)); + } + + if (parseInt(topicData.pinned, 10) !== 1) { + tasks.push(async.apply(db.sortedSetAdd, 'cid:' + topicData.cid + ':tids', lastposttime, tid)); + } + async.series(tasks, next); }, ], function (err) { callback(err); }); }; + Topics.updateTimestamp = function (tid, lastposttime, callback) { + winston.warn('[deprecated] Topics.updateTimestamp is deprecated please use Topics.updateLastPostTime'); + Topics.updateLastPostTime(tid, lastposttime, callback); + }; + Topics.updateRecent = function (tid, timestamp, callback) { callback = callback || function () {}; diff --git a/src/topics/teaser.js b/src/topics/teaser.js index 508bd51bc8..2a636b0702 100644 --- a/src/topics/teaser.js +++ b/src/topics/teaser.js @@ -5,6 +5,7 @@ var async = require('async'); var _ = require('lodash'); var winston = require('winston'); +var db = require('../database'); var meta = require('../meta'); var user = require('../user'); var posts = require('../posts'); @@ -57,9 +58,14 @@ module.exports = function (Topics) { function (next) { posts.getPostsFields(teaserPids, ['pid', 'uid', 'timestamp', 'tid', 'content'], next); }, - async.apply(user.blocks.filter, uid), function (_postData, next) { - postData = _postData; + _postData = _postData.filter(function (post) { + return post && parseInt(post.pid, 10); + }); + handleBlocks(uid, _postData, next); + }, + function (_postData, next) { + postData = _postData.filter(Boolean); var uids = _.uniq(postData.map(function (post) { return post.uid; })); @@ -72,7 +78,6 @@ module.exports = function (Topics) { users[user.uid] = user; }); - async.each(postData, function (post, next) { // If the post author isn't represented in the retrieved users' data, then it means they were deleted, assume guest. if (!users.hasOwnProperty(post.uid)) { @@ -107,6 +112,58 @@ module.exports = function (Topics) { ], callback); }; + function handleBlocks(uid, teasers, callback) { + user.blocks.list(uid, function (err, blockedUids) { + if (err || !blockedUids.length) { + return callback(err, teasers); + } + async.mapSeries(teasers, function (postData, nextPost) { + if (blockedUids.includes(parseInt(postData.uid, 10))) { + getPreviousNonBlockedPost(postData, blockedUids, nextPost); + } else { + setImmediate(nextPost, null, postData); + } + }, callback); + }); + } + + function getPreviousNonBlockedPost(postData, blockedUids, callback) { + let isBlocked = false; + let prevPost = postData; + const postsPerIteration = 5; + let start = 0; + let stop = start + postsPerIteration - 1; + + async.doWhilst(function (next) { + async.waterfall([ + function (next) { + db.getSortedSetRevRange('tid:' + postData.tid + ':posts', start, stop, next); + }, + function (pids, next) { + if (!pids.length) { + return callback(null, null); + } + + posts.getPostsFields(pids, ['pid', 'uid', 'timestamp', 'tid', 'content'], next); + }, + function (prevPosts, next) { + isBlocked = prevPosts.every(function (post) { + const isPostBlocked = blockedUids.includes(parseInt(post.uid, 10)); + prevPost = !isPostBlocked ? post : prevPost; + return isPostBlocked; + }); + start += postsPerIteration; + stop = start + postsPerIteration - 1; + next(); + }, + ], next); + }, function () { + return isBlocked && prevPost && prevPost.pid; + }, function (err) { + callback(err, prevPost); + }); + } + Topics.getTeasersByTids = function (tids, uid, callback) { if (typeof uid === 'function') { winston.warn('[Topics.getTeasersByTids] this usage is deprecated please provide uid'); diff --git a/src/topics/thumb.js b/src/topics/thumb.js index cfd13f3fb8..17d2806810 100644 --- a/src/topics/thumb.js +++ b/src/topics/thumb.js @@ -49,7 +49,6 @@ module.exports = function (Topics) { var size = parseInt(meta.config.topicThumbSize, 10) || 120; image.resizeImage({ path: pathToUpload, - extension: path.extname(pathToUpload), width: size, height: size, }, next); diff --git a/src/topics/unread.js b/src/topics/unread.js index 86d3cac56a..c8b3552af5 100644 --- a/src/topics/unread.js +++ b/src/topics/unread.js @@ -6,6 +6,7 @@ var _ = require('lodash'); var db = require('../database'); var user = require('../user'); +var posts = require('../posts'); var notifications = require('../notifications'); var categories = require('../categories'); var privileges = require('../privileges'); @@ -19,8 +20,8 @@ module.exports = function (Topics) { callback = filter; filter = ''; } - Topics.getUnreadTids({ cid: 0, uid: uid, filter: filter }, function (err, tids) { - callback(err, Array.isArray(tids) ? tids.length : 0); + Topics.getUnreadTids({ cid: 0, uid: uid, count: true }, function (err, counts) { + callback(err, counts && counts[filter]); }); }; @@ -65,10 +66,18 @@ module.exports = function (Topics) { Topics.getUnreadTids = function (params, callback) { var uid = parseInt(params.uid, 10); - if (uid === 0) { - return callback(null, []); + var counts = { + '': 0, + new: 0, + watched: 0, + unreplied: 0, + }; + if (uid <= 0) { + return callback(null, params.count ? counts : []); } + params.filter = params.filter || ''; + var cutoff = params.cutoff || Topics.unreadCutoff(); if (params.cid && !Array.isArray(params.cid)) { @@ -94,106 +103,210 @@ module.exports = function (Topics) { }, function (results, next) { if (results.recentTids && !results.recentTids.length && !results.tids_unread.length) { - return callback(null, []); + return callback(null, params.count ? counts : []); } - var userRead = {}; - results.userScores.forEach(function (userItem) { - userRead[userItem.value] = userItem.score; - }); - - results.recentTids = results.recentTids.concat(results.tids_unread); - results.recentTids.sort(function (a, b) { - return b.score - a.score; - }); - - var tids = results.recentTids.filter(function (recentTopic) { - if (results.ignoredTids.indexOf(recentTopic.value.toString()) !== -1) { - return false; - } - switch (params.filter) { - case 'new': - return !userRead[recentTopic.value]; - default: - return !userRead[recentTopic.value] || recentTopic.score > userRead[recentTopic.value]; - } - }).map(function (topic) { - return topic.value; - }); - - tids = _.uniq(tids); - - if (params.filter === 'watched') { - Topics.filterWatchedTids(tids, uid, next); - } else if (params.filter === 'unreplied') { - Topics.filterUnrepliedTids(tids, next); - } else { - next(null, tids); - } + filterTopics(params, results, next); }, - function (tids, next) { - tids = tids.slice(0, 200); - - filterTopics(uid, tids, params.cid, params.filter, next); - }, - function (tids, next) { + function (data, next) { plugins.fireHook('filter:topics.getUnreadTids', { uid: uid, - tids: tids, + tids: data.tids, + counts: data.counts, + tidsByFilter: data.tidsByFilter, cid: params.cid, filter: params.filter, }, next); }, function (results, next) { - next(null, results.tids); + next(null, params.count ? results.counts : results.tids); }, ], callback); }; + function filterTopics(params, results, callback) { + const counts = { + '': 0, + new: 0, + watched: 0, + unreplied: 0, + }; + + const tidsByFilter = { + '': [], + new: [], + watched: [], + unreplied: [], + }; + + var userRead = {}; + results.userScores.forEach(function (userItem) { + userRead[userItem.value] = userItem.score; + }); + + results.recentTids = results.recentTids.concat(results.tids_unread); + results.recentTids.sort(function (a, b) { + return b.score - a.score; + }); + + var tids = results.recentTids.filter(function (recentTopic) { + if (results.ignoredTids.includes(String(recentTopic.value))) { + return false; + } + return !userRead[recentTopic.value] || recentTopic.score > userRead[recentTopic.value]; + }); + + tids = _.uniq(tids.map(topic => topic.value)); + + var cid = params.cid; + var uid = params.uid; + var cids; + var topicData; + var blockedUids; + + tids = tids.slice(0, 200); - function filterTopics(uid, tids, cid, filter, callback) { if (!tids.length) { - return callback(null, tids); + return callback(null, { counts: counts, tids: tids }); } async.waterfall([ function (next) { - privileges.topics.filterTids('read', tids, uid, next); + user.blocks.list(uid, next); }, - function (tids, next) { + function (_blockedUids, next) { + blockedUids = _blockedUids; + filterTidsThatHaveBlockedPosts({ + uid: uid, + tids: tids, + blockedUids: blockedUids, + recentTids: results.recentTids, + }, next); + }, + function (_tids, next) { + tids = _tids; + Topics.getTopicsFields(tids, ['tid', 'cid', 'uid', 'postcount'], next); + }, + function (_topicData, next) { + topicData = _topicData; + cids = _.uniq(topicData.map(topic => topic.cid)).filter(Boolean); + async.parallel({ - topics: function (next) { - Topics.getTopicsFields(tids, ['tid', 'cid'], next); - }, isTopicsFollowed: function (next) { - if (filter === 'watched' || filter === 'new') { - return next(null, []); - } db.sortedSetScores('uid:' + uid + ':followed_tids', tids, next); }, ignoredCids: function (next) { - if (filter === 'watched') { - return next(null, []); - } user.getIgnoredCategories(uid, next); }, + readableCids: function (next) { + privileges.categories.filterCids('read', cids, uid, next); + }, }, next); }, function (results, next) { - var topics = results.topics; cid = cid && cid.map(String); - tids = topics.filter(function (topic, index) { - return topic && topic.cid && - (!!results.isTopicsFollowed[index] || results.ignoredCids.indexOf(topic.cid.toString()) === -1) && - (!cid || (cid.length && cid.indexOf(String(topic.cid)) !== -1)); - }).map(function (topic) { - return topic.tid; + results.readableCids = results.readableCids.map(String); + + topicData.forEach(function (topic, index) { + function cidMatch(topicCid) { + return (!cid || (cid.length && cid.includes(String(topicCid)))) && results.readableCids.includes(String(topicCid)); + } + + if (topic && topic.cid && cidMatch(topic.cid) && !blockedUids.includes(parseInt(topic.uid, 10))) { + if ((results.isTopicsFollowed[index] || !results.ignoredCids.includes(String(topic.cid)))) { + counts[''] += 1; + tidsByFilter[''].push(topic.tid); + } + + if (results.isTopicsFollowed[index]) { + counts.watched += 1; + tidsByFilter.watched.push(topic.tid); + } + + if (parseInt(topic.postcount, 10) <= 1) { + counts.unreplied += 1; + tidsByFilter.unreplied.push(topic.tid); + } + + if (!userRead[topic.tid]) { + counts.new += 1; + tidsByFilter.new.push(topic.tid); + } + } + }); + + next(null, { + counts: counts, + tids: tidsByFilter[params.filter], + tidsByFilter: tidsByFilter, }); - next(null, tids); }, ], callback); } + function filterTidsThatHaveBlockedPosts(params, callback) { + if (!params.blockedUids.length) { + return setImmediate(callback, null, params.tids); + } + const topicScores = _.mapValues(_.keyBy(params.recentTids, 'value'), 'score'); + + db.sortedSetScores('uid:' + params.uid + ':tids_read', params.tids, function (err, results) { + if (err) { + return callback(err); + } + const userScores = _.zipObject(params.tids, results); + + async.filter(params.tids, function (tid, next) { + doesTidHaveUnblockedUnreadPosts(tid, { + blockedUids: params.blockedUids, + topicTimestamp: topicScores[tid], + userLastReadTimestamp: userScores[tid], + }, next); + }, callback); + }); + } + + function doesTidHaveUnblockedUnreadPosts(tid, params, callback) { + var userLastReadTimestamp = params.userLastReadTimestamp; + if (!userLastReadTimestamp) { + return setImmediate(callback, null, true); + } + var start = 0; + var count = 3; + var done = false; + var hasUnblockedUnread = params.topicTimestamp > userLastReadTimestamp; + + async.whilst(function () { + return !done; + }, function (_next) { + async.waterfall([ + function (next) { + db.getSortedSetRangeByScore('tid:' + tid + ':posts', start, count, userLastReadTimestamp, '+inf', next); + }, + function (pidsSinceLastVisit, next) { + if (!pidsSinceLastVisit.length) { + done = true; + return _next(); + } + + posts.getPostsFields(pidsSinceLastVisit, ['pid', 'uid'], next); + }, + function (postData, next) { + postData = postData.filter(function (post) { + return !params.blockedUids.includes(parseInt(post.uid, 10)); + }); + + done = postData.length > 0; + hasUnblockedUnread = postData.length > 0; + start += count; + next(); + }, + ], _next); + }, function (err) { + callback(err, hasUnblockedUnread); + }); + } + Topics.pushUnreadCount = function (uid, callback) { callback = callback || function () {}; @@ -203,14 +316,15 @@ module.exports = function (Topics) { async.waterfall([ function (next) { - async.parallel({ - unreadTopicCount: async.apply(Topics.getTotalUnread, uid), - unreadNewTopicCount: async.apply(Topics.getTotalUnread, uid, 'new'), - unreadWatchedTopicCount: async.apply(Topics.getTotalUnread, uid, 'watched'), - }, next); + Topics.getUnreadTids({ uid: uid, count: true }, next); }, function (results, next) { - require('../socket.io').in('uid_' + uid).emit('event:unread.updateCount', results); + require('../socket.io').in('uid_' + uid).emit('event:unread.updateCount', { + unreadTopicCount: results[''], + unreadNewTopicCount: results.new, + unreadWatchedTopicCount: results.watched, + unreadUnrepliedTopicCount: results.unreplied, + }); setImmediate(next); }, ], callback); @@ -333,7 +447,7 @@ module.exports = function (Topics) { async.waterfall([ function (next) { async.parallel({ - recentScores: function (next) { + topicScores: function (next) { db.sortedSetScores('topics:recent', tids, next); }, userScores: function (next) { @@ -342,17 +456,38 @@ module.exports = function (Topics) { tids_unread: function (next) { db.sortedSetScores('uid:' + uid + ':tids_unread', tids, next); }, + blockedUids: function (next) { + user.blocks.list(uid, next); + }, }, next); }, function (results, next) { var cutoff = Topics.unreadCutoff(); var result = tids.map(function (tid, index) { - return !results.tids_unread[index] && - (results.recentScores[index] < cutoff || - !!(results.userScores[index] && results.userScores[index] >= results.recentScores[index])); + var read = !results.tids_unread[index] && + (results.topicScores[index] < cutoff || + !!(results.userScores[index] && results.userScores[index] >= results.topicScores[index])); + return { tid: tid, read: read, index: index }; }); - next(null, result); + async.map(result, function (data, next) { + if (data.read) { + return next(null, true); + } + doesTidHaveUnblockedUnreadPosts(data.tid, { + topicTimestamp: results.topicScores[data.index], + userLastReadTimestamp: results.userScores[data.index], + blockedUids: results.blockedUids, + }, function (err, hasUnblockedUnread) { + if (err) { + return next(err); + } + if (!hasUnblockedUnread) { + data.read = true; + } + next(null, data.read); + }); + }, next); }, ], callback); }; diff --git a/src/upgrades/1.10.0/hash_recent_ip_addresses.js b/src/upgrades/1.10.0/hash_recent_ip_addresses.js index 9e13db0252..a2d7d1a99e 100644 --- a/src/upgrades/1.10.0/hash_recent_ip_addresses.js +++ b/src/upgrades/1.10.0/hash_recent_ip_addresses.js @@ -9,7 +9,7 @@ var nconf = require('nconf'); module.exports = { name: 'Hash all IP addresses stored in Recent IPs zset', - timestamp: Date.UTC(2017, 5, 22), + timestamp: Date.UTC(2018, 5, 22), method: function (callback) { const progress = this.progress; var hashed = /[a-f0-9]{32}/; diff --git a/src/upgrades/1.10.2/event_filters.js b/src/upgrades/1.10.2/event_filters.js new file mode 100644 index 0000000000..4f2e87485e --- /dev/null +++ b/src/upgrades/1.10.2/event_filters.js @@ -0,0 +1,46 @@ +'use strict'; + +var db = require('../../database'); + +var async = require('async'); +var batch = require('../../batch'); + +module.exports = { + name: 'add filters to events', + timestamp: Date.UTC(2018, 9, 4), + method: function (callback) { + const progress = this.progress; + + batch.processSortedSet('events:time', function (eids, next) { + async.eachSeries(eids, function (eid, next) { + progress.incr(); + + db.getObject('event:' + eid, function (err, eventData) { + if (err) { + return next(err); + } + if (!eventData) { + return db.sortedSetRemove('events:time', eid, next); + } + // privilege events we're missing type field + if (!eventData.type && eventData.privilege) { + eventData.type = 'privilege-change'; + async.waterfall([ + function (next) { + db.setObjectField('event:' + eid, 'type', 'privilege-change', next); + }, + function (next) { + db.sortedSetAdd('events:time:' + eventData.type, eventData.timestamp, eid, next); + }, + ], next); + return; + } + + db.sortedSetAdd('events:time:' + (eventData.type || ''), eventData.timestamp, eid, next); + }); + }, next); + }, { + progress: this.progress, + }, callback); + }, +}; diff --git a/src/upgrades/1.10.2/fix_category_post_zsets.js b/src/upgrades/1.10.2/fix_category_post_zsets.js new file mode 100644 index 0000000000..216026f753 --- /dev/null +++ b/src/upgrades/1.10.2/fix_category_post_zsets.js @@ -0,0 +1,60 @@ +'use strict'; + +var db = require('../../database'); + +var async = require('async'); +var batch = require('../../batch'); + +module.exports = { + name: 'Fix category post zsets', + timestamp: Date.UTC(2018, 9, 10), + method: function (callback) { + const progress = this.progress; + + db.getSortedSetRange('categories:cid', 0, -1, function (err, cids) { + if (err) { + return callback(err); + } + var keys = cids.map(function (cid) { + return 'cid:' + cid + ':pids'; + }); + var posts = require('../../posts'); + batch.processSortedSet('posts:pid', function (postData, next) { + async.eachSeries(postData, function (postData, next) { + progress.incr(); + var pid = postData.value; + var timestamp = postData.score; + var cid; + async.waterfall([ + function (next) { + posts.getCidByPid(pid, next); + }, + function (_cid, next) { + cid = _cid; + db.isMemberOfSortedSets(keys, pid, next); + }, + function (isMembers, next) { + var memberCids = []; + isMembers.forEach(function (isMember, index) { + if (isMember) { + memberCids.push(cids[index]); + } + }); + if (memberCids.length > 1) { + async.waterfall([ + async.apply(db.sortedSetRemove, memberCids.map(cid => 'cid:' + cid + ':pids'), pid), + async.apply(db.sortedSetAdd, 'cid:' + cid + ':pids', timestamp, pid), + ], next); + } else { + next(); + } + }, + ], next); + }, next); + }, { + progress: progress, + withScores: true, + }, callback); + }); + }, +}; diff --git a/src/upgrades/1.10.2/fix_category_topic_zsets.js b/src/upgrades/1.10.2/fix_category_topic_zsets.js new file mode 100644 index 0000000000..11b7b1a9da --- /dev/null +++ b/src/upgrades/1.10.2/fix_category_topic_zsets.js @@ -0,0 +1,39 @@ +'use strict'; + +var db = require('../../database'); + +var async = require('async'); +var batch = require('../../batch'); + +module.exports = { + name: 'Fix category topic zsets', + timestamp: Date.UTC(2018, 9, 11), + method: function (callback) { + const progress = this.progress; + + var topics = require('../../topics'); + batch.processSortedSet('topics:tid', function (tids, next) { + async.eachSeries(tids, function (tid, next) { + progress.incr(); + + async.waterfall([ + function (next) { + db.getObjectFields('topic:' + tid, ['cid', 'pinned', 'postcount'], next); + }, + function (topicData, next) { + if (parseInt(topicData.pinned, 10) === 1) { + return setImmediate(next); + } + + db.sortedSetAdd('cid:' + topicData.cid + ':tids:posts', topicData.postcount, tid, next); + }, + function (next) { + topics.updateLastPostTimeFromLastPid(tid, next); + }, + ], next); + }, next); + }, { + progress: progress, + }, callback); + }, +}; diff --git a/src/upgrades/1.10.2/local_login_privileges.js b/src/upgrades/1.10.2/local_login_privileges.js new file mode 100644 index 0000000000..6b1e5cc090 --- /dev/null +++ b/src/upgrades/1.10.2/local_login_privileges.js @@ -0,0 +1,17 @@ +'use strict'; + +module.exports = { + name: 'Give global local login privileges', + timestamp: Date.UTC(2018, 8, 28), + method: function (callback) { + var meta = require('../../meta'); + var privileges = require('../../privileges'); + var allowLocalLogin = parseInt(meta.config.allowLocalLogin, 10) !== 0; + + if (allowLocalLogin) { + privileges.global.give(['local:login'], 'registered-users', callback); + } else { + callback(); + } + }, +}; diff --git a/src/upgrades/1.10.2/postgres_sessions.js b/src/upgrades/1.10.2/postgres_sessions.js new file mode 100644 index 0000000000..b77a3f9b92 --- /dev/null +++ b/src/upgrades/1.10.2/postgres_sessions.js @@ -0,0 +1,41 @@ +'use strict'; + +var db = require('../../database'); +var nconf = require('nconf'); + +module.exports = { + name: 'Optimize PostgreSQL sessions', + timestamp: Date.UTC(2018, 9, 1), + method: function (callback) { + if (nconf.get('database') !== 'postgres' || nconf.get('redis')) { + return callback(); + } + + db.pool.query(` +BEGIN TRANSACTION; + +CREATE TABLE IF NOT EXISTS "session" ( + "sid" CHAR(32) NOT NULL + COLLATE "C" + PRIMARY KEY, + "sess" JSONB NOT NULL, + "expire" TIMESTAMPTZ NOT NULL +) WITHOUT OIDS; + +CREATE INDEX IF NOT EXISTS "session_expire_idx" ON "session"("expire"); + +ALTER TABLE "session" + ALTER "sid" TYPE CHAR(32) COLLATE "C", + ALTER "sid" SET STORAGE PLAIN, + ALTER "sess" TYPE JSONB, + ALTER "expire" TYPE TIMESTAMPTZ, + CLUSTER ON "session_expire_idx"; + +CLUSTER "session"; +ANALYZE "session"; + +COMMIT;`, function (err) { + callback(err); + }); + }, +}; diff --git a/src/upgrades/1.10.2/upgrade_bans_to_hashes.js b/src/upgrades/1.10.2/upgrade_bans_to_hashes.js new file mode 100644 index 0000000000..bce9b7ce43 --- /dev/null +++ b/src/upgrades/1.10.2/upgrade_bans_to_hashes.js @@ -0,0 +1,83 @@ +'use strict'; + +var db = require('../../database'); + +var async = require('async'); +var batch = require('../../batch'); +// var user = require('../../user'); + +module.exports = { + name: 'Upgrade bans to hashes', + timestamp: Date.UTC(2018, 8, 24), + method: function (callback) { + const progress = this.progress; + + batch.processSortedSet('users:joindate', function (uids, next) { + async.eachSeries(uids, function (uid, next) { + progress.incr(); + + async.parallel({ + bans: function (next) { + db.getSortedSetRevRangeWithScores('uid:' + uid + ':bans', 0, -1, next); + }, + reasons: function (next) { + db.getSortedSetRevRangeWithScores('banned:' + uid + ':reasons', 0, -1, next); + }, + userData: function (next) { + db.getObjectFields('user:' + uid, ['banned', 'banned:expire', 'joindate', 'lastposttime', 'lastonline'], next); + }, + }, function (err, results) { + function addBan(key, data, callback) { + async.waterfall([ + function (next) { + db.setObject(key, data, next); + }, + function (next) { + db.sortedSetAdd('uid:' + uid + ':bans:timestamp', data.timestamp, key, next); + }, + ], callback); + } + if (err) { + return next(err); + } + // has no ban history and isn't banned, skip + if (!results.bans.length && !parseInt(results.userData.banned, 10)) { + return next(); + } + + // has no history, but is banned, create plain object with just uid and timestmap + if (!results.bans.length && parseInt(results.userData.banned, 10)) { + const banTimestamp = results.userData.lastonline || results.userData.lastposttime || results.userData.joindate || Date.now(); + const banKey = 'uid:' + uid + ':ban:' + banTimestamp; + addBan(banKey, { uid: uid, timestamp: banTimestamp }, next); + return; + } + + // process ban history + async.eachSeries(results.bans, function (ban, next) { + function findReason(score) { + return results.reasons.find(function (reasonData) { + return reasonData.score === score; + }); + } + const reasonData = findReason(ban.score); + const banKey = 'uid:' + uid + ':ban:' + ban.score; + var data = { + uid: uid, + timestamp: ban.score, + expire: parseInt(ban.value, 10), + }; + if (reasonData) { + data.reason = reasonData.value; + } + addBan(banKey, data, next); + }, function (err) { + next(err); + }); + }); + }, next); + }, { + progress: this.progress, + }, callback); + }, +}; diff --git a/src/upgrades/1.10.2/username_email_history.js b/src/upgrades/1.10.2/username_email_history.js new file mode 100644 index 0000000000..1aea342b4b --- /dev/null +++ b/src/upgrades/1.10.2/username_email_history.js @@ -0,0 +1,69 @@ +'use strict'; + +var db = require('../../database'); + +var async = require('async'); +var batch = require('../../batch'); +var user = require('../../user'); + +module.exports = { + name: 'Record first entry in username/email history', + timestamp: Date.UTC(2018, 7, 28), + method: function (callback) { + const progress = this.progress; + + batch.processSortedSet('users:joindate', function (ids, next) { + async.each(ids, function (uid, next) { + async.parallel([ + function (next) { + // Username + async.waterfall([ + async.apply(db.sortedSetCard, 'user:' + uid + ':usernames'), + (count, next) => { + if (count > 0) { + // User has changed their username before, no record of original username, skip. + return setImmediate(next, null, null); + } + + user.getUserFields(uid, ['username', 'joindate'], next); + }, + (userdata, next) => { + if (!userdata) { + return setImmediate(next); + } + + db.sortedSetAdd('user:' + uid + ':usernames', userdata.joindate, [userdata.username, userdata.joindate].join(':'), next); + }, + ], next); + }, + function (next) { + // Email + async.waterfall([ + async.apply(db.sortedSetCard, 'user:' + uid + ':emails'), + (count, next) => { + if (count > 0) { + // User has changed their email before, no record of original email, skip. + return setImmediate(next, null, null); + } + + user.getUserFields(uid, ['email', 'joindate'], next); + }, + (userdata, next) => { + if (!userdata) { + return setImmediate(next); + } + + db.sortedSetAdd('user:' + uid + ':emails', userdata.joindate, [userdata.email, userdata.joindate].join(':'), next); + }, + ], next); + }, + ], function (err) { + progress.incr(); + setImmediate(next, err); + }); + }, next); + }, { + progress: this.progress, + }, callback); + }, +}; diff --git a/src/upgrades/1.4.6/delete_sessions.js b/src/upgrades/1.4.6/delete_sessions.js index c899126bde..28db5948b5 100644 --- a/src/upgrades/1.4.6/delete_sessions.js +++ b/src/upgrades/1.4.6/delete_sessions.js @@ -45,10 +45,12 @@ module.exports = { ], function (err) { next(err); }); - } else { + } else if (db.client && db.client.collection) { db.client.collection('sessions').deleteMany({}, {}, function (err) { next(err); }); + } else { + next(); } }, ], callback); diff --git a/src/upgrades/1.6.0/generate-email-logo.js b/src/upgrades/1.6.0/generate-email-logo.js index 6115e773a1..5832fb9739 100644 --- a/src/upgrades/1.6.0/generate-email-logo.js +++ b/src/upgrades/1.6.0/generate-email-logo.js @@ -34,7 +34,6 @@ module.exports = { image.resizeImage({ path: sourcePath, target: uploadPath, - extension: 'png', height: 50, }, next); }); diff --git a/src/upgrades/1.7.3/key_value_schema_change.js b/src/upgrades/1.7.3/key_value_schema_change.js index a8abefb10a..882f8ff200 100644 --- a/src/upgrades/1.7.3/key_value_schema_change.js +++ b/src/upgrades/1.7.3/key_value_schema_change.js @@ -23,7 +23,7 @@ module.exports = { var cursor; async.waterfall([ function (next) { - client.collection('objects').count({ + client.collection('objects').countDocuments({ _key: { $exists: true }, value: { $exists: true }, score: { $exists: false }, @@ -55,7 +55,7 @@ module.exports = { } delete item.expireAt; if (Object.keys(item).length === 3 && item.hasOwnProperty('_key') && item.hasOwnProperty('value')) { - client.collection('objects').update({ _key: item._key }, { $rename: { value: 'data' } }, next); + client.collection('objects').updateOne({ _key: item._key }, { $rename: { value: 'data' } }, next); } else { next(); } diff --git a/src/upgrades/1.7.6/update_min_pass_strength.js b/src/upgrades/1.7.6/update_min_pass_strength.js index c051d1d72e..e12ed5d067 100644 --- a/src/upgrades/1.7.6/update_min_pass_strength.js +++ b/src/upgrades/1.7.6/update_min_pass_strength.js @@ -6,7 +6,7 @@ var async = require('async'); module.exports = { name: 'Revising minimum password strength to 1 (from 0)', - timestamp: Date.UTC(2017, 1, 21), + timestamp: Date.UTC(2018, 1, 21), method: function (callback) { async.waterfall([ async.apply(db.getObjectField.bind(db), 'config', 'minimumPasswordStrength'), diff --git a/src/upgrades/1.8.1/diffs_zset_to_listhash.js b/src/upgrades/1.8.1/diffs_zset_to_listhash.js index b7a2bba296..d5e065c698 100644 --- a/src/upgrades/1.8.1/diffs_zset_to_listhash.js +++ b/src/upgrades/1.8.1/diffs_zset_to_listhash.js @@ -7,7 +7,7 @@ var async = require('async'); module.exports = { name: 'Reformatting post diffs to be stored in lists and hash instead of single zset', - timestamp: Date.UTC(2017, 2, 15), + timestamp: Date.UTC(2018, 2, 15), method: function (callback) { var progress = this.progress; diff --git a/src/user.js b/src/user.js index 3b005d5572..72a0d1d74c 100644 --- a/src/user.js +++ b/src/user.js @@ -435,3 +435,4 @@ User.addInterstitials = function (callback) { callback(); }; +User.async = require('./promisify')(User); diff --git a/src/user/bans.js b/src/user/bans.js index e2cf2193b3..9868c10776 100644 --- a/src/user/bans.js +++ b/src/user/bans.js @@ -24,25 +24,31 @@ module.exports = function (User) { return callback(new Error('[[error:ban-expiry-missing]]')); } + const banKey = 'uid:' + uid + ':ban:' + now; + var banData = { + uid: uid, + timestamp: now, + expire: until > now ? until : 0, + }; + if (reason) { + banData.reason = reason; + } var tasks = [ async.apply(User.setUserField, uid, 'banned', 1), async.apply(db.sortedSetAdd, 'users:banned', now, uid), - async.apply(db.sortedSetAdd, 'uid:' + uid + ':bans', now, until), + async.apply(db.sortedSetAdd, 'uid:' + uid + ':bans:timestamp', now, banKey), + async.apply(db.setObject, banKey, banData), ]; - if (until > 0 && now < until) { + if (until > now) { tasks.push(async.apply(db.sortedSetAdd, 'users:banned:expire', until, uid)); tasks.push(async.apply(User.setUserField, uid, 'banned:expire', until)); } else { until = 0; } - if (reason) { - tasks.push(async.apply(db.sortedSetAdd, 'banned:' + uid + ':reasons', now, reason)); - } - async.series(tasks, function (err) { - callback(err); + callback(err, banData); }); }; @@ -86,10 +92,16 @@ module.exports = function (User) { User.getBannedReason = function (uid, callback) { async.waterfall([ function (next) { - db.getSortedSetRevRange('banned:' + uid + ':reasons', 0, 0, next); + db.getSortedSetRevRange('uid:' + uid + ':bans:timestamp', 0, 0, next); }, - function (reasons, next) { - next(null, reasons.length ? reasons[0] : ''); + function (keys, next) { + if (!keys.length) { + return callback(null, ''); + } + db.getObject(keys[0], next); + }, + function (banObj, next) { + next(null, banObj && banObj.reason ? banObj.reason : ''); }, ], callback); }; diff --git a/src/user/blocks.js b/src/user/blocks.js index a3021968e8..51bb4520f1 100644 --- a/src/user/blocks.js +++ b/src/user/blocks.js @@ -48,8 +48,8 @@ module.exports = function (User) { }; User.blocks.list = function (uid, callback) { - if (User.blocks._cache.has(uid)) { - return setImmediate(callback, null, User.blocks._cache.get(uid)); + if (User.blocks._cache.has(parseInt(uid, 10))) { + return setImmediate(callback, null, User.blocks._cache.get(parseInt(uid, 10))); } db.getSortedSetRange('uid:' + uid + ':blocked_uids', 0, -1, function (err, blocked) { @@ -73,8 +73,8 @@ module.exports = function (User) { async.apply(db.sortedSetAdd.bind(db), 'uid:' + uid + ':blocked_uids', Date.now(), targetUid), async.apply(User.incrementUserFieldBy, uid, 'blocksCount', 1), function (_blank, next) { - User.blocks._cache.del(uid); - pubsub.publish('user:blocks:cache:del', uid); + User.blocks._cache.del(parseInt(uid, 10)); + pubsub.publish('user:blocks:cache:del', parseInt(uid, 10)); setImmediate(next); }, ], callback); @@ -86,8 +86,8 @@ module.exports = function (User) { async.apply(db.sortedSetRemove.bind(db), 'uid:' + uid + ':blocked_uids', targetUid), async.apply(User.decrementUserFieldBy, uid, 'blocksCount', 1), function (_blank, next) { - User.blocks._cache.del(uid); - pubsub.publish('user:blocks:cache:del', uid); + User.blocks._cache.del(parseInt(uid, 10)); + pubsub.publish('user:blocks:cache:del', parseInt(uid, 10)); setImmediate(next); }, ], callback); diff --git a/src/user/create.js b/src/user/create.js index b0eba41c6b..a82c050409 100644 --- a/src/user/create.js +++ b/src/user/create.js @@ -93,6 +93,9 @@ module.exports = function (User) { function (next) { db.sortedSetsAdd(['users:postcount', 'users:reputation'], 0, userData.uid, next); }, + function (next) { + db.sortedSetAdd('user:' + userData.uid + ':usernames', timestamp, userData.username, next); + }, function (next) { groups.join('registered-users', userData.uid, next); }, @@ -104,6 +107,7 @@ module.exports = function (User) { async.parallel([ async.apply(db.sortedSetAdd, 'email:uid', userData.uid, userData.email.toLowerCase()), async.apply(db.sortedSetAdd, 'email:sorted', 0, userData.email.toLowerCase() + ':' + userData.uid), + async.apply(db.sortedSetAdd, 'user:' + userData.uid + ':emails', timestamp, userData.email), ], next); if (parseInt(userData.uid, 10) !== 1 && parseInt(meta.config.requireEmailConfirmation, 10) === 1) { diff --git a/src/user/data.js b/src/user/data.js index 05ebc2f9b2..dc851dc8c4 100644 --- a/src/user/data.js +++ b/src/user/data.js @@ -119,6 +119,11 @@ module.exports = function (User) { return memo; }, {}); var users = uids.map(function (uid) { + const returnPayload = usersData[ref[uid]]; + if (uid > 0 && !returnPayload.uid) { + returnPayload.oldUid = parseInt(uid, 10); + } + return usersData[ref[uid]]; }); return users; @@ -144,7 +149,7 @@ module.exports = function (User) { if (!parseInt(user.uid, 10)) { user.uid = 0; - user.username = '[[global:guest]]'; + user.username = (user.hasOwnProperty('oldUid') && parseInt(user.oldUid, 10)) ? '[[global:former_user]]' : '[[global:guest]]'; user.userslug = ''; user.picture = User.getDefaultAvatar(); user['icon:text'] = '?'; diff --git a/src/user/delete.js b/src/user/delete.js index 6712569634..f84ee25393 100644 --- a/src/user/delete.js +++ b/src/user/delete.js @@ -15,12 +15,20 @@ var batch = require('../batch'); var file = require('../file'); module.exports = function (User) { + var deletesInProgress = {}; + User.delete = function (callerUid, uid, callback) { if (!parseInt(uid, 10)) { - return callback(new Error('[[error:invalid-uid]]')); + return setImmediate(callback, new Error('[[error:invalid-uid]]')); } - + if (deletesInProgress[uid]) { + return setImmediate(callback, new Error('[[error:already-deleting]]')); + } + deletesInProgress[uid] = 'user.delete'; async.waterfall([ + function (next) { + removeFromSortedSets(uid, next); + }, function (next) { deletePosts(callerUid, uid, next); }, @@ -67,15 +75,39 @@ module.exports = function (User) { }, { alwaysStartAt: 0 }, callback); } + function removeFromSortedSets(uid, callback) { + db.sortedSetsRemove([ + 'users:joindate', + 'users:postcount', + 'users:reputation', + 'users:banned', + 'users:banned:expire', + 'users:flags', + 'users:online', + 'users:notvalidated', + 'digest:day:uids', + 'digest:week:uids', + 'digest:month:uids', + ], uid, callback); + } + User.deleteAccount = function (uid, callback) { + if (deletesInProgress[uid] === 'user.deleteAccount') { + return setImmediate(callback, new Error('[[error:already-deleting]]')); + } + deletesInProgress[uid] = 'user.deleteAccount'; var userData; async.waterfall([ + function (next) { + removeFromSortedSets(uid, next); + }, function (next) { db.getObject('user:' + uid, next); }, function (_userData, next) { if (!_userData || !_userData.username) { - return callback(); + delete deletesInProgress[uid]; + return callback(new Error('[[error:no-user]]')); } userData = _userData; plugins.fireHook('static:user.delete', { uid: uid }, next); @@ -113,19 +145,6 @@ module.exports = function (User) { next(); } }, - function (next) { - db.sortedSetsRemove([ - 'users:joindate', - 'users:postcount', - 'users:reputation', - 'users:banned', - 'users:online', - 'users:notvalidated', - 'digest:day:uids', - 'digest:week:uids', - 'digest:month:uids', - ], uid, next); - }, function (next) { db.decrObjectField('global', 'userCount', next); }, @@ -149,6 +168,9 @@ module.exports = function (User) { function (next) { deleteUserIps(uid, next); }, + function (next) { + deleteBans(uid, next); + }, function (next) { deleteUserFromFollowers(uid, next); }, @@ -160,7 +182,10 @@ module.exports = function (User) { function (results, next) { db.deleteAll(['followers:' + uid, 'following:' + uid, 'user:' + uid], next); }, - ], callback); + ], function (err) { + delete deletesInProgress[uid]; + callback(err, userData); + }); }; function deleteVotes(uid, callback) { @@ -220,6 +245,20 @@ module.exports = function (User) { ], callback); } + function deleteBans(uid, callback) { + async.waterfall([ + function (next) { + db.getSortedSetRange('uid:' + uid + ':bans:timestamp', 0, -1, next); + }, + function (bans, next) { + db.deleteAll(bans, next); + }, + function (next) { + db.delete('uid:' + uid + ':bans:timestamp', next); + }, + ], callback); + } + function deleteUserFromFollowers(uid, callback) { async.parallel({ followers: async.apply(db.getSortedSetRange, 'followers:' + uid, 0, -1), diff --git a/src/user/email.js b/src/user/email.js index 598536ad29..24559104a3 100644 --- a/src/user/email.js +++ b/src/user/email.js @@ -100,14 +100,16 @@ UserEmail.sendValidationEmail = function (uid, options, callback) { }, function (username, next) { var title = meta.config.title || meta.config.browserTitle || 'NodeBB'; - translator.translate('[[email:welcome-to, ' + title + ']]', meta.config.defaultLang, function (subject) { + var subject = options.subject || '[[email:welcome-to, ' + title + ']]'; + var template = options.template || 'welcome'; + translator.translate(subject, meta.config.defaultLang, function (subject) { var data = { username: username, confirm_link: confirm_link, confirm_code: confirm_code, subject: subject, - template: 'welcome', + template: template, uid: uid, }; @@ -115,7 +117,7 @@ UserEmail.sendValidationEmail = function (uid, options, callback) { plugins.fireHook('action:user.verify', { uid: uid, data: data }); next(); } else { - emailer.send('welcome', uid, data, next); + emailer.send(template, uid, data, next); } }); }, diff --git a/src/user/info.js b/src/user/info.js index 5e91c6cf08..aabdac3534 100644 --- a/src/user/info.js +++ b/src/user/info.js @@ -5,6 +5,7 @@ var _ = require('lodash'); var validator = require('validator'); var db = require('../database'); +var user = require('../user'); var posts = require('../posts'); var topics = require('../topics'); var utils = require('../../public/src/utils'); @@ -12,30 +13,25 @@ var utils = require('../../public/src/utils'); module.exports = function (User) { User.getLatestBanInfo = function (uid, callback) { // Simply retrieves the last record of the user's ban, even if they've been unbanned since then. - var timestamp; - var expiry; - var reason; - async.waterfall([ - async.apply(db.getSortedSetRevRangeWithScores, 'uid:' + uid + ':bans', 0, 0), + function (next) { + db.getSortedSetRevRange('uid:' + uid + ':bans:timestamp', 0, 0, next); + }, function (record, next) { if (!record.length) { return next(new Error('no-ban-info')); } - - timestamp = record[0].score; - expiry = record[0].value; - - db.getSortedSetRangeByScore('banned:' + uid + ':reasons', 0, -1, timestamp, timestamp, next); + db.getObject(record[0], next); }, - function (_reason, next) { - reason = _reason && _reason.length ? _reason[0] : ''; + function (banInfo, next) { + var expiry = banInfo.expire; + next(null, { uid: uid, - timestamp: timestamp, + timestamp: banInfo.timestamp, expiry: parseInt(expiry, 10), expiry_readable: new Date(parseInt(expiry, 10)).toString(), - reason: validator.escape(String(reason)), + reason: validator.escape(String(banInfo.reason || '')), }); }, ], callback); @@ -46,8 +42,7 @@ module.exports = function (User) { function (next) { async.parallel({ flags: async.apply(db.getSortedSetRevRangeWithScores, 'flags:byTargetUid:' + uid, 0, 19), - bans: async.apply(db.getSortedSetRevRangeWithScores, 'uid:' + uid + ':bans', 0, 19), - reasons: async.apply(db.getSortedSetRevRangeWithScores, 'banned:' + uid + ':reasons', 0, 19), + bans: async.apply(db.getSortedSetRevRange, 'uid:' + uid + ':bans:timestamp', 0, 19), }, next); }, function (data, next) { @@ -76,8 +71,7 @@ module.exports = function (User) { }); }, function (data, next) { - formatBanData(data); - next(null, data); + formatBanData(data, next); }, ], callback); }; @@ -131,26 +125,31 @@ module.exports = function (User) { ], callback); } - function formatBanData(data) { - var reasons = data.reasons.reduce(function (memo, cur) { - memo[cur.score] = cur.value; - return memo; - }, {}); + function formatBanData(data, callback) { + var banData; + async.waterfall([ + function (next) { + db.getObjects(data.bans, next); + }, + function (_banData, next) { + banData = _banData; + var uids = banData.map(banData => banData.fromUid); - data.bans = data.bans.map(function (banObj) { - banObj.until = parseInt(banObj.value, 10); - banObj.untilReadable = new Date(banObj.until).toString(); - banObj.timestamp = parseInt(banObj.score, 10); - banObj.timestampReadable = new Date(banObj.score).toString(); - banObj.timestampISO = new Date(banObj.score).toISOString(); - banObj.reason = validator.escape(String(reasons[banObj.score] || '')) || '[[user:info.banned-no-reason]]'; - - delete banObj.value; - delete banObj.score; - delete data.reasons; - - return banObj; - }); + user.getUsersFields(uids, ['uid', 'username', 'userslug', 'picture'], next); + }, + function (usersData, next) { + data.bans = banData.map(function (banObj, index) { + banObj.user = usersData[index]; + banObj.until = parseInt(banObj.expire, 10); + banObj.untilReadable = new Date(banObj.until).toString(); + banObj.timestampReadable = new Date(banObj.timestamp).toString(); + banObj.timestampISO = utils.toISOString(banObj.timestamp); + banObj.reason = validator.escape(String(banObj.reason || '')) || '[[user:info.banned-no-reason]]'; + return banObj; + }); + next(null, data); + }, + ], callback); } User.getModerationNotes = function (uid, start, stop, callback) { diff --git a/src/user/password.js b/src/user/password.js index d644fd2e8f..bfb65f9cb3 100644 --- a/src/user/password.js +++ b/src/user/password.js @@ -15,7 +15,7 @@ module.exports = function (User) { Password.hash(nconf.get('bcrypt_rounds') || 12, password, callback); }; - User.isPasswordCorrect = function (uid, password, callback) { + User.isPasswordCorrect = function (uid, password, ip, callback) { password = password || ''; var hashedPassword; async.waterfall([ @@ -25,14 +25,22 @@ module.exports = function (User) { function (_hashedPassword, next) { hashedPassword = _hashedPassword; if (!hashedPassword) { - return callback(null, true); + // Non-existant user, submit fake hash for comparison + hashedPassword = ''; } User.isPasswordValid(password, 0, next); }, + async.apply(User.auth.logAttempt, uid, ip), function (next) { Password.compare(password, hashedPassword, next); }, + function (ok, next) { + if (ok) { + User.auth.clearLoginAttempts(uid); + } + next(null, ok); + }, ], callback); }; diff --git a/src/user/picture.js b/src/user/picture.js index d1a4dac7b0..d39bce659d 100644 --- a/src/user/picture.js +++ b/src/user/picture.js @@ -37,7 +37,7 @@ module.exports = function (User) { async.waterfall([ function (next) { - var size = data.file ? data.file.size : data.imageData.length; + var size = data.file ? data.file.size : image.sizeFromBase64(data.imageData); meta.config.maximumCoverImageSize = meta.config.maximumCoverImageSize || 2048; if (size > parseInt(meta.config.maximumCoverImageSize, 10) * 1024) { return next(new Error('[[error:file-too-big, ' + meta.config.maximumCoverImageSize + ']]')); @@ -89,10 +89,10 @@ module.exports = function (User) { return callback(new Error('[[error:invalid-data]]')); } - var size = data.file ? data.file.size : data.imageData.length; + var size = data.file ? data.file.size : image.sizeFromBase64(data.imageData); var uploadSize = parseInt(meta.config.maximumProfileImageSize, 10) || 256; if (size > uploadSize * 1024) { - return callback(new Error('[[error:file-too-big, ' + meta.config.maximumProfileImageSize + ']]')); + return callback(new Error('[[error:file-too-big, ' + uploadSize + ']]')); } var type = data.file ? data.file.type : image.mimeFromBase64(data.imageData); @@ -123,11 +123,9 @@ module.exports = function (User) { }, function (path, next) { picture.path = path; - var imageDimension = parseInt(meta.config.profileImageDimension, 10) || 200; image.resizeImage({ path: picture.path, - extension: extension, width: imageDimension, height: imageDimension, }, next); diff --git a/src/user/profile.js b/src/user/profile.js index 5da6eb7409..6f2d3252b5 100644 --- a/src/user/profile.js +++ b/src/user/profile.js @@ -45,8 +45,6 @@ module.exports = function (User) { return updateUsername(updateUid, data.username, next); } else if (field === 'fullname') { return updateFullname(updateUid, data.fullname, next); - } else if (field === 'signature') { - data[field] = utils.stripHTMLTags(data[field]); } User.setUserField(updateUid, field, data[field], next); @@ -170,7 +168,7 @@ module.exports = function (User) { User.checkMinReputation = function (callerUid, uid, setting, callback) { var isSelf = parseInt(callerUid, 10) === parseInt(uid, 10); - if (!isSelf) { + if (!isSelf || parseInt(meta.config['reputation:disabled'], 10) === 1) { return setImmediate(callback); } async.waterfall([ @@ -223,6 +221,8 @@ module.exports = function (User) { if (parseInt(meta.config.requireEmailConfirmation, 10) === 1 && newEmail) { User.email.sendValidationEmail(uid, { email: newEmail, + subject: '[[email:email.verify-your-email.subject]]', + template: 'verify_email', }); } User.setUserField(uid, 'email:confirmed', 0, next); @@ -322,12 +322,12 @@ module.exports = function (User) { if (parseInt(uid, 10) !== parseInt(data.uid, 10)) { User.isAdministrator(uid, next); } else { - User.isPasswordCorrect(uid, data.currentPassword, next); + User.isPasswordCorrect(uid, data.currentPassword, data.ip, next); } }, function (isAdminOrPasswordMatch, next) { if (!isAdminOrPasswordMatch) { - return next(new Error('[[error:change_password_error_wrong_current]]')); + return next(new Error('[[user:change_password_error_wrong_current]]')); } User.hashPassword(data.newPassword, next); diff --git a/src/views/admin/advanced/database.tpl b/src/views/admin/advanced/database.tpl index 9519ce9141..53533a9249 100644 --- a/src/views/admin/advanced/database.tpl +++ b/src/views/admin/advanced/database.tpl @@ -57,6 +57,19 @@ + +
    +
    [[admin/advanced/database:postgres]]
    +
    +
    + [[admin/advanced/database:postgres.version]] {postgres.version}
    +
    + [[admin/advanced/database:uptime-seconds]] {postgres.uptime}
    +
    +
    +
    + +
    @@ -84,5 +97,19 @@
    + + +
    +
    +

    [[admin/advanced/database:postgres.raw-info]]

    +
    + +
    +
    + 
    {postgres.raw}
    +
    +
    +
    + diff --git a/src/views/admin/advanced/events.tpl b/src/views/admin/advanced/events.tpl index 65c707920f..4372d12b93 100644 --- a/src/views/admin/advanced/events.tpl +++ b/src/views/admin/advanced/events.tpl @@ -1,8 +1,13 @@
    +
    [[admin/advanced/events:events]]
    -
    +
    [[admin/advanced/events:no-events]]
    diff --git a/src/views/admin/extend/widgets.tpl b/src/views/admin/extend/widgets.tpl index 9d02b2fb81..3d984516c1 100644 --- a/src/views/admin/extend/widgets.tpl +++ b/src/views/admin/extend/widgets.tpl @@ -85,18 +85,18 @@
    [[admin/extend/widgets:containers.none]]
    -
    +
    [[admin/extend/widgets:container.well]]
    -
    +
    [[admin/extend/widgets:container.jumbotron]]
    -
    +
    [[admin/extend/widgets:container.panel]]
    -
    +
    [[admin/extend/widgets:container.panel-header]]
    @@ -113,7 +113,7 @@
    -
    +
    [[admin/extend/widgets:container.alert]]
    diff --git a/src/views/admin/partials/categories/create.tpl b/src/views/admin/partials/categories/create.tpl index 436705b1d8..c1daa78925 100644 --- a/src/views/admin/partials/categories/create.tpl +++ b/src/views/admin/partials/categories/create.tpl @@ -21,5 +21,17 @@ +
    + +
    + +
    + \ No newline at end of file diff --git a/src/views/admin/settings/advanced.tpl b/src/views/admin/settings/advanced.tpl index 44d34fa80f..4aec0e51fa 100644 --- a/src/views/admin/settings/advanced.tpl +++ b/src/views/admin/settings/advanced.tpl @@ -67,6 +67,12 @@
    [[admin/settings/advanced:hsts]]
    +
    + +

    diff --git a/src/views/admin/settings/pagination.tpl b/src/views/admin/settings/pagination.tpl index 5f4d77bde3..6a0496f07d 100644 --- a/src/views/admin/settings/pagination.tpl +++ b/src/views/admin/settings/pagination.tpl @@ -30,7 +30,6 @@ [[admin/settings/pagination:topics-per-page]]

    [[admin/settings/pagination:max-topics-per-page]]

    - [[admin/settings/pagination:initial-num-load]]
    diff --git a/src/views/admin/settings/uploads.tpl b/src/views/admin/settings/uploads.tpl index e523c928c5..d008b0d9d3 100644 --- a/src/views/admin/settings/uploads.tpl +++ b/src/views/admin/settings/uploads.tpl @@ -20,6 +20,14 @@
    +
    + + +

    + [[admin/settings/uploads:private-uploads-extensions-help]] +

    +
    +
    @@ -44,6 +52,22 @@

    +
    + + +

    + [[admin/settings/uploads:reject-image-width-help]] +

    +
    + +
    + + +

    + [[admin/settings/uploads:reject-image-height-help]] +

    +
    +