diff --git a/Gruntfile.js b/Gruntfile.js index b0a6abd548..4ed9949c6e 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -179,23 +179,22 @@ module.exports = function (grunt) { }; function addBaseThemes(plugins) { - const themeId = plugins.find(p => p.startsWith('nodebb-theme-')); + let themeId = plugins.find(p => p.includes('nodebb-theme-')); if (!themeId) { return plugins; } - function getBaseRecursive(themeId) { + let baseTheme; + do { try { - const baseTheme = require(themeId + '/theme').baseTheme; - - if (baseTheme) { - plugins.push(baseTheme); - getBaseRecursive(baseTheme); - } + baseTheme = require(themeId + '/theme').baseTheme; } catch (err) { console.log(err); } - } - getBaseRecursive(themeId); + if (baseTheme) { + plugins.push(baseTheme); + themeId = baseTheme; + } + } while (baseTheme); return plugins; } diff --git a/install/data/defaults.json b/install/data/defaults.json index 797347f20f..090a1d4026 100644 --- a/install/data/defaults.json +++ b/install/data/defaults.json @@ -32,9 +32,8 @@ "registrationType": "normal", "registrationApprovalType": "normal", "allowAccountDelete": 1, - "allowFileUploads": 0, "privateUploads": 0, - "allowedFileExtensions": "png,jpg,bmp", + "allowedFileExtensions": "png,jpg,bmp,txt", "allowUserHomePage": 1, "allowMultipleBadges": 0, "maximumFileSize": 2048, diff --git a/install/package.json b/install/package.json index 5fceee99fa..05599281ca 100644 --- a/install/package.json +++ b/install/package.json @@ -79,7 +79,7 @@ "mousetrap": "^1.6.5", "@nodebb/mubsub": "^1.6.0", "nconf": "^0.10.0", - "nodebb-plugin-composer-default": "6.3.30", + "nodebb-plugin-composer-default": "6.3.31", "nodebb-plugin-dbsearch": "4.0.7", "nodebb-plugin-emoji": "^3.3.0", "nodebb-plugin-emoji-android": "2.0.0", @@ -143,7 +143,7 @@ "grunt-contrib-watch": "1.1.0", "husky": "4.2.5", "jsdom": "16.2.2", - "lint-staged": "10.2.0", + "lint-staged": "10.2.4", "mocha": "7.1.2", "mocha-lcov-reporter": "1.3.0", "nyc": "15.0.1", diff --git a/public/openapi/read.yaml b/public/openapi/read.yaml index 30f81ebc3c..7a7d1ce091 100644 --- a/public/openapi/read.yaml +++ b/public/openapi/read.yaml @@ -2795,8 +2795,6 @@ paths: type: boolean allowGuestHandles: type: boolean - allowFileUploads: - type: boolean allowTopicsThumbnail: type: boolean usePagination: diff --git a/public/src/app.js b/public/src/app.js index 67058eb516..8d8568e9cf 100644 --- a/public/src/app.js +++ b/public/src/app.js @@ -596,7 +596,7 @@ app.cacheBuster = null; $(window).trigger('action:search.quick', { data: data }); search.api(data, function (data) { data.posts.forEach(function (p) { - p.snippet = utils.escapeHTML($(p.content).text().slice(0, 80) + '...'); + p.snippet = utils.escapeHTML($('
' + p.content + '
').text().slice(0, 80) + '...'); }); app.parseAndTranslate(template, data, function (html) { if (html.length) { diff --git a/src/controllers/api.js b/src/controllers/api.js index 7b9b404de1..faf4695533 100644 --- a/src/controllers/api.js +++ b/src/controllers/api.js @@ -34,7 +34,6 @@ apiController.loadConfig = async function (req) { useOutgoingLinksPage: meta.config.useOutgoingLinksPage === 1, outgoingLinksWhitelist: meta.config.useOutgoingLinksPage === 1 ? meta.config['outgoingLinks:whitelist'] : undefined, allowGuestHandles: meta.config.allowGuestHandles === 1, - allowFileUploads: meta.config.allowFileUploads === 1, allowTopicsThumbnail: meta.config.allowTopicsThumbnail === 1, usePagination: meta.config.usePagination === 1, disableChat: meta.config.disableChat === 1, diff --git a/src/controllers/uploads.js b/src/controllers/uploads.js index 36ca7959e8..efb93c196e 100644 --- a/src/controllers/uploads.js +++ b/src/controllers/uploads.js @@ -84,10 +84,6 @@ async function uploadAsFile(req, uploadedFile) { throw new Error('[[error:no-privileges]]'); } - if (!meta.config.allowFileUploads) { - throw new Error('[[error:uploads-are-disabled]]'); - } - const fileObj = await uploadsController.uploadFile(req.uid, uploadedFile); return { url: fileObj.url, diff --git a/src/database/mongo/sorted/intersect.js b/src/database/mongo/sorted/intersect.js index 87f537665c..242dfd5c70 100644 --- a/src/database/mongo/sorted/intersect.js +++ b/src/database/mongo/sorted/intersect.js @@ -5,18 +5,54 @@ module.exports = function (module) { if (!Array.isArray(keys) || !keys.length) { return 0; } + const objects = module.client.collection('objects'); + const counts = await countSets(keys, 50000); + if (counts.minCount === 0) { + return 0; + } + let items = await objects.find({ _key: counts.smallestSet }, { + projection: { _id: 0, value: 1 }, + }).toArray(); - var pipeline = [ - { $match: { _key: { $in: keys } } }, - { $group: { _id: { value: '$value' }, count: { $sum: 1 } } }, - { $match: { count: keys.length } }, - { $group: { _id: null, count: { $sum: 1 } } }, - ]; - - const data = await module.client.collection('objects').aggregate(pipeline).toArray(); - return Array.isArray(data) && data.length ? data[0].count : 0; + items = items.map(i => i.value); + const otherSets = keys.filter(s => s !== counts.smallestSet); + if (otherSets.length === 1) { + return await objects.countDocuments({ + _key: otherSets[0], value: { $in: items }, + }); + } + items = await intersectValuesWithSets(items, otherSets); + return items.length; }; + async function intersectValuesWithSets(items, sets) { + for (let i = 0; i < sets.length; i++) { + /* eslint-disable no-await-in-loop */ + items = await module.client.collection('objects').find({ + _key: sets[i], value: { $in: items }, + }, { + projection: { _id: 0, value: 1 }, + }).toArray(); + items = items.map(i => i.value); + } + return items; + } + + async function countSets(sets, limit) { + const objects = module.client.collection('objects'); + const counts = await Promise.all( + sets.map(s => objects.countDocuments({ _key: s }, { + limit: limit || 25000, + })) + ); + const minCount = Math.min(...counts); + const index = counts.indexOf(minCount); + const smallestSet = sets[index]; + return { + minCount: minCount, + smallestSet: smallestSet, + }; + } module.getSortedSetIntersect = async function (params) { params.sort = 1; @@ -29,10 +65,100 @@ module.exports = function (module) { }; async function getSortedSetRevIntersect(params) { - var sets = params.sets; - var start = params.hasOwnProperty('start') ? params.start : 0; - var stop = params.hasOwnProperty('stop') ? params.stop : -1; - var weights = params.weights || []; + params.start = params.hasOwnProperty('start') ? params.start : 0; + params.stop = params.hasOwnProperty('stop') ? params.stop : -1; + params.weights = params.weights || []; + + params.limit = params.stop - params.start + 1; + if (params.limit <= 0) { + params.limit = 0; + } + params.counts = await countSets(params.sets); + if (params.counts.minCount === 0) { + return []; + } + + const simple = params.weights.filter(w => w === 1).length === 1 && params.limit !== 0; + if (params.counts.minCount < 25000 && simple) { + return await intersectSingle(params); + } else if (simple) { + return await intersectBatch(params); + } + return await intersectAggregate(params); + } + + async function intersectSingle(params) { + const objects = module.client.collection('objects'); + let items = await objects.find({ _key: params.counts.smallestSet }, { + projection: { _id: 0, value: 1 }, + }).toArray(); + if (!items.length) { + return []; + } + + const otherSets = params.sets.filter(s => s !== params.counts.smallestSet); + items = await intersectValuesWithSets(items.map(i => i.value), otherSets); + if (!items.length) { + return []; + } + const project = { _id: 0, value: 1 }; + if (params.withScores) { + project.score = 1; + } + const sortSet = params.sets[params.weights.indexOf(1)]; + let res = await objects + .find({ _key: sortSet, value: { $in: items } }, { projection: project }) + .sort({ score: params.sort }) + .skip(params.start) + .limit(params.limit) + .toArray(); + if (!params.withScores) { + res = res.map(i => i.value); + } + return res; + } + + async function intersectBatch(params) { + const project = { _id: 0, value: 1 }; + if (params.withScores) { + project.score = 1; + } + const sortSet = params.sets[params.weights.indexOf(1)]; + const batchSize = 10000; + const cursor = await module.client.collection('objects') + .find({ _key: sortSet }, { projection: project }) + .sort({ score: params.sort }) + .batchSize(batchSize); + + const otherSets = params.sets.filter(s => s !== sortSet); + let inters = []; + let done = false; + while (!done) { + /* eslint-disable no-await-in-loop */ + const items = []; + while (items.length < batchSize) { + const nextItem = await cursor.next(); + if (!nextItem) { + done = true; + break; + } + items.push(nextItem); + } + + const members = await Promise.all(otherSets.map(s => module.isSortedSetMembers(s, items.map(i => i.value)))); + inters = inters.concat(items.filter((item, idx) => members.every(arr => arr[idx]))); + if (inters.length >= params.stop) { + done = true; + inters = inters.slice(params.start, params.stop + 1); + } + } + if (!params.withScores) { + inters = inters.map(item => item.value); + } + return inters; + } + + async function intersectAggregate(params) { var aggregate = {}; if (params.aggregate) { @@ -40,15 +166,9 @@ module.exports = function (module) { } else { aggregate.$sum = '$score'; } + const pipeline = [{ $match: { _key: { $in: params.sets } } }]; - var limit = stop - start + 1; - if (limit <= 0) { - limit = 0; - } - - var pipeline = [{ $match: { _key: { $in: sets } } }]; - - weights.forEach(function (weight, index) { + params.weights.forEach(function (weight, index) { if (weight !== 1) { pipeline.push({ $project: { @@ -56,7 +176,7 @@ module.exports = function (module) { score: { $cond: { if: { - $eq: ['$_key', sets[index]], + $eq: ['$_key', params.sets[index]], }, then: { $multiply: ['$score', weight], @@ -70,25 +190,24 @@ module.exports = function (module) { }); pipeline.push({ $group: { _id: { value: '$value' }, totalScore: aggregate, count: { $sum: 1 } } }); - pipeline.push({ $match: { count: sets.length } }); + pipeline.push({ $match: { count: params.sets.length } }); pipeline.push({ $sort: { totalScore: params.sort } }); - if (start) { - pipeline.push({ $skip: start }); + if (params.start) { + pipeline.push({ $skip: params.start }); } - if (limit > 0) { - pipeline.push({ $limit: limit }); + if (params.limit > 0) { + pipeline.push({ $limit: params.limit }); } - var project = { _id: 0, value: '$_id.value' }; + const project = { _id: 0, value: '$_id.value' }; if (params.withScores) { project.score = '$totalScore'; } pipeline.push({ $project: project }); let data = await module.client.collection('objects').aggregate(pipeline).toArray(); - if (!params.withScores) { data = data.map(item => item.value); } diff --git a/src/upgrade.js b/src/upgrade.js index 73d489bcba..f85ca13f19 100644 --- a/src/upgrade.js +++ b/src/upgrade.js @@ -2,6 +2,7 @@ var async = require('async'); var path = require('path'); +var util = require('util'); var semver = require('semver'); var readline = require('readline'); var winston = require('winston'); @@ -140,7 +141,7 @@ Upgrade.process = function (files, skipCount, callback) { }, next); }, function (results, next) { - async.eachSeries(files, function (file, next) { + async.eachSeries(files, async (file) => { var scriptExport = require(file); var date = new Date(scriptExport.timestamp); var version = path.dirname(file).split('/').pop(); @@ -152,35 +153,34 @@ Upgrade.process = function (files, skipCount, callback) { date: date, }; - console.log(' → '.white + String('[' + [date.getUTCFullYear(), date.getUTCMonth() + 1, date.getUTCDate()].join('/') + '] ').gray + String(scriptExport.name).reset + '...'); + process.stdout.write(' → '.white + String('[' + [date.getUTCFullYear(), date.getUTCMonth() + 1, date.getUTCDate()].join('/') + '] ').gray + String(scriptExport.name).reset + '...'); // For backwards compatibility, cross-reference with schemaDate (if found). If a script's date is older, skip it if ((!results.schemaDate && !results.schemaLogCount) || (scriptExport.timestamp <= results.schemaDate && semver.lt(version, '1.5.0'))) { - readline.clearLine(process.stdout, 0); - readline.cursorTo(process.stdout, 0); - readline.moveCursor(process.stdout, 0, -1); - console.log(' → '.white + String('[' + [date.getUTCFullYear(), date.getUTCMonth() + 1, date.getUTCDate()].join('/') + '] ').gray + String(scriptExport.name).reset + '... ' + 'skipped'.grey); - db.sortedSetAdd('schemaLog', Date.now(), path.basename(file, '.js'), next); + process.stdout.write(' skipped\n'.grey); + await db.sortedSetAdd('schemaLog', Date.now(), path.basename(file, '.js')); return; } + // Promisify method if necessary + if (scriptExport.method.constructor && scriptExport.method.constructor.name !== 'AsyncFunction') { + scriptExport.method = util.promisify(scriptExport.method); + } + // Do the upgrade... - scriptExport.method.bind({ - progress: progress, - })(function (err) { - if (err) { - console.error('Error occurred'); - return next(err); - } + try { + await scriptExport.method.bind({ + progress: progress, + })(); + } catch (err) { + console.error('Error occurred'); + throw err; + } - readline.clearLine(process.stdout, 0); - readline.cursorTo(process.stdout, 0); - readline.moveCursor(process.stdout, 0, -1); - console.log(' → '.white + String('[' + [date.getUTCFullYear(), date.getUTCMonth() + 1, date.getUTCDate()].join('/') + '] ').gray + String(scriptExport.name).reset + '... ' + 'OK'.green); + process.stdout.write(' OK\n'.green); - // Record success in schemaLog - db.sortedSetAdd('schemaLog', Date.now(), path.basename(file, '.js'), next); - }); + // Record success in schemaLog + await db.sortedSetAdd('schemaLog', Date.now(), path.basename(file, '.js')); }, next); }, function (next) { diff --git a/src/upgrades/1.13.0/clean_flag_byCid.js b/src/upgrades/1.13.0/clean_flag_byCid.js index a8ea7188a8..ab350c417e 100644 --- a/src/upgrades/1.13.0/clean_flag_byCid.js +++ b/src/upgrades/1.13.0/clean_flag_byCid.js @@ -6,7 +6,7 @@ const batch = require('../../batch'); module.exports = { name: 'Clean flag byCid zsets', timestamp: Date.UTC(2019, 8, 24), - method: async function (callback) { + method: async function () { const progress = this.progress; await batch.processSortedSet('flags:datetime', async function (flagIds) { @@ -23,6 +23,5 @@ module.exports = { }, { progress: progress, }); - callback(); }, }; diff --git a/src/upgrades/1.13.0/clean_post_topic_hash.js b/src/upgrades/1.13.0/clean_post_topic_hash.js index 3b5c5cef36..8b35b50dac 100644 --- a/src/upgrades/1.13.0/clean_post_topic_hash.js +++ b/src/upgrades/1.13.0/clean_post_topic_hash.js @@ -6,11 +6,10 @@ const batch = require('../../batch'); module.exports = { name: 'Clean up post hash data', timestamp: Date.UTC(2019, 9, 7), - method: async function (callback) { + method: async function () { const progress = this.progress; await cleanPost(progress); await cleanTopic(progress); - callback(); }, }; diff --git a/src/upgrades/1.13.0/cleanup_old_notifications.js b/src/upgrades/1.13.0/cleanup_old_notifications.js index dcc1f28a42..95d446d810 100644 --- a/src/upgrades/1.13.0/cleanup_old_notifications.js +++ b/src/upgrades/1.13.0/cleanup_old_notifications.js @@ -7,7 +7,7 @@ const user = require('../../user'); module.exports = { name: 'Clean up old notifications and hash data', timestamp: Date.UTC(2019, 9, 7), - method: async function (callback) { + method: async function () { const progress = this.progress; const week = 604800000; const cutoffTime = Date.now() - week; @@ -47,6 +47,5 @@ module.exports = { batch: 500, progress: progress, }); - callback(); }, }; diff --git a/src/upgrades/1.13.3/fix_users_sorted_sets.js b/src/upgrades/1.13.3/fix_users_sorted_sets.js index 9afd97915f..9cf90bd855 100644 --- a/src/upgrades/1.13.3/fix_users_sorted_sets.js +++ b/src/upgrades/1.13.3/fix_users_sorted_sets.js @@ -6,7 +6,7 @@ const batch = require('../../batch'); module.exports = { name: 'Fix user sorted sets', timestamp: Date.UTC(2020, 4, 2), - method: async function (callback) { + method: async function () { const progress = this.progress; const nextUid = await db.getObjectField('global', 'nextUid'); const allUids = []; @@ -58,6 +58,5 @@ module.exports = { }); await db.setObjectField('global', 'userCount', totalUserCount); - callback(); }, }; diff --git a/src/upgrades/1.13.4/remove_allowFileUploads_priv.js b/src/upgrades/1.13.4/remove_allowFileUploads_priv.js new file mode 100644 index 0000000000..f75cbd078e --- /dev/null +++ b/src/upgrades/1.13.4/remove_allowFileUploads_priv.js @@ -0,0 +1,22 @@ +'use strict'; + +const db = require('../../database'); +const privileges = require('../../privileges'); + +module.exports = { + name: 'Removing file upload privilege if file uploads were disabled (`allowFileUploads`)', + timestamp: Date.UTC(2020, 4, 21), + method: async () => { + const allowFileUploads = parseInt(await db.getObjectField('config', 'allowFileUploads'), 10); + if (allowFileUploads === 1) { + await db.deleteObjectField('config', 'allowFileUploads'); + return; + } + + // Remove `upload:post:file` privilege for all groups + await privileges.categories.rescind(['upload:post:file'], 0, ['guests', 'registered-users', 'Global Moderators']); + + // Clean up the old option from the config hash + await db.deleteObjectField('config', 'allowFileUploads'); + }, +}; diff --git a/src/upgrades/TEMPLATE b/src/upgrades/TEMPLATE index 60722ab70c..98eb0288a6 100644 --- a/src/upgrades/TEMPLATE +++ b/src/upgrades/TEMPLATE @@ -1,17 +1,15 @@ 'use strict'; -var db = require('../../database'); - -var async = require('async'); -var winston = require('winston'); +const db = require('../../database'); +const winston = require('winston'); module.exports = { // you should use spaces // the underscores are there so you can double click to select the whole thing name: 'User_friendly_upgrade_script_name', // remember, month is zero-indexed (so January is 0, December is 11) - timestamp: Date.UTC(2019, 0, 1), - method: function (callback) { + timestamp: Date.UTC(2020, 0, 1), + method: async () => { // Do stuff here... }, }; diff --git a/src/views/admin/settings/uploads.tpl b/src/views/admin/settings/uploads.tpl index 3d63f999a1..dca5f135f4 100644 --- a/src/views/admin/settings/uploads.tpl +++ b/src/views/admin/settings/uploads.tpl @@ -6,13 +6,6 @@
-
- -
-