Nemcio/NodeBB
1
0
Fork 0
mirror of https://github.com/NodeBB/NodeBB.git synced 2026-06-22 13:11:34 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit '6f504c4142d7584cdfe9347cd5cf183c655e2159' into v1.13.x

Browse Source
This commit is contained in:
Misty (Bot)
2020-05-21 23:45:47 +00:00
parent 7fee153438 6f504c4142
commit a6ab71f373
17 changed files with 212 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
Gruntfile.js
View File

@@ -179,23 +179,22 @@ module.exports = function (grunt) {
};
function addBaseThemes(plugins) {
const themeId = plugins.find(p => p.startsWith('nodebb-theme-'));
let themeId = plugins.find(p => p.includes('nodebb-theme-'));
if (!themeId) {
return plugins;
}
function getBaseRecursive(themeId) {
let baseTheme;
do {
try {
const baseTheme = require(themeId + '/theme').baseTheme;
if (baseTheme) {
plugins.push(baseTheme);
getBaseRecursive(baseTheme);
}
baseTheme = require(themeId + '/theme').baseTheme;
} catch (err) {
console.log(err);
}
}
getBaseRecursive(themeId);
if (baseTheme) {
plugins.push(baseTheme);
themeId = baseTheme;
}
} while (baseTheme);
return plugins;
}

3
install/data/defaults.json
View File

@@ -32,9 +32,8 @@
"registrationType": "normal",
"registrationApprovalType": "normal",
"allowAccountDelete": 1,
"allowFileUploads": 0,
"privateUploads": 0,
"allowedFileExtensions": "png,jpg,bmp",
"allowedFileExtensions": "png,jpg,bmp,txt",
"allowUserHomePage": 1,
"allowMultipleBadges": 0,
"maximumFileSize": 2048,

4
install/package.json
View File

@@ -79,7 +79,7 @@
"mousetrap": "^1.6.5",
"@nodebb/mubsub": "^1.6.0",
"nconf": "^0.10.0",
"nodebb-plugin-composer-default": "6.3.30",
"nodebb-plugin-composer-default": "6.3.31",
"nodebb-plugin-dbsearch": "4.0.7",
"nodebb-plugin-emoji": "^3.3.0",
"nodebb-plugin-emoji-android": "2.0.0",
@@ -143,7 +143,7 @@
"grunt-contrib-watch": "1.1.0",
"husky": "4.2.5",
"jsdom": "16.2.2",
"lint-staged": "10.2.0",
"lint-staged": "10.2.4",
"mocha": "7.1.2",
"mocha-lcov-reporter": "1.3.0",
"nyc": "15.0.1",

2
public/openapi/read.yaml
View File

@@ -2795,8 +2795,6 @@ paths:
type: boolean
allowGuestHandles:
type: boolean
allowFileUploads:
type: boolean
allowTopicsThumbnail:
type: boolean
usePagination:

2
public/src/app.js
View File

@@ -596,7 +596,7 @@ app.cacheBuster = null;
$(window).trigger('action:search.quick', { data: data });
search.api(data, function (data) {
data.posts.forEach(function (p) {
p.snippet = utils.escapeHTML($(p.content).text().slice(0, 80) + '...');
p.snippet = utils.escapeHTML($('<div>' + p.content + '</div>').text().slice(0, 80) + '...');
});
app.parseAndTranslate(template, data, function (html) {
if (html.length) {

1
src/controllers/api.js
View File

@@ -34,7 +34,6 @@ apiController.loadConfig = async function (req) {
useOutgoingLinksPage: meta.config.useOutgoingLinksPage === 1,
outgoingLinksWhitelist: meta.config.useOutgoingLinksPage === 1 ? meta.config['outgoingLinks:whitelist'] : undefined,
allowGuestHandles: meta.config.allowGuestHandles === 1,
allowFileUploads: meta.config.allowFileUploads === 1,
allowTopicsThumbnail: meta.config.allowTopicsThumbnail === 1,
usePagination: meta.config.usePagination === 1,
disableChat: meta.config.disableChat === 1,

4
src/controllers/uploads.js
View File

@@ -84,10 +84,6 @@ async function uploadAsFile(req, uploadedFile) {
throw new Error('[[error:no-privileges]]');
}
if (!meta.config.allowFileUploads) {
throw new Error('[[error:uploads-are-disabled]]');
}
const fileObj = await uploadsController.uploadFile(req.uid, uploadedFile);
return {
url: fileObj.url,

177
src/database/mongo/sorted/intersect.js
View File

@@ -5,18 +5,54 @@ module.exports = function (module) {
if (!Array.isArray(keys) || !keys.length) {
return 0;
}
const objects = module.client.collection('objects');
const counts = await countSets(keys, 50000);
if (counts.minCount === 0) {
return 0;
}
let items = await objects.find({ _key: counts.smallestSet }, {
projection: { _id: 0, value: 1 },
}).toArray();
var pipeline = [
{ $match: { _key: { $in: keys } } },
{ $group: { _id: { value: '$value' }, count: { $sum: 1 } } },
{ $match: { count: keys.length } },
{ $group: { _id: null, count: { $sum: 1 } } },
];
const data = await module.client.collection('objects').aggregate(pipeline).toArray();
return Array.isArray(data) && data.length ? data[0].count : 0;
items = items.map(i => i.value);
const otherSets = keys.filter(s => s !== counts.smallestSet);
if (otherSets.length === 1) {
return await objects.countDocuments({
_key: otherSets[0], value: { $in: items },
});
}
items = await intersectValuesWithSets(items, otherSets);
return items.length;
};
async function intersectValuesWithSets(items, sets) {
for (let i = 0; i < sets.length; i++) {
/* eslint-disable no-await-in-loop */
items = await module.client.collection('objects').find({
_key: sets[i], value: { $in: items },
}, {
projection: { _id: 0, value: 1 },
}).toArray();
items = items.map(i => i.value);
}
return items;
}
async function countSets(sets, limit) {
const objects = module.client.collection('objects');
const counts = await Promise.all(
sets.map(s => objects.countDocuments({ _key: s }, {
limit: limit || 25000,
}))
);
const minCount = Math.min(...counts);
const index = counts.indexOf(minCount);
const smallestSet = sets[index];
return {
minCount: minCount,
smallestSet: smallestSet,
};
}
module.getSortedSetIntersect = async function (params) {
params.sort = 1;
@@ -29,10 +65,100 @@ module.exports = function (module) {
};
async function getSortedSetRevIntersect(params) {
var sets = params.sets;
var start = params.hasOwnProperty('start') ? params.start : 0;
var stop = params.hasOwnProperty('stop') ? params.stop : -1;
var weights = params.weights || [];
params.start = params.hasOwnProperty('start') ? params.start : 0;
params.stop = params.hasOwnProperty('stop') ? params.stop : -1;
params.weights = params.weights || [];
params.limit = params.stop - params.start + 1;
if (params.limit <= 0) {
params.limit = 0;
}
params.counts = await countSets(params.sets);
if (params.counts.minCount === 0) {
return [];
}
const simple = params.weights.filter(w => w === 1).length === 1 && params.limit !== 0;
if (params.counts.minCount < 25000 && simple) {
return await intersectSingle(params);
} else if (simple) {
return await intersectBatch(params);
}
return await intersectAggregate(params);
}
async function intersectSingle(params) {
const objects = module.client.collection('objects');
let items = await objects.find({ _key: params.counts.smallestSet }, {
projection: { _id: 0, value: 1 },
}).toArray();
if (!items.length) {
return [];
}
const otherSets = params.sets.filter(s => s !== params.counts.smallestSet);
items = await intersectValuesWithSets(items.map(i => i.value), otherSets);
if (!items.length) {
return [];
}
const project = { _id: 0, value: 1 };
if (params.withScores) {
project.score = 1;
}
const sortSet = params.sets[params.weights.indexOf(1)];
let res = await objects
.find({ _key: sortSet, value: { $in: items } }, { projection: project })
.sort({ score: params.sort })
.skip(params.start)
.limit(params.limit)
.toArray();
if (!params.withScores) {
res = res.map(i => i.value);
}
return res;
}
async function intersectBatch(params) {
const project = { _id: 0, value: 1 };
if (params.withScores) {
project.score = 1;
}
const sortSet = params.sets[params.weights.indexOf(1)];
const batchSize = 10000;
const cursor = await module.client.collection('objects')
.find({ _key: sortSet }, { projection: project })
.sort({ score: params.sort })
.batchSize(batchSize);
const otherSets = params.sets.filter(s => s !== sortSet);
let inters = [];
let done = false;
while (!done) {
/* eslint-disable no-await-in-loop */
const items = [];
while (items.length < batchSize) {
const nextItem = await cursor.next();
if (!nextItem) {
done = true;
break;
}
items.push(nextItem);
}
const members = await Promise.all(otherSets.map(s => module.isSortedSetMembers(s, items.map(i => i.value))));
inters = inters.concat(items.filter((item, idx) => members.every(arr => arr[idx])));
if (inters.length >= params.stop) {
done = true;
inters = inters.slice(params.start, params.stop + 1);
}
}
if (!params.withScores) {
inters = inters.map(item => item.value);
}
return inters;
}
async function intersectAggregate(params) {
var aggregate = {};
if (params.aggregate) {
@@ -40,15 +166,9 @@ module.exports = function (module) {
} else {
aggregate.$sum = '$score';
}
const pipeline = [{ $match: { _key: { $in: params.sets } } }];
var limit = stop - start + 1;
if (limit <= 0) {
limit = 0;
}
var pipeline = [{ $match: { _key: { $in: sets } } }];
weights.forEach(function (weight, index) {
params.weights.forEach(function (weight, index) {
if (weight !== 1) {
pipeline.push({
$project: {
@@ -56,7 +176,7 @@ module.exports = function (module) {
score: {
$cond: {
if: {
$eq: ['$_key', sets[index]],
$eq: ['$_key', params.sets[index]],
},
then: {
$multiply: ['$score', weight],
@@ -70,25 +190,24 @@ module.exports = function (module) {
});
pipeline.push({ $group: { _id: { value: '$value' }, totalScore: aggregate, count: { $sum: 1 } } });
pipeline.push({ $match: { count: sets.length } });
pipeline.push({ $match: { count: params.sets.length } });
pipeline.push({ $sort: { totalScore: params.sort } });
if (start) {
pipeline.push({ $skip: start });
if (params.start) {
pipeline.push({ $skip: params.start });
}
if (limit > 0) {
pipeline.push({ $limit: limit });
if (params.limit > 0) {
pipeline.push({ $limit: params.limit });
}
var project = { _id: 0, value: '$_id.value' };
const project = { _id: 0, value: '$_id.value' };
if (params.withScores) {
project.score = '$totalScore';
}
pipeline.push({ $project: project });
let data = await module.client.collection('objects').aggregate(pipeline).toArray();
if (!params.withScores) {
data = data.map(item => item.value);
}

42
src/upgrade.js
View File

@@ -2,6 +2,7 @@
var async = require('async');
var path = require('path');
var util = require('util');
var semver = require('semver');
var readline = require('readline');
var winston = require('winston');
@@ -140,7 +141,7 @@ Upgrade.process = function (files, skipCount, callback) {
}, next);
},
function (results, next) {
async.eachSeries(files, function (file, next) {
async.eachSeries(files, async (file) => {
var scriptExport = require(file);
var date = new Date(scriptExport.timestamp);
var version = path.dirname(file).split('/').pop();
@@ -152,35 +153,34 @@ Upgrade.process = function (files, skipCount, callback) {
date: date,
};
console.log(' → '.white + String('[' + [date.getUTCFullYear(), date.getUTCMonth() + 1, date.getUTCDate()].join('/') + '] ').gray + String(scriptExport.name).reset + '...');
process.stdout.write(' → '.white + String('[' + [date.getUTCFullYear(), date.getUTCMonth() + 1, date.getUTCDate()].join('/') + '] ').gray + String(scriptExport.name).reset + '...');
// For backwards compatibility, cross-reference with schemaDate (if found). If a script's date is older, skip it
if ((!results.schemaDate && !results.schemaLogCount) || (scriptExport.timestamp <= results.schemaDate && semver.lt(version, '1.5.0'))) {
readline.clearLine(process.stdout, 0);
readline.cursorTo(process.stdout, 0);
readline.moveCursor(process.stdout, 0, -1);
console.log(' → '.white + String('[' + [date.getUTCFullYear(), date.getUTCMonth() + 1, date.getUTCDate()].join('/') + '] ').gray + String(scriptExport.name).reset + '... ' + 'skipped'.grey);
db.sortedSetAdd('schemaLog', Date.now(), path.basename(file, '.js'), next);
process.stdout.write(' skipped\n'.grey);
await db.sortedSetAdd('schemaLog', Date.now(), path.basename(file, '.js'));
return;
}
// Promisify method if necessary
if (scriptExport.method.constructor && scriptExport.method.constructor.name !== 'AsyncFunction') {
scriptExport.method = util.promisify(scriptExport.method);
}
// Do the upgrade...
scriptExport.method.bind({
progress: progress,
})(function (err) {
if (err) {
console.error('Error occurred');
return next(err);
}
try {
await scriptExport.method.bind({
progress: progress,
})();
} catch (err) {
console.error('Error occurred');
throw err;
}
readline.clearLine(process.stdout, 0);
readline.cursorTo(process.stdout, 0);
readline.moveCursor(process.stdout, 0, -1);
console.log(' → '.white + String('[' + [date.getUTCFullYear(), date.getUTCMonth() + 1, date.getUTCDate()].join('/') + '] ').gray + String(scriptExport.name).reset + '... ' + 'OK'.green);
process.stdout.write(' OK\n'.green);
// Record success in schemaLog
db.sortedSetAdd('schemaLog', Date.now(), path.basename(file, '.js'), next);
});
// Record success in schemaLog
await db.sortedSetAdd('schemaLog', Date.now(), path.basename(file, '.js'));
}, next);
},
function (next) {

3
src/upgrades/1.13.0/clean_flag_byCid.js
View File

@@ -6,7 +6,7 @@ const batch = require('../../batch');
module.exports = {
name: 'Clean flag byCid zsets',
timestamp: Date.UTC(2019, 8, 24),
method: async function (callback) {
method: async function () {
const progress = this.progress;
await batch.processSortedSet('flags:datetime', async function (flagIds) {
@@ -23,6 +23,5 @@ module.exports = {
}, {
progress: progress,
});
callback();
},
};

3
src/upgrades/1.13.0/clean_post_topic_hash.js
View File

@@ -6,11 +6,10 @@ const batch = require('../../batch');
module.exports = {
name: 'Clean up post hash data',
timestamp: Date.UTC(2019, 9, 7),
method: async function (callback) {
method: async function () {
const progress = this.progress;
await cleanPost(progress);
await cleanTopic(progress);
callback();
},
};

3
src/upgrades/1.13.0/cleanup_old_notifications.js
View File

@@ -7,7 +7,7 @@ const user = require('../../user');
module.exports = {
name: 'Clean up old notifications and hash data',
timestamp: Date.UTC(2019, 9, 7),
method: async function (callback) {
method: async function () {
const progress = this.progress;
const week = 604800000;
const cutoffTime = Date.now() - week;
@@ -47,6 +47,5 @@ module.exports = {
batch: 500,
progress: progress,
});
callback();
},
};

3
src/upgrades/1.13.3/fix_users_sorted_sets.js
View File

@@ -6,7 +6,7 @@ const batch = require('../../batch');
module.exports = {
name: 'Fix user sorted sets',
timestamp: Date.UTC(2020, 4, 2),
method: async function (callback) {
method: async function () {
const progress = this.progress;
const nextUid = await db.getObjectField('global', 'nextUid');
const allUids = [];
@@ -58,6 +58,5 @@ module.exports = {
});
await db.setObjectField('global', 'userCount', totalUserCount);
callback();
},
};

22
src/upgrades/1.13.4/remove_allowFileUploads_priv.js Normal file
View File

@@ -0,0 +1,22 @@
'use strict';
const db = require('../../database');
const privileges = require('../../privileges');
module.exports = {
name: 'Removing file upload privilege if file uploads were disabled (`allowFileUploads`)',
timestamp: Date.UTC(2020, 4, 21),
method: async () => {
const allowFileUploads = parseInt(await db.getObjectField('config', 'allowFileUploads'), 10);
if (allowFileUploads === 1) {
await db.deleteObjectField('config', 'allowFileUploads');
return;
}
// Remove `upload:post:file` privilege for all groups
await privileges.categories.rescind(['upload:post:file'], 0, ['guests', 'registered-users', 'Global Moderators']);
// Clean up the old option from the config hash
await db.deleteObjectField('config', 'allowFileUploads');
},
};

10
src/upgrades/TEMPLATE
View File

@@ -1,17 +1,15 @@
'use strict';
var db = require('../../database');
var async = require('async');
var winston = require('winston');
const db = require('../../database');
const winston = require('winston');
module.exports = {
// you should use spaces
// the underscores are there so you can double click to select the whole thing
name: 'User_friendly_upgrade_script_name',
// remember, month is zero-indexed (so January is 0, December is 11)
timestamp: Date.UTC(2019, 0, 1),
method: function (callback) {
timestamp: Date.UTC(2020, 0, 1),
method: async () => {
// Do stuff here...
},
};

7
src/views/admin/settings/uploads.tpl
View File

@@ -6,13 +6,6 @@
</div>
<div class="col-sm-10 col-xs-12">
<form>
<div class="checkbox">
<label class="mdl-switch mdl-js-switch mdl-js-ripple-effect">
<input class="mdl-switch__input" type="checkbox" data-field="allowFileUploads">
<span class="mdl-switch__label"><strong>[[admin/settings/uploads:allow-files]]</strong></span>
</label>
</div>
<div class="checkbox">
<label class="mdl-switch mdl-js-switch mdl-js-ripple-effect">
<input class="mdl-switch__input" type="checkbox" data-field="privateUploads">

1
test/uploads.js
View File

@@ -131,7 +131,6 @@ describe('Upload Controllers', function () {
it('should upload a file to a post', function (done) {
meta.config.allowFileUploads = 1;
var oldValue = meta.config.allowedFileExtensions;
meta.config.allowedFileExtensions = 'png,jpg,bmp,html';
helpers.uploadFile(nconf.get('url') + '/api/post/upload', path.join(__dirname, '../test/files/503.html'), {}, jar, csrf_token, function (err, res, body) {