From 94b79ce4024f72a3eee2cfa06b05d8f66898149f Mon Sep 17 00:00:00 2001 From: ppenguin Date: Tue, 15 Feb 2022 19:13:43 +0100 Subject: [PATCH 01/11] Allow NodeBB setup with env vars (#9850) * initial try [WIP] * typo; add test start script; initial Dockerfile mod with integrated setup [WIP] * minor fixes * add some winston debug... * typos * fix pass confirm setup * more fixes * fix entrypoint * cleanup * remove echo sensitive setupVal * remove obsolete code and comments * fix linting errors * Merge branch 'additional-fixes' * Merge branch 'pitaj-suggested-fixes' * Merge branch 'pitaj-fixes2' * merge checkSetup functions (env vars and flags) * comment (lint) * remove tab * finalise PR; tested ok locally (setup json overrides env vars) --- Dockerfile | 2 +- src/install.js | 46 +++++++++++++++++++++++++++++++++++++++------- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 80dce7cfb5..8a5b7ae9bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,4 +22,4 @@ ENV NODE_ENV=production \ EXPOSE 4567 -CMD node ./nodebb build ; node ./nodebb start +CMD test -n "${SETUP}" && ./nodebb setup || node ./nodebb build; node ./nodebb start diff --git a/src/install.js b/src/install.js index 3067a05ab5..f9f7912d9f 100644 --- a/src/install.js +++ b/src/install.js @@ -46,22 +46,54 @@ questions.optional = [ }, ]; -function checkSetupFlag() { - let setupVal = install.values; +function checkSetupFlagEnv() { + let setupVal = install.values || {}; + const envConfMap = { + NODEBB_URL: 'url', + NODEBB_PORT: 'port', + NODEBB_ADMIN_USERNAME: 'admin:username', + NODEBB_ADMIN_PASSWORD: 'admin:password', + NODEBB_ADMIN_EMAIL: 'admin:email', + NODEBB_DB: 'database', + NODEBB_DB_HOST: 'host', + NODEBB_DB_PORT: 'port', + NODEBB_DB_USER: 'username', + NODEBB_DB_PASSWORD: 'password', + NODEBB_DB_NAME: 'database', + NODEBB_DB_SSL: 'ssl', + }; + + // Set setup values from env vars (if set) + winston.info('[install/checkSetupFlagEnv] checking env vars for setup info...'); + + Object.entries(process.env).forEach(([evName, evValue]) => { // get setup values from env + if (evName.startsWith('NODEBB_DB_')) { + setupVal[`${process.env.NODEBB_DB}:${envConfMap[evName]}`] = evValue; + } else if (evName.startsWith('NODEBB_')) { + setupVal[envConfMap[evName]] = evValue; + } + }); + + setupVal['admin:password:confirm'] = setupVal['admin:password']; + + // try to get setup values from json, if successful this overwrites all values set by env + // TODO: better behaviour would be to support overrides per value, i.e. in order of priority (generic pattern): + // flag, env, config file, default try { if (nconf.get('setup')) { - setupVal = JSON.parse(nconf.get('setup')); + const setupJSON = JSON.parse(nconf.get('setup')); + setupVal = { ...setupVal, ...setupJSON }; } } catch (err) { - winston.error('Invalid json in nconf.get(\'setup\'), ignoring setup values'); + winston.error('[install/checkSetupFlagEnv] invalid json in nconf.get(\'setup\'), ignoring setup values from json'); } if (setupVal && typeof setupVal === 'object') { if (setupVal['admin:username'] && setupVal['admin:password'] && setupVal['admin:password:confirm'] && setupVal['admin:email']) { install.values = setupVal; } else { - winston.error('Required values are missing for automated setup:'); + winston.error('[install/checkSetupFlagEnv] required values are missing for automated setup:'); if (!setupVal['admin:username']) { winston.error(' admin:username'); } @@ -95,7 +127,7 @@ function checkCIFlag() { if (ciVals.hasOwnProperty('host') && ciVals.hasOwnProperty('port') && ciVals.hasOwnProperty('database')) { install.ciVals = ciVals; } else { - winston.error('Required values are missing for automated CI integration:'); + winston.error('[install/checkCIFlag] required values are missing for automated CI integration:'); if (!ciVals.hasOwnProperty('host')) { winston.error(' host'); } @@ -521,7 +553,7 @@ async function checkUpgrade() { install.setup = async function () { try { - checkSetupFlag(); + checkSetupFlagEnv(); checkCIFlag(); await setupConfig(); await setupDefaultConfigs(); From 55a9818389cc30ba5badf40687b6795149ad3168 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 15 Feb 2022 17:14:15 -0500 Subject: [PATCH 02/11] fix(deps): update dependency nodebb-plugin-mentions to v3.0.5 (#10294) Co-authored-by: Renovate Bot --- install/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/package.json b/install/package.json index 992af9563c..7ccbe69ef7 100644 --- a/install/package.json +++ b/install/package.json @@ -91,7 +91,7 @@ "nodebb-plugin-emoji": "3.5.12", "nodebb-plugin-emoji-android": "2.0.5", "nodebb-plugin-markdown": "9.0.6", - "nodebb-plugin-mentions": "3.0.4", + "nodebb-plugin-mentions": "3.0.5", "nodebb-plugin-spam-be-gone": "0.7.13", "nodebb-rewards-essentials": "0.2.1", "nodebb-theme-lavender": "5.3.2", From 7af057fa512c7b292ca2b91603580363da5cc367 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 15 Feb 2022 17:16:17 -0500 Subject: [PATCH 03/11] fix(deps): update dependency nodebb-plugin-emoji to v3.5.14 (#10295) Co-authored-by: Renovate Bot --- install/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/package.json b/install/package.json index 7ccbe69ef7..65f860cf91 100644 --- a/install/package.json +++ b/install/package.json @@ -88,7 +88,7 @@ "nodebb-plugin-2factor": "3.0.4", "nodebb-plugin-composer-default": "7.0.20", "nodebb-plugin-dbsearch": "5.1.1", - "nodebb-plugin-emoji": "3.5.12", + "nodebb-plugin-emoji": "3.5.14", "nodebb-plugin-emoji-android": "2.0.5", "nodebb-plugin-markdown": "9.0.6", "nodebb-plugin-mentions": "3.0.5", From 5b0d4a8ec9a321eca42bc783a0eba3f253171e31 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 15 Feb 2022 17:20:39 -0500 Subject: [PATCH 04/11] fix(deps): update dependency nodebb-plugin-markdown to v9.0.7 (#10293) Co-authored-by: Renovate Bot --- install/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/package.json b/install/package.json index 65f860cf91..675c2c99f7 100644 --- a/install/package.json +++ b/install/package.json @@ -90,7 +90,7 @@ "nodebb-plugin-dbsearch": "5.1.1", "nodebb-plugin-emoji": "3.5.14", "nodebb-plugin-emoji-android": "2.0.5", - "nodebb-plugin-markdown": "9.0.6", + "nodebb-plugin-markdown": "9.0.7", "nodebb-plugin-mentions": "3.0.5", "nodebb-plugin-spam-be-gone": "0.7.13", "nodebb-rewards-essentials": "0.2.1", From 58b5781cea9acb129e6604a82ab5a5bfc0d8394d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Tue, 15 Feb 2022 19:22:34 -0500 Subject: [PATCH 05/11] feat: closes #10296 asset_base_url in nconf keep assetBaseUrl in config for backwards compat --- public/src/ajaxify.js | 2 +- public/src/modules/translator.js | 2 +- src/controllers/api.js | 4 +++- src/prestart.js | 1 + test/mocks/databasemock.js | 1 + 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/public/src/ajaxify.js b/public/src/ajaxify.js index 04bcb6949e..63f590bc12 100644 --- a/public/src/ajaxify.js +++ b/public/src/ajaxify.js @@ -434,7 +434,7 @@ ajaxify = window.ajaxify || {}; }; ajaxify.loadTemplate = function (template, callback) { - require([config.assetBaseUrl + '/templates/' + template + '.js'], callback, function (err) { + require([config.asset_base_url + '/templates/' + template + '.js'], callback, function (err) { console.error('Unable to load template: ' + template); throw err; }); diff --git a/public/src/modules/translator.js b/public/src/modules/translator.js index 00572536d5..eb40cf658f 100644 --- a/public/src/modules/translator.js +++ b/public/src/modules/translator.js @@ -3,7 +3,7 @@ (function (factory) { function loadClient(language, namespace) { return new Promise(function (resolve, reject) { - jQuery.getJSON([config.assetBaseUrl, 'language', language, namespace].join('/') + '.json?' + config['cache-buster'], function (data) { + jQuery.getJSON([config.asset_base_url, 'language', language, namespace].join('/') + '.json?' + config['cache-buster'], function (data) { const payload = { language: language, namespace: namespace, diff --git a/src/controllers/api.js b/src/controllers/api.js index 1194a75399..7474f6e7a0 100644 --- a/src/controllers/api.js +++ b/src/controllers/api.js @@ -14,6 +14,7 @@ const apiController = module.exports; const relative_path = nconf.get('relative_path'); const upload_url = nconf.get('upload_url'); +const asset_base_url = nconf.get('asset_base_url'); const socketioTransports = nconf.get('socket.io:transports') || ['polling', 'websocket']; const socketioOrigins = nconf.get('socket.io:origins'); const websocketAddress = nconf.get('socket.io:address') || ''; @@ -22,7 +23,8 @@ apiController.loadConfig = async function (req) { const config = { relative_path, upload_url, - assetBaseUrl: `${relative_path}/assets`, + asset_base_url, + assetBaseUrl: asset_base_url, // deprecate in 1.20.x siteTitle: validator.escape(String(meta.config.title || meta.config.browserTitle || 'NodeBB')), browserTitle: validator.escape(String(meta.config.browserTitle || meta.config.title || 'NodeBB')), titleLayout: (meta.config.titleLayout || '{pageTitle} | {browserTitle}').replace(/{/g, '{').replace(/}/g, '}'), diff --git a/src/prestart.js b/src/prestart.js index 55dcb72235..b93bf05838 100644 --- a/src/prestart.js +++ b/src/prestart.js @@ -95,6 +95,7 @@ function loadConfig(configFile) { nconf.set('secure', urlObject.protocol === 'https:'); nconf.set('use_port', !!urlObject.port); nconf.set('relative_path', relativePath); + nconf.set('asset_base_url', `${relativePath}/assets`); nconf.set('port', nconf.get('PORT') || nconf.get('port') || urlObject.port || (nconf.get('PORT_ENV_VAR') ? nconf.get(nconf.get('PORT_ENV_VAR')) : false) || 4567); // cookies don't provide isolation by port: http://stackoverflow.com/a/16328399/122353 diff --git a/test/mocks/databasemock.js b/test/mocks/databasemock.js index 911398b708..414cb7cfdd 100644 --- a/test/mocks/databasemock.js +++ b/test/mocks/databasemock.js @@ -38,6 +38,7 @@ nconf.defaults({ const urlObject = url.parse(nconf.get('url')); const relativePath = urlObject.pathname !== '/' ? urlObject.pathname : ''; nconf.set('relative_path', relativePath); +nconf.set('asset_base_url', `${relativePath}/assets`); nconf.set('upload_path', path.join(nconf.get('base_dir'), nconf.get('upload_path'))); nconf.set('upload_url', '/assets/uploads'); nconf.set('url_parsed', urlObject); From dbf7a45828611c9ca96a4ea8bf41d28b32089e3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Tue, 15 Feb 2022 19:33:52 -0500 Subject: [PATCH 06/11] fix: #10292, delete missing fields --- src/user/delete.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/user/delete.js b/src/user/delete.js index 47cc3b2742..917d8a97c9 100644 --- a/src/user/delete.js +++ b/src/user/delete.js @@ -108,8 +108,11 @@ module.exports = function (User) { `uid:${uid}:notifications:read`, `uid:${uid}:notifications:unread`, `uid:${uid}:bookmarks`, + `uid:${uid}:tids_read`, + `uid:${uid}:tids_unread`, `uid:${uid}:followed_tids`, `uid:${uid}:ignored_tids`, + `uid:${uid}:blocked_uids`, `user:${uid}:settings`, `user:${uid}:usernames`, `user:${uid}:emails`, From b47ca86db5f39850b743bbf466a251f6c3f3a17b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 15 Feb 2022 19:58:49 -0500 Subject: [PATCH 07/11] fix(deps): update dependency nodebb-plugin-emoji to v3.5.16 (#10297) Co-authored-by: Renovate Bot --- install/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/package.json b/install/package.json index 675c2c99f7..61f576200b 100644 --- a/install/package.json +++ b/install/package.json @@ -88,7 +88,7 @@ "nodebb-plugin-2factor": "3.0.4", "nodebb-plugin-composer-default": "7.0.20", "nodebb-plugin-dbsearch": "5.1.1", - "nodebb-plugin-emoji": "3.5.14", + "nodebb-plugin-emoji": "3.5.16", "nodebb-plugin-emoji-android": "2.0.5", "nodebb-plugin-markdown": "9.0.7", "nodebb-plugin-mentions": "3.0.5", From 770fcd9ea82630df2fe3ca9e41421f25fbeb83f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Tue, 15 Feb 2022 20:12:42 -0500 Subject: [PATCH 08/11] fix: dupe key errors --- src/upgrades/1.19.3/fix_user_uploads_zset.js | 15 ++++++--------- src/upgrades/1.19.3/rename_post_upload_hashes.js | 9 +++++---- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/upgrades/1.19.3/fix_user_uploads_zset.js b/src/upgrades/1.19.3/fix_user_uploads_zset.js index 155697bc6d..3b545eea06 100644 --- a/src/upgrades/1.19.3/fix_user_uploads_zset.js +++ b/src/upgrades/1.19.3/fix_user_uploads_zset.js @@ -1,3 +1,4 @@ +/* eslint-disable no-await-in-loop */ 'use strict'; const crypto = require('crypto'); @@ -14,13 +15,11 @@ module.exports = { const { progress } = this; await batch.processSortedSet('users:joindate', async (uids) => { - let keys = uids.map(uid => `uid:${uid}:uploads`); - const exists = await db.exists(keys); - keys = keys.filter((key, idx) => exists[idx]); + const keys = uids.map(uid => `uid:${uid}:uploads`); + progress.incr(uids.length); - progress.incr(uids.length - keys.length); - - await Promise.all(keys.map(async (key, idx) => { + for (let idx = 0; idx < uids.length; idx++) { + const key = keys[idx]; // Rename the paths within let uploads = await db.getSortedSetRangeWithScores(key, 0, -1); @@ -37,9 +36,7 @@ module.exports = { // Add uid to the upload's hash object uploads = await db.getSortedSetMembers(key); await db.setObjectBulk(uploads.map(relativePath => [`upload:${md5(relativePath)}`, { uid: uids[idx] }])); - - progress.incr(); - })); + } }, { batch: 100, progress: progress, diff --git a/src/upgrades/1.19.3/rename_post_upload_hashes.js b/src/upgrades/1.19.3/rename_post_upload_hashes.js index 5e702c39ee..017be8cac1 100644 --- a/src/upgrades/1.19.3/rename_post_upload_hashes.js +++ b/src/upgrades/1.19.3/rename_post_upload_hashes.js @@ -1,3 +1,5 @@ +/* eslint-disable no-await-in-loop */ + 'use strict'; const crypto = require('crypto'); @@ -18,9 +20,9 @@ module.exports = { const exists = await db.exists(keys); keys = keys.filter((key, idx) => exists[idx]); - progress.incr(pids.length - keys.length); + progress.incr(pids.length); - await Promise.all(keys.map(async (key) => { + for (const key of keys) { // Rename the paths within let uploads = await db.getSortedSetRangeWithScores(key, 0, -1); @@ -42,8 +44,7 @@ module.exports = { promises.concat(hashes.map((hash, idx) => db.rename(`upload:${hash}:pids`, `upload:${newHashes[idx]}:pids`))); await Promise.all(promises); - progress.incr(); - })); + } }, { batch: 100, progress: progress, From cfdfbf32804591ae0cd2ed8d654e7b03cb7e7a50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Tue, 15 Feb 2022 20:21:21 -0500 Subject: [PATCH 09/11] fix: one more fix --- src/upgrades/1.19.3/fix_user_uploads_zset.js | 1 + src/upgrades/1.19.3/rename_post_upload_hashes.js | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/upgrades/1.19.3/fix_user_uploads_zset.js b/src/upgrades/1.19.3/fix_user_uploads_zset.js index 3b545eea06..e5989ba75d 100644 --- a/src/upgrades/1.19.3/fix_user_uploads_zset.js +++ b/src/upgrades/1.19.3/fix_user_uploads_zset.js @@ -1,4 +1,5 @@ /* eslint-disable no-await-in-loop */ + 'use strict'; const crypto = require('crypto'); diff --git a/src/upgrades/1.19.3/rename_post_upload_hashes.js b/src/upgrades/1.19.3/rename_post_upload_hashes.js index 017be8cac1..c199a8cf83 100644 --- a/src/upgrades/1.19.3/rename_post_upload_hashes.js +++ b/src/upgrades/1.19.3/rename_post_upload_hashes.js @@ -3,6 +3,7 @@ 'use strict'; const crypto = require('crypto'); +const _ = require('lodash'); const db = require('../../database'); const batch = require('../../batch'); @@ -27,7 +28,7 @@ module.exports = { let uploads = await db.getSortedSetRangeWithScores(key, 0, -1); // Don't process those that have already the right format - uploads = uploads.filter(upload => !upload.value.startsWith('files/')); + uploads = _.uniq(uploads.filter(upload => !upload.value.startsWith('files/'))); // Rename the zset members await db.sortedSetRemove(key, uploads.map(upload => upload.value)); From 2f64d63369b8f56f1f1e4f2e2ea97abd71058516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Tue, 15 Feb 2022 20:22:22 -0500 Subject: [PATCH 10/11] fix: doggy.gif --- src/upgrades/1.19.3/rename_post_upload_hashes.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/upgrades/1.19.3/rename_post_upload_hashes.js b/src/upgrades/1.19.3/rename_post_upload_hashes.js index c199a8cf83..4986dc78d7 100644 --- a/src/upgrades/1.19.3/rename_post_upload_hashes.js +++ b/src/upgrades/1.19.3/rename_post_upload_hashes.js @@ -3,7 +3,6 @@ 'use strict'; const crypto = require('crypto'); -const _ = require('lodash'); const db = require('../../database'); const batch = require('../../batch'); @@ -28,7 +27,7 @@ module.exports = { let uploads = await db.getSortedSetRangeWithScores(key, 0, -1); // Don't process those that have already the right format - uploads = _.uniq(uploads.filter(upload => !upload.value.startsWith('files/'))); + uploads = uploads.filter(upload => upload && upload.value && !upload.value.startsWith('files/')); // Rename the zset members await db.sortedSetRemove(key, uploads.map(upload => upload.value)); From 9205169f002a7fb16aed4875940ea05ae9a91a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Tue, 15 Feb 2022 20:51:52 -0500 Subject: [PATCH 11/11] fix: one last try --- src/upgrades/1.19.3/rename_post_upload_hashes.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/upgrades/1.19.3/rename_post_upload_hashes.js b/src/upgrades/1.19.3/rename_post_upload_hashes.js index 4986dc78d7..5664243c3f 100644 --- a/src/upgrades/1.19.3/rename_post_upload_hashes.js +++ b/src/upgrades/1.19.3/rename_post_upload_hashes.js @@ -40,10 +40,20 @@ module.exports = { // Rename the object and pids zsets const hashes = uploads.map(upload => md5(upload.value)); const newHashes = uploads.map(upload => md5(`files/${upload.value}`)); - const promises = hashes.map((hash, idx) => db.rename(`upload:${hash}`, `upload:${newHashes[idx]}`)); - promises.concat(hashes.map((hash, idx) => db.rename(`upload:${hash}:pids`, `upload:${newHashes[idx]}:pids`))); - await Promise.all(promises); + // cant use db.rename since `fix_user_uploads_zset.js` upgrade script already creates + // `upload:md5(upload.value) hash, trying to rename to existing key results in dupe error + const oldData = await db.getObjects(hashes.map(hash => `upload:${hash}`)); + const bulkSet = []; + oldData.forEach((data, idx) => { + if (data) { + bulkSet.push([`upload:${newHashes[idx]}`, data]); + } + }); + await db.setObjectBulk(bulkSet); + await db.deleteAll(hashes.map(hash => `upload:${hash}`)); + + await Promise.all(hashes.map((hash, idx) => db.rename(`upload:${hash}:pids`, `upload:${newHashes[idx]}:pids`))); } }, { batch: 100,