diff --git a/src/upgrades/3.8.0/user-upload-folders.js b/src/upgrades/3.8.0/user-upload-folders.js index fbacbc64c4..dbca210322 100644 --- a/src/upgrades/3.8.0/user-upload-folders.js +++ b/src/upgrades/3.8.0/user-upload-folders.js @@ -20,13 +20,28 @@ module.exports = { await mkdirp(folder); const userPicRegex = /^\d+-profile/; - const files = (await fs.promises.readdir(folder, { withFileTypes: true })) - .filter(item => !item.isDirectory() && String(item.name).match(userPicRegex)) - .map(item => item.name); + const dir = await fs.promises.opendir(folder); - progress.total = files.length; - await batch.processArray(files, async (files) => { - progress.incr(files.length); + let batchBuffer = []; + const BATCH_SIZE = 500; + + for await (const entry of dir) { + if (!entry.isDirectory() && userPicRegex.test(entry.name)) { + batchBuffer.push(entry.name); + } + if (batchBuffer.length >= BATCH_SIZE) { + await processBatch(batchBuffer); + progress.incr(batchBuffer.length); + batchBuffer = []; + } + } + if (batchBuffer.length > 0) { + await processBatch(batchBuffer); + progress.incr(batchBuffer.length); + batchBuffer = []; + } + + async function processBatch(files) { await Promise.all(files.map(async (file) => { const uid = file.split('-')[0]; if (parseInt(uid, 10) > 0) { @@ -37,9 +52,7 @@ module.exports = { ); } })); - }, { - batch: 500, - }); + } await batch.processSortedSet('users:joindate', async (uids) => { progress.incr(uids.length);