refactor: use opendir instead of loading all files

src/upgrades/3.8.0/user-upload-folders.js

@@ -20,13 +20,28 @@ module.exports = {
 		await mkdirp(folder);
 		const userPicRegex = /^\d+-profile/;
 
-		const files = (await fs.promises.readdir(folder, { withFileTypes: true }))
-			.filter(item => !item.isDirectory() && String(item.name).match(userPicRegex))
-			.map(item => item.name);
+		const dir = await fs.promises.opendir(folder);
 
-		progress.total = files.length;
-		await batch.processArray(files, async (files) => {
-			progress.incr(files.length);
+		let batchBuffer = [];
+		const BATCH_SIZE = 500;
+
+		for await (const entry of dir) {
+			if (!entry.isDirectory() && userPicRegex.test(entry.name)) {
+				batchBuffer.push(entry.name);
+			}
+			if (batchBuffer.length >= BATCH_SIZE) {
+				await processBatch(batchBuffer);
+				progress.incr(batchBuffer.length);
+				batchBuffer = [];
+			}
+		}
+		if (batchBuffer.length > 0) {
+			await processBatch(batchBuffer);
+			progress.incr(batchBuffer.length);
+			batchBuffer = [];
+		}
+
+		async function processBatch(files) {
 			await Promise.all(files.map(async (file) => {
 				const uid = file.split('-')[0];
 				if (parseInt(uid, 10) > 0) {
@@ -37,9 +52,7 @@ module.exports = {
 					);
 				}
 			}));
-		}, {
-			batch: 500,
-		});
+		}
 
 		await batch.processSortedSet('users:joindate', async (uids) => {
 			progress.incr(uids.length);