From 3e62412ef855a94d3bf01250341e4068f89c8af5 Mon Sep 17 00:00:00 2001 From: rubikscraft Date: Tue, 20 Sep 2022 21:01:45 +0200 Subject: [PATCH] fix some bugs and try things --- backend/package.json | 4 +- .../image-db/image-file-db.service.ts | 21 +++++-- .../sys-preference-db.service.ts | 1 - .../usr-preference-db.service.ts | 1 - .../config/early/type-orm.config.service.ts | 1 + .../src/managers/image/convert.consumer.ts | 6 +- backend/src/managers/image/convert.service.ts | 18 ++++-- .../managers/image/image-manager.module.ts | 14 ++++- backend/src/managers/image/image.queue.ts | 11 ++-- backend/src/managers/image/ingest.consumer.ts | 8 ++- backend/src/managers/image/ingest.service.ts | 63 +++++++++++++++---- .../routes/image/image-manage.controller.ts | 53 +++++++++++----- backend/src/routes/image/image.controller.ts | 12 +++- .../components/masonry/masonry.component.scss | 1 + .../processing/processing.component.html | 8 +-- .../routes/processing/processing.component.ts | 34 ++++++++-- .../src/app/services/api/image.service.ts | 49 ++++++++++++--- shared/src/dto/api/image-manage.dto.ts | 30 +++++++++ yarn.lock | 30 +++++---- 19 files changed, 282 insertions(+), 83 deletions(-) diff --git a/backend/package.json b/backend/package.json index cd5d025..61f32a2 100644 --- a/backend/package.json +++ b/backend/package.json @@ -11,7 +11,7 @@ "prebuild": "rimraf dist", "build": "nest build", "start": "nest start --exec \"node --es-module-specifier-resolution=node\"", - "start:dev": "yarn clean && nest start --watch --exec \"node --es-module-specifier-resolution=node\"", + "start:dev": "yarn clean && nest start --watch --exec \"node --inspect --es-module-specifier-resolution=node\"", "start:debug": "nest start --debug --watch --exec \"node --es-module-specifier-resolution=node\"", "start:prod": "node --es-module-specifier-resolution=node dist/main", "typeorm": "typeorm-ts-node-esm", @@ -64,6 +64,7 @@ "stream-parser": "^0.3.1", "thunks": "^4.9.6", "typeorm": "0.3.9", + "uuid": "^9.0.0", "zod": "^3.19.1" }, "devDependencies": { @@ -80,6 +81,7 @@ "@types/passport-strategy": "^0.2.35", "@types/sharp": "^0.30.5", "@types/supertest": "^2.0.12", + "@types/uuid": "^8.3.4", "@typescript-eslint/eslint-plugin": "^5.37.0", "@typescript-eslint/parser": "^5.37.0", "eslint": "^8.23.1", diff --git a/backend/src/collections/image-db/image-file-db.service.ts b/backend/src/collections/image-db/image-file-db.service.ts index 7991a39..e4eb465 100644 --- a/backend/src/collections/image-db/image-file-db.service.ts +++ b/backend/src/collections/image-db/image-file-db.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum'; import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types'; @@ -10,6 +10,8 @@ const A_DAY_IN_SECONDS = 24 * 60 * 60; @Injectable() export class ImageFileDBService { + private readonly logger = new Logger(ImageFileDBService.name); + constructor( @InjectRepository(EImageFileBackend) private readonly imageFileRepo: Repository, @@ -51,6 +53,11 @@ export class ImageFileDBService { }); if (!found) return Fail(FT.NotFound, 'Image not found'); + + if (!(found.data instanceof Buffer)) { + found.data = Buffer.from(found.data); + } + return found; } catch (e) { return Fail(FT.Database, e); @@ -146,10 +153,16 @@ export class ImageFileDBService { if (!derivative) return null; // Ensure read time updated to within 1 day precision - const yesterday = new Date(Date.now() - A_DAY_IN_SECONDS * 1000); - if (derivative.last_read > yesterday) { + const aMinuteAgo = new Date(Date.now() - 60 * 1000); + if (derivative.last_read > aMinuteAgo) { derivative.last_read = new Date(); - return await this.imageDerivativeRepo.save(derivative); + this.imageDerivativeRepo.save(derivative).then(r => { + if (HasFailed(r)) r.print(this.logger); + }) + } + + if (!(derivative.data instanceof Buffer)) { + derivative.data = Buffer.from(derivative.data); } return derivative; diff --git a/backend/src/collections/preference-db/sys-preference-db.service.ts b/backend/src/collections/preference-db/sys-preference-db.service.ts index 98ac167..f8fd6a6 100644 --- a/backend/src/collections/preference-db/sys-preference-db.service.ts +++ b/backend/src/collections/preference-db/sys-preference-db.service.ts @@ -68,7 +68,6 @@ export class SysPreferenceDbService { try { existing = await this.sysPreferenceRepository.findOne({ where: { key: validatedKey as SysPreference }, - cache: 60000, }); if (!existing) return null; } catch (e) { diff --git a/backend/src/collections/preference-db/usr-preference-db.service.ts b/backend/src/collections/preference-db/usr-preference-db.service.ts index 40dcb60..768ade8 100644 --- a/backend/src/collections/preference-db/usr-preference-db.service.ts +++ b/backend/src/collections/preference-db/usr-preference-db.service.ts @@ -77,7 +77,6 @@ export class UsrPreferenceDbService { try { existing = await this.usrPreferenceRepository.findOne({ where: { key: validatedKey as UsrPreference, user_id: userid }, - cache: 60000, }); if (!existing) return null; } catch (e) { diff --git a/backend/src/config/early/type-orm.config.service.ts b/backend/src/config/early/type-orm.config.service.ts index e411adb..55e88fd 100644 --- a/backend/src/config/early/type-orm.config.service.ts +++ b/backend/src/config/early/type-orm.config.service.ts @@ -71,6 +71,7 @@ export class TypeOrmConfigService implements TypeOrmOptionsFactory { cache: { duration: 60000, type: 'ioredis', + //alwaysEnabled: true, options: this.redisConfig.getRedisUrl(), }, diff --git a/backend/src/managers/image/convert.consumer.ts b/backend/src/managers/image/convert.consumer.ts index bf1a42d..bfa046c 100644 --- a/backend/src/managers/image/convert.consumer.ts +++ b/backend/src/managers/image/convert.consumer.ts @@ -9,7 +9,7 @@ import { ImageFileDBService } from '../../collections/image-db/image-file-db.ser import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service'; import { ImageConverterService } from './image-converter.service'; import { ImageManagerService } from './image-manager.service'; -import { ImageQueueID, ImageQueueSubject } from './image.queue'; +import { ImageConvertQueueID } from './image.queue'; // This contains the job to convert an image to a derivative and store it @@ -21,7 +21,7 @@ export interface ImageConvertJobData { } export type ImageConvertJob = Job; -@Processor(ImageQueueID) +@Processor(ImageConvertQueueID) export class ConvertConsumer { private readonly logger = new Logger(ConvertConsumer.name); @@ -32,7 +32,7 @@ export class ConvertConsumer { private readonly imageService: ImageManagerService, ) {} - @Process(ImageQueueSubject.CONVERT) + @Process() async convertImage(job: ImageConvertJob): Promise { const { imageId, fileType, options, uniqueKey } = job.data; diff --git a/backend/src/managers/image/convert.service.ts b/backend/src/managers/image/convert.service.ts index ef259c5..d542517 100644 --- a/backend/src/managers/image/convert.service.ts +++ b/backend/src/managers/image/convert.service.ts @@ -7,7 +7,7 @@ import { Fail, FT, HasFailed, - ThrowIfFailed + ThrowIfFailed, } from 'picsur-shared/dist/types'; import { ImageFileDBService } from '../../collections/image-db/image-file-db.service'; import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity'; @@ -17,8 +17,8 @@ import * as ImageQueue from './image.queue'; @Injectable() export class ConvertService { constructor( - @InjectQueue(ImageQueue.ImageQueueID) - private readonly imageQueue: ImageQueue.ImageQueueType, + @InjectQueue(ImageQueue.ImageConvertQueueID) + private readonly imageQueue: ImageQueue.ImageConvertQueue, private readonly imageFilesService: ImageFileDBService, ) {} @@ -38,7 +38,6 @@ export class ConvertService { let job: ImageConvertJob; try { job = (await this.imageQueue.add( - ImageQueue.ImageQueueSubject.CONVERT, { imageId, fileType, @@ -64,10 +63,14 @@ export class ConvertService { ): AsyncFailable { const uniqueKey = this.getConvertHash(imageId, { fileType, ...options }); + const startime = Date.now(); const findExisting = ThrowIfFailed( await this.imageFilesService.getDerivative(imageId, uniqueKey), ); - if (findExisting !== null) return findExisting; + if (findExisting !== null) { + console.log('Found existing derivative in ' + (Date.now() - startime)); + return findExisting; + } const job = await this.convertJob(imageId, fileType, options); if (HasFailed(job)) return job; @@ -81,7 +84,10 @@ export class ConvertService { const findResult = ThrowIfFailed( await this.imageFilesService.getDerivative(imageId, uniqueKey), ); - if (findResult !== null) return findResult; + if (findResult !== null) { + console.log('Found new derivative'); + return findResult; + } return Fail(FT.Internal, 'Failed to convert image'); } diff --git a/backend/src/managers/image/image-manager.module.ts b/backend/src/managers/image/image-manager.module.ts index 79d8fe8..f8affd2 100644 --- a/backend/src/managers/image/image-manager.module.ts +++ b/backend/src/managers/image/image-manager.module.ts @@ -13,7 +13,10 @@ import { ConvertConsumer } from './convert.consumer'; import { ConvertService } from './convert.service'; import { ImageConverterService } from './image-converter.service'; import { ImageManagerService } from './image-manager.service'; -import { ImageQueueID } from './image.queue'; +import { + ImageConvertQueueID, + ImageIngestQueueID, +} from './image.queue'; import { IngestConsumer } from './ingest.consumer'; import { IngestService } from './ingest.service'; @@ -22,7 +25,14 @@ import { IngestService } from './ingest.service'; ImageDBModule, PreferenceDbModule, BullModule.registerQueue({ - name: ImageQueueID, + name: ImageConvertQueueID, + defaultJobOptions: { + removeOnComplete: true, + removeOnFail: true, + }, + }), + BullModule.registerQueue({ + name: ImageIngestQueueID, }), ], providers: [ diff --git a/backend/src/managers/image/image.queue.ts b/backend/src/managers/image/image.queue.ts index 6747680..885a607 100644 --- a/backend/src/managers/image/image.queue.ts +++ b/backend/src/managers/image/image.queue.ts @@ -2,11 +2,8 @@ import { Queue } from 'bull'; import { ImageConvertJobData } from './convert.consumer'; import { ImageIngestJobData } from './ingest.consumer'; -export const ImageQueueID = 'image-queue'; -export type ImageQueueType = Queue; - -export enum ImageQueueSubject { - INGEST = 'ingest', - CONVERT = 'convert', -} +export const ImageConvertQueueID = 'image-convert-queue'; +export const ImageIngestQueueID = 'image-ingest-queue'; +export type ImageConvertQueue = Queue; +export type ImageIngestQueue = Queue; diff --git a/backend/src/managers/image/ingest.consumer.ts b/backend/src/managers/image/ingest.consumer.ts index 69e397a..8b1e681 100644 --- a/backend/src/managers/image/ingest.consumer.ts +++ b/backend/src/managers/image/ingest.consumer.ts @@ -21,7 +21,7 @@ import { ImageFileDBService } from '../../collections/image-db/image-file-db.ser import { EImageBackend } from '../../database/entities/images/image.entity'; import { ImageConverterService } from '../image/image-converter.service'; import { ImageResult } from '../image/imageresult'; -import { ImageQueueID, ImageQueueSubject } from './image.queue'; +import { ImageIngestQueueID } from './image.queue'; export interface ImageIngestJobData { imageID: string; @@ -29,7 +29,7 @@ export interface ImageIngestJobData { } export type ImageIngestJob = Job; -@Processor(ImageQueueID) +@Processor(ImageIngestQueueID) export class IngestConsumer { private readonly logger = new Logger(IngestConsumer.name); @@ -39,7 +39,9 @@ export class IngestConsumer { private readonly imageConverter: ImageConverterService, ) {} - @Process(ImageQueueSubject.INGEST) + @Process({ + concurrency: 5, + }) async ingestImage(job: ImageIngestJob): Promise { const { imageID, storeOriginal } = job.data; diff --git a/backend/src/managers/image/ingest.service.ts b/backend/src/managers/image/ingest.service.ts index 7922a90..8ecc6c2 100644 --- a/backend/src/managers/image/ingest.service.ts +++ b/backend/src/managers/image/ingest.service.ts @@ -6,7 +6,7 @@ import { AnimFileType, FileType, ImageFileType, - Mime2FileType + Mime2FileType, } from 'picsur-shared/dist/dto/mimes.dto'; import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.enum'; import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types'; @@ -19,14 +19,15 @@ import { EImageBackend } from '../../database/entities/images/image.entity'; import { WebPInfo } from '../image/webpinfo/webpinfo'; import * as ImageQueue from './image.queue'; import { ImageIngestJob } from './ingest.consumer'; +import { v4 as uuidv4 } from 'uuid'; @Injectable() export class IngestService { private readonly logger = new Logger(IngestService.name); constructor( - @InjectQueue(ImageQueue.ImageQueueID) - private readonly imageQueue: ImageQueue.ImageQueueType, + @InjectQueue(ImageQueue.ImageIngestQueueID) + private readonly imageQueue: ImageQueue.ImageIngestQueue, private readonly imagesService: ImageDBService, private readonly imageFilesService: ImageFileDBService, private readonly userPref: UsrPreferenceDbService, @@ -63,21 +64,26 @@ export class IngestService { ); if (HasFailed(imageEntity)) return imageEntity; - const imageFileEntity = await this.imageFilesService.setFile( - imageEntity.id, - ImageEntryVariant.INGEST, - image, - fileType.identifier, - ); - if (HasFailed(imageFileEntity)) return imageFileEntity; + { + const imageFileEntity = await this.imageFilesService.setFile( + imageEntity.id, + ImageEntryVariant.INGEST, + image, + fileType.identifier, + ); + if (HasFailed(imageFileEntity)) return imageFileEntity; + } try { const job = (await this.imageQueue.add( - ImageQueue.ImageQueueSubject.INGEST, { imageID: imageEntity.id, storeOriginal: keepOriginal, }, + { + jobId: uuidv4(), + delay: 30000, + }, )) as ImageIngestJob; if (!job.id) return Fail(FT.Internal, undefined, 'Failed to queue job'); @@ -106,6 +112,41 @@ export class IngestService { } } + public async getProgress(jobsIds: string[]): AsyncFailable<{ + progress: number; + failed: string[]; + }> { + try { + const jobs = await Promise.all( + jobsIds.map((id) => this.imageQueue.getJob(id)), + ); + + const cleanJobs: ImageIngestJob[] = jobs.filter( + (job) => job !== null, + ) as ImageIngestJob[]; + + if (cleanJobs.length === 0) return { progress: 1, failed: [] }; + + const statefulJobs = await Promise.all( + cleanJobs.map(async (job) => ({ job, state: await job.getState() })), + ); + + const progress = + statefulJobs.filter( + (job) => job.state === 'completed' || job.state === 'failed', + ).length / cleanJobs.length; + + return { + progress, + failed: statefulJobs + .filter((job) => job.state === 'failed') + .map((job) => job.job.id.toString()), + }; + } catch (e) { + return Fail(FT.Internal, e); + } + } + private async getFileTypeFromBuffer(image: Buffer): AsyncFailable { const filetypeResult: FileTypeResult | undefined = await fileTypeFromBuffer( image, diff --git a/backend/src/routes/image/image-manage.controller.ts b/backend/src/routes/image/image-manage.controller.ts index 6376b2b..5b724c9 100644 --- a/backend/src/routes/image/image-manage.controller.ts +++ b/backend/src/routes/image/image-manage.controller.ts @@ -5,7 +5,7 @@ import { Logger, Param, Post, - Res + Res, } from '@nestjs/common'; import { Throttle } from '@nestjs/throttler'; import type { FastifyReply } from 'fastify'; @@ -16,17 +16,22 @@ import { ImageDeleteWithKeyResponse, ImageListRequest, ImageListResponse, + ImagesProgressRequest, + ImagesProgressResponse, + ImagesUploadResponse, ImageUpdateRequest, ImageUpdateResponse, - ImageUploadResponse + ImageUploadResponse, } from 'picsur-shared/dist/dto/api/image-manage.dto'; import { Permission } from 'picsur-shared/dist/dto/permissions.enum'; +import { EImage } from 'picsur-shared/dist/entities/image.entity'; import { Fail, FT, HasFailed, ThrowIfFailed } from 'picsur-shared/dist/types'; +import { EImageBackend } from '../../database/entities/images/image.entity'; import { PostFiles } from '../../decorators/multipart/multipart.decorator'; import type { FileIterator } from '../../decorators/multipart/postfiles.pipe'; import { HasPermission, - RequiredPermissions + RequiredPermissions, } from '../../decorators/permissions.decorator'; import { ReqUserID } from '../../decorators/request-user.decorator'; import { Returns, ReturnsAnything } from '../../decorators/returns.decorator'; @@ -73,31 +78,51 @@ export class ImageManageController { } @Post('upload/bulk') - @ReturnsAnything() + @Returns(ImagesUploadResponse) @Throttle(20) async uploadImages( @PostFiles() multipart: FileIterator, @ReqUserID() userid: string, @HasPermission(Permission.ImageDeleteKey) withDeleteKey: boolean, - ): Promise { - let ids: string[] = []; + ): Promise { + let jobs: { + job_id: string; + image: EImage; + }[] = []; for await (const file of multipart) { const buffer = await file.toBuffer(); const filename = file.filename; - console.log(filename); - // const id = ThrowIfFailed( - // await this.ingressDB.uploadFile(filename, buffer), - // ); - // ids.push(id); + const [job, image] = ThrowIfFailed( + await this.ingestService.uploadJob( + userid, + filename, + buffer, + withDeleteKey, + ), + ); + + jobs.push({ + job_id: job.id.toString(), + image: image, + }); } - if (ids.length === 0) { + if (jobs.length === 0) { throw Fail(FT.BadRequest, 'No files uploaded'); } - console.log(ids); + return { + count: jobs.length, + results: jobs, + }; + } - return; + @Post('upload/status') + @Returns(ImagesProgressResponse) + async getImagesProgress( + @Body() body: ImagesProgressRequest, + ): Promise { + return ThrowIfFailed(await this.ingestService.getProgress(body.job_ids)); } @Post('list') diff --git a/backend/src/routes/image/image.controller.ts b/backend/src/routes/image/image.controller.ts index 2eb5a0a..2b73c29 100644 --- a/backend/src/routes/image/image.controller.ts +++ b/backend/src/routes/image/image.controller.ts @@ -3,7 +3,7 @@ import { SkipThrottle } from '@nestjs/throttler'; import type { FastifyReply } from 'fastify'; import { ImageMetaResponse, - ImageRequestParams + ImageRequestParams, } from 'picsur-shared/dist/dto/api/image.dto'; import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum'; import { FileType2Mime } from 'picsur-shared/dist/dto/mimes.dto'; @@ -75,6 +75,16 @@ export class ImageController { params, ), ); + + const isbfufer = image.data instanceof Buffer; + console.log('isabuffer', isbfufer); + if (!isbfufer) { + console.log('not a buffer'); + console.log(image.data); + + console.trace(); + process.exit(); + } res.type(ThrowIfFailed(FileType2Mime(image.filetype))); return image.data; diff --git a/frontend/src/app/components/masonry/masonry.component.scss b/frontend/src/app/components/masonry/masonry.component.scss index 1c3c4e9..cfdeacc 100644 --- a/frontend/src/app/components/masonry/masonry.component.scss +++ b/frontend/src/app/components/masonry/masonry.component.scss @@ -7,5 +7,6 @@ display: flex; flex-grow: 1; flex-basis: 0; + width: 0; flex-direction: column; } diff --git a/frontend/src/app/routes/processing/processing.component.html b/frontend/src/app/routes/processing/processing.component.html index c88be0c..dde96b8 100644 --- a/frontend/src/app/routes/processing/processing.component.html +++ b/frontend/src/app/routes/processing/processing.component.html @@ -6,18 +6,18 @@

Uploading

-

{{ (progress * 100).toFixed(0) }}% complete

+

{{ (progress * 100).toFixed(0) }}% complete

Processing

-

{{ (progress * 100).toFixed(0) }}% complete

+

{{ (progress * 100).toFixed(0) }}% complete

diff --git a/frontend/src/app/routes/processing/processing.component.ts b/frontend/src/app/routes/processing/processing.component.ts index c878993..a3d84c9 100644 --- a/frontend/src/app/routes/processing/processing.component.ts +++ b/frontend/src/app/routes/processing/processing.component.ts @@ -1,6 +1,6 @@ import { Component, OnDestroy, OnInit } from '@angular/core'; import { Router } from '@angular/router'; -import { Fail, FT } from 'picsur-shared/dist/types'; +import { Fail, Failable, FT, HasFailed } from 'picsur-shared/dist/types'; import { ProcessingViewMeta } from 'src/app/models/dto/processing-view-meta.dto'; import { ApiService } from 'src/app/services/api/api.service'; import { ImageService } from 'src/app/services/api/image.service'; @@ -21,7 +21,8 @@ export class ProcessingComponent implements OnInit, OnDestroy { private readonly logger = new Logger(ProcessingComponent.name); public state = ProcessingState.Idle; - public progress = 0; + public progress = -1; + public poller?: number; constructor( private readonly router: Router, @@ -47,16 +48,41 @@ export class ProcessingComponent implements OnInit, OnDestroy { }); this.state = ProcessingState.Uploading; - await request.result; - + const result = await request.result; + if (HasFailed(result)) { + return this.errorService.quitFailure(result, this.logger); + } this.logger.debug('Upload finished'); + this.state = ProcessingState.Processing; + this.progress = -1; + + const jobIds = result.map((v) => v.job_id); + + this.poller = window.setInterval(async () => { + const progress = await this.imageService.GetUploadProgress(jobIds); + if (HasFailed(progress)) { + return this.errorService.showFailure(progress, this.logger); + } + + this.progress = progress; + if (progress === 1) { + if (this.poller) { + clearInterval(this.poller); + } + this.router.navigate(['/']); + } + }, 1000); + // if (HasFailed(id)) return this.errorService.quitFailure(id, this.logger); // this.router.navigate([`/view/`, id], { replaceUrl: true }); } ngOnDestroy(): void { + if (this.poller) { + clearInterval(this.poller); + } if (this.state === ProcessingState.Idle) return; this.errorService.info('Upload continued in background'); diff --git a/frontend/src/app/services/api/image.service.ts b/frontend/src/app/services/api/image.service.ts index 52ba06c..7a330de 100644 --- a/frontend/src/app/services/api/image.service.ts +++ b/frontend/src/app/services/api/image.service.ts @@ -4,13 +4,16 @@ import { ImageDeleteResponse, ImageListRequest, ImageListResponse, + ImagesProgressRequest, + ImagesProgressResponse, + ImagesUploadResponse, ImageUpdateRequest, ImageUpdateResponse, - ImageUploadResponse + ImageUploadResponse, } from 'picsur-shared/dist/dto/api/image-manage.dto'; import { ImageMetaResponse, - ImageRequestParams + ImageRequestParams, } from 'picsur-shared/dist/dto/api/image.dto'; import { ImageLinks } from 'picsur-shared/dist/dto/image-links.class'; import { FileType2Ext } from 'picsur-shared/dist/dto/mimes.dto'; @@ -21,7 +24,7 @@ import { FT, HasFailed, HasSuccess, - Open + Open, } from 'picsur-shared/dist/types/failable'; import { Observable, Subject } from 'rxjs'; import { ImagesUploadRequest } from 'src/app/models/dto/images-upload-request.dto'; @@ -52,7 +55,12 @@ export class ImageService { public UploadImages(images: File[]): { progress: Observable; - result: AsyncFailable; + result: AsyncFailable< + Array<{ + job_id: string; + image: EImage; + }> + >; cancel: () => void; } { console.log('Uploading images', images); @@ -71,9 +79,14 @@ export class ImageService { const result = (async () => { let processedBytes = 0; + let results: Array<{ + job_id: string; + image: EImage; + }> = []; + for (const group of groups) { - const request = await this.api.postForm( - ImageUploadResponse, + const request = this.api.postForm( + ImagesUploadResponse, '/api/image/upload/bulk', new ImagesUploadRequest(group.images), ); @@ -86,21 +99,37 @@ export class ImageService { request.cancel(); }); - await request.result; + const partResults = await request.result; + if (HasFailed(partResults)) return partResults; + + results.push(...partResults.results); progress.next((processedBytes += group.groupSize) / totalBytes); } - return ''; + return results; })(); return { progress: progress.asObservable(), - result: result, + result, cancel: () => aborter.abort(), }; } + public async GetUploadProgress(jobIds: string[]): AsyncFailable { + const result = await this.api.post( + ImagesProgressRequest, + ImagesProgressResponse, + '/api/image/upload/status', + { + job_ids: jobIds, + }, + ).result; + + return Open(result, 'progress'); + } + public async GetImageMeta(image: string): AsyncFailable { return await this.api.get(ImageMetaResponse, `/i/meta/${image}`).result; } @@ -126,7 +155,7 @@ export class ImageService { count: number, page: number, ): AsyncFailable { - const userID = await this.userService.snapshot?.id; + const userID = this.userService.snapshot?.id; if (userID === undefined) { return Fail(FT.Authentication, 'User not logged in'); } diff --git a/shared/src/dto/api/image-manage.dto.ts b/shared/src/dto/api/image-manage.dto.ts index de6757c..fe3492b 100644 --- a/shared/src/dto/api/image-manage.dto.ts +++ b/shared/src/dto/api/image-manage.dto.ts @@ -13,6 +13,36 @@ export class ImageUploadResponse extends createZodDto( ImageUploadResponseSchema, ) {} +// Images upload +export const ImagesUploadResponseSchema = z.object({ + count: IsPosInt(), + results: z.array( + z.object({ + job_id: z.string(), + image: EImageSchema, + }), + ), +}); +export class ImagesUploadResponse extends createZodDto( + ImagesUploadResponseSchema, +) {} + +// Images progress +export const ImagesProgressRequestSchema = z.object({ + job_ids: z.array(z.string()), +}); +export class ImagesProgressRequest extends createZodDto( + ImagesProgressRequestSchema, +) {} + +export const ImagesProgressResponseSchema = z.object({ + progress: z.number(), + failed: z.array(z.string()), +}); +export class ImagesProgressResponse extends createZodDto( + ImagesProgressResponseSchema, +) {} + // Image list export const ImageListRequestSchema = z.object({ diff --git a/yarn.lock b/yarn.lock index f224889..61771db 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3195,6 +3195,13 @@ __metadata: languageName: node linkType: hard +"@types/uuid@npm:^8.3.4": + version: 8.3.4 + resolution: "@types/uuid@npm:8.3.4" + checksum: 6f11f3ff70f30210edaa8071422d405e9c1d4e53abbe50fdce365150d3c698fe7bbff65c1e71ae080cbfb8fded860dbb5e174da96fdbbdfcaa3fb3daa474d20f + languageName: node + linkType: hard + "@types/validator@npm:^13.7.6": version: 13.7.6 resolution: "@types/validator@npm:13.7.6" @@ -6136,17 +6143,7 @@ __metadata: languageName: node linkType: hard -"follow-redirects@npm:^1.0.0": - version: 1.15.1 - resolution: "follow-redirects@npm:1.15.1" - peerDependenciesMeta: - debug: - optional: true - checksum: 6aa4e3e3cdfa3b9314801a1cd192ba756a53479d9d8cca65bf4db3a3e8834e62139245cd2f9566147c8dfe2efff1700d3e6aefd103de4004a7b99985e71dd533 - languageName: node - linkType: hard - -"follow-redirects@npm:^1.14.9": +"follow-redirects@npm:^1.0.0, follow-redirects@npm:^1.14.9": version: 1.15.2 resolution: "follow-redirects@npm:1.15.2" peerDependenciesMeta: @@ -9134,6 +9131,7 @@ __metadata: "@types/passport-strategy": ^0.2.35 "@types/sharp": ^0.30.5 "@types/supertest": ^2.0.12 + "@types/uuid": ^8.3.4 "@typescript-eslint/eslint-plugin": ^5.37.0 "@typescript-eslint/parser": ^5.37.0 bcrypt: ^5.0.1 @@ -9170,6 +9168,7 @@ __metadata: tsconfig-paths: ^4.1.0 typeorm: 0.3.9 typescript: 4.8.3 + uuid: ^9.0.0 zod: ^3.19.1 languageName: unknown linkType: soft @@ -11878,6 +11877,15 @@ __metadata: languageName: node linkType: hard +"uuid@npm:^9.0.0": + version: 9.0.0 + resolution: "uuid@npm:9.0.0" + bin: + uuid: dist/bin/uuid + checksum: 8dd2c83c43ddc7e1c71e36b60aea40030a6505139af6bee0f382ebcd1a56f6cd3028f7f06ffb07f8cf6ced320b76aea275284b224b002b289f89fe89c389b028 + languageName: node + linkType: hard + "v8-compile-cache-lib@npm:^3.0.1": version: 3.0.1 resolution: "v8-compile-cache-lib@npm:3.0.1"