diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index ef4cfb9..3affe3e 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -7,12 +7,12 @@ import { IncomingMessage, ServerResponse } from 'http'; import { BullConfigService } from './config/early/bull.config.service'; import { EarlyConfigModule } from './config/early/early-config.module'; import { ServeStaticConfigService } from './config/early/serve-static.config.service'; -import { ConsumersModule } from './consumers/consumers.module'; import { DatabaseModule } from './database/database.module'; import { PicsurLayersModule } from './layers/PicsurLayers.module'; import { PicsurLoggerModule } from './logger/logger.module'; import { AuthManagerModule } from './managers/auth/auth.module'; import { DemoManagerModule } from './managers/demo/demo.module'; +import { IngestManagerModule } from './managers/ingest/ingest.module'; import { UsageManagerModule } from './managers/usage/usage.module'; import { PicsurRoutesModule } from './routes/routes.module'; @@ -59,7 +59,7 @@ const imageCorsOverride = ( DemoManagerModule, PicsurRoutesModule, PicsurLayersModule, - ConsumersModule, + IngestManagerModule, ], }) export class AppModule implements NestModule { diff --git a/backend/src/collections/image-db/image-file-db.service.ts b/backend/src/collections/image-db/image-file-db.service.ts index 00b46a7..7991a39 100644 --- a/backend/src/collections/image-db/image-file-db.service.ts +++ b/backend/src/collections/image-db/image-file-db.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum'; -import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types'; +import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types'; import { LessThan, Repository } from 'typeorm'; import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity'; import { EImageFileBackend } from '../../database/entities/images/image-file.entity'; @@ -57,6 +57,40 @@ export class ImageFileDBService { } } + public async migrateFile( + imageId: string, + sourceVariant: ImageEntryVariant, + targetVariant: ImageEntryVariant, + ): AsyncFailable { + try { + const sourceFile = await this.getFile(imageId, sourceVariant); + if (HasFailed(sourceFile)) return sourceFile; + + sourceFile.variant = targetVariant; + return await this.imageFileRepo.save(sourceFile); + } catch (e) { + return Fail(FT.Database, e); + } + } + + public async deleteFile( + imageId: string, + variant: ImageEntryVariant, + ): AsyncFailable { + try { + const found = await this.imageFileRepo.findOne({ + where: { image_id: imageId, variant: variant }, + }); + + if (!found) return Fail(FT.NotFound, 'Image not found'); + + await this.imageFileRepo.delete({ image_id: imageId, variant: variant }); + return found; + } catch (e) { + return Fail(FT.Database, e); + } + } + // This is useful because you dont have to pull the whole image file public async getFileTypes( imageId: string, diff --git a/backend/src/collections/ingest-file-db/ingest-file-db.module.ts b/backend/src/collections/ingest-file-db/ingest-file-db.module.ts new file mode 100644 index 0000000..32351f6 --- /dev/null +++ b/backend/src/collections/ingest-file-db/ingest-file-db.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { EIngestFileBackend } from '../../database/entities/ingest-file.entity'; +import { IngestFileDbService } from './ingest-file-db.service'; + +@Module({ + imports: [TypeOrmModule.forFeature([EIngestFileBackend])], + providers: [IngestFileDbService], + exports: [IngestFileDbService], +}) +export class IngestFileDbModule {} diff --git a/backend/src/collections/ingest-file-db/ingest-file-db.service.ts b/backend/src/collections/ingest-file-db/ingest-file-db.service.ts new file mode 100644 index 0000000..9f81b8f --- /dev/null +++ b/backend/src/collections/ingest-file-db/ingest-file-db.service.ts @@ -0,0 +1,32 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types'; +import { Repository } from 'typeorm'; +import { EIngestFileBackend } from '../../database/entities/ingest-file.entity'; + +@Injectable() +export class IngestFileDbService { + private readonly logger = new Logger(IngestFileDbService.name); + + constructor( + @InjectRepository(EIngestFileBackend) + private readonly ingressFileRepo: Repository, + ) {} + + public async uploadFile( + filename: string, + file: Buffer, + ): AsyncFailable { + const ingressFile = new EIngestFileBackend(); + ingressFile.filename = filename; + ingressFile.data = file; + + try { + await this.ingressFileRepo.save(ingressFile); + } catch (e) { + return Fail(FT.Database, e); + } + + return ingressFile.id; + } +} diff --git a/backend/src/collections/ingress-file-db/ingress-file-db.module.ts b/backend/src/collections/ingress-file-db/ingress-file-db.module.ts deleted file mode 100644 index 6131318..0000000 --- a/backend/src/collections/ingress-file-db/ingress-file-db.module.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Module } from '@nestjs/common'; -import { TypeOrmModule } from '@nestjs/typeorm'; -import { EIngressFileBackend } from '../../database/entities/ingress-file.entity'; -import { IngressFileDbService } from './ingress-file-db.service'; - -@Module({ - imports: [TypeOrmModule.forFeature([EIngressFileBackend])], - providers: [IngressFileDbService], - exports: [IngressFileDbService], -}) -export class IngressFileDbModule {} diff --git a/backend/src/collections/ingress-file-db/ingress-file-db.service.ts b/backend/src/collections/ingress-file-db/ingress-file-db.service.ts deleted file mode 100644 index bb4b24d..0000000 --- a/backend/src/collections/ingress-file-db/ingress-file-db.service.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; -import { Repository } from 'typeorm'; -import { EIngressFileBackend } from '../../database/entities/ingress-file.entity'; - -@Injectable() -export class IngressFileDbService { - private readonly logger = new Logger(IngressFileDbService.name); - - constructor( - @InjectRepository(EIngressFileBackend) - private readonly ingressFileRepo: Repository, - ) {} -} diff --git a/backend/src/config/early/bull.config.service.ts b/backend/src/config/early/bull.config.service.ts index 079c7b5..abc310f 100644 --- a/backend/src/config/early/bull.config.service.ts +++ b/backend/src/config/early/bull.config.service.ts @@ -17,7 +17,16 @@ export class BullConfigService implements SharedBullConfigurationFactory { }, defaultJobOptions: { attempts: 3, - removeOnFail: true, + backoff: { + delay: 500, + type: 'fixed', + }, + removeOnFail: { + age: 1000 * 60 * 60 * 24 * 7, // 7 days + }, + removeOnComplete: { + age: 1000 * 60 * 60 * 24 * 7, // 7 days + }, }, }; return options; diff --git a/backend/src/config/late/info.config.service.ts b/backend/src/config/late/info.config.service.ts index fea9b2c..891aadc 100644 --- a/backend/src/config/late/info.config.service.ts +++ b/backend/src/config/late/info.config.service.ts @@ -14,7 +14,7 @@ export class InfoConfigService { SysPreference.HostOverride, ); if (HasFailed(hostname)) { - this.logger.warn(hostname.print()); + hostname.print(this.logger); return undefined; } diff --git a/backend/src/consumers/consumers.module.ts b/backend/src/consumers/consumers.module.ts deleted file mode 100644 index 0a98916..0000000 --- a/backend/src/consumers/consumers.module.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { BullModule } from '@nestjs/bull'; -import { Module } from '@nestjs/common'; -import { IngestConsumer } from './ingest.consumer'; - -@Module({ - imports: [ - BullModule.registerQueue({ - name: 'image-ingest', - }), - ], - providers: [IngestConsumer], - exports: [BullModule], -}) -export class ConsumersModule {} diff --git a/backend/src/consumers/ingest.consumer.ts b/backend/src/consumers/ingest.consumer.ts deleted file mode 100644 index 0e984bf..0000000 --- a/backend/src/consumers/ingest.consumer.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { OnQueueError, Process, Processor } from '@nestjs/bull'; -import { Logger } from '@nestjs/common'; -import type { Job } from 'bull'; - -@Processor('image-ingest') -export class IngestConsumer { - private readonly logger = new Logger(IngestConsumer.name); - - @Process() - async processJob(job: Job) { - console.log('processJob', job); - } - - @OnQueueError() - async handleError(error: any) { - this.logger.error(error); - } -} diff --git a/backend/src/database/entities/index.ts b/backend/src/database/entities/index.ts index c34d063..755b6fa 100644 --- a/backend/src/database/entities/index.ts +++ b/backend/src/database/entities/index.ts @@ -2,7 +2,7 @@ import { EApiKeyBackend } from './apikey.entity'; import { EImageDerivativeBackend } from './images/image-derivative.entity'; import { EImageFileBackend } from './images/image-file.entity'; import { EImageBackend } from './images/image.entity'; -import { EIngressFileBackend } from './ingress-file.entity'; +import { EIngestFileBackend } from './ingest-file.entity'; import { ESysPreferenceBackend } from './system/sys-preference.entity'; import { ESystemStateBackend } from './system/system-state.entity'; import { EUsrPreferenceBackend } from './system/usr-preference.entity'; @@ -19,5 +19,5 @@ export const EntityList = [ EUsrPreferenceBackend, EApiKeyBackend, ESystemStateBackend, - EIngressFileBackend, + EIngestFileBackend, ]; diff --git a/backend/src/database/entities/ingress-file.entity.ts b/backend/src/database/entities/ingest-file.entity.ts similarity index 89% rename from backend/src/database/entities/ingress-file.entity.ts rename to backend/src/database/entities/ingest-file.entity.ts index 3e6ef88..dd4137f 100644 --- a/backend/src/database/entities/ingress-file.entity.ts +++ b/backend/src/database/entities/ingest-file.entity.ts @@ -1,7 +1,7 @@ import { Column, Entity, PrimaryGeneratedColumn } from 'typeorm'; @Entity() -export class EIngressFileBackend { +export class EIngestFileBackend { @PrimaryGeneratedColumn('uuid') id: string; diff --git a/backend/src/decorators/multipart/postfiles.pipe.ts b/backend/src/decorators/multipart/postfiles.pipe.ts index 3a3f18f..dd80b39 100644 --- a/backend/src/decorators/multipart/postfiles.pipe.ts +++ b/backend/src/decorators/multipart/postfiles.pipe.ts @@ -26,7 +26,7 @@ export class MultiPartPipe implements PipeTransform { ) { const filesLimit = typeof data === 'number' ? data : undefined; - if (!request.isMultipart()) throw Fail(FT.UsrValidation, 'Invalid file'); + if (!request.isMultipart()) throw Fail(FT.UsrValidation, 'Invalid files'); const files = request.files({ limits: this.multipartConfigService.getLimits(filesLimit), diff --git a/backend/src/layers/exception/exception.filter.ts b/backend/src/layers/exception/exception.filter.ts index a7de68a..8ea90b3 100644 --- a/backend/src/layers/exception/exception.filter.ts +++ b/backend/src/layers/exception/exception.filter.ts @@ -6,7 +6,7 @@ import { Logger, MethodNotAllowedException, NotFoundException, - UnauthorizedException, + UnauthorizedException } from '@nestjs/common'; import { FastifyReply, FastifyRequest } from 'fastify'; import { ApiErrorResponse } from 'picsur-shared/dist/dto/api/api.dto'; @@ -14,7 +14,7 @@ import { Fail, Failure, FT, - IsFailure, + IsFailure } from 'picsur-shared/dist/types/failable'; // This will catch any exception that is made in any request @@ -39,23 +39,7 @@ export class MainExceptionFilter implements ExceptionFilter { const status = exception.getCode(); const type = exception.getType(); - const message = exception.getReason(); - const logmessage = - message + - (exception.getDebugMessage() ? ' - ' + exception.getDebugMessage() : ''); - - if (exception.isImportant()) { - MainExceptionFilter.logger.error( - `${traceString} ${exception.getName()}: ${logmessage}`, - ); - if (exception.getStack()) { - MainExceptionFilter.logger.debug(exception.getStack()); - } - } else { - MainExceptionFilter.logger.warn( - `${traceString} ${exception.getName()}: ${logmessage}`, - ); - } + exception.print(MainExceptionFilter.logger, { prefix: traceString }); const toSend: ApiErrorResponse = { success: false, @@ -65,7 +49,7 @@ export class MainExceptionFilter implements ExceptionFilter { data: { type, - message, + message: exception.getReason(), }, }; diff --git a/backend/src/main.ts b/backend/src/main.ts index 815c771..10864e2 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -4,7 +4,7 @@ import fastifyReplyFrom from '@fastify/reply-from'; import { NestFactory } from '@nestjs/core'; import { FastifyAdapter, - NestFastifyApplication, + NestFastifyApplication } from '@nestjs/platform-fastify'; import { AppModule } from './app.module'; import { HostConfigService } from './config/early/host.config.service'; @@ -39,14 +39,17 @@ async function bootstrap() { fastifyAdapter, { bufferLogs: isProduction, + autoFlushLogs: true, }, ); // Configure logger - app.useLogger(app.get(PicsurLoggerService)); - + const logger = app.get(PicsurLoggerService) + app.useLogger(logger); app.flushLogs(); + console.log(logger); + app.useGlobalFilters(app.get(MainExceptionFilter)); app.useGlobalInterceptors(app.get(SuccessInterceptor)); app.useGlobalPipes(app.get(ZodValidationPipe)); diff --git a/backend/src/managers/image/image-converter.service.ts b/backend/src/managers/image/image-converter.service.ts index f814d6a..db810e5 100644 --- a/backend/src/managers/image/image-converter.service.ts +++ b/backend/src/managers/image/image-converter.service.ts @@ -3,7 +3,7 @@ import ms from 'ms'; import { ImageRequestParams } from 'picsur-shared/dist/dto/api/image.dto'; import { FileType, - SupportedFileTypeCategory, + SupportedFileTypeCategory } from 'picsur-shared/dist/dto/mimes.dto'; import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum'; import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types'; @@ -122,16 +122,4 @@ export class ImageConverterService { filetype: targetFiletype.identifier, }; } - - private async convertAnimation( - image: Buffer, - targetFiletype: FileType, - options: ImageRequestParams, - ): AsyncFailable { - // Apng and gif are stored as is for now - return { - image: image, - filetype: targetFiletype.identifier, - }; - } } diff --git a/backend/src/managers/image/image-processor.service.ts b/backend/src/managers/image/image-processor.service.ts index ee451d0..045b69f 100644 --- a/backend/src/managers/image/image-processor.service.ts +++ b/backend/src/managers/image/image-processor.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common'; import { FileType, ImageFileType, - SupportedFileTypeCategory, + SupportedFileTypeCategory } from 'picsur-shared/dist/dto/mimes.dto'; import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types'; diff --git a/backend/src/managers/image/image.module.ts b/backend/src/managers/image/image.module.ts index ef4001c..f90cc62 100644 --- a/backend/src/managers/image/image.module.ts +++ b/backend/src/managers/image/image.module.ts @@ -19,7 +19,7 @@ import { ImageManagerService } from './image.service'; ImageProcessorService, ImageConverterService, ], - exports: [ImageManagerService], + exports: [ImageManagerService, ImageConverterService], }) export class ImageManagerModule implements OnModuleInit { private readonly logger = new Logger(ImageManagerModule.name); @@ -31,7 +31,7 @@ export class ImageManagerModule implements OnModuleInit { ) {} async onModuleInit() { - await this.imageManagerCron() + await this.imageManagerCron(); } @Interval(1000 * 60) @@ -57,7 +57,7 @@ export class ImageManagerModule implements OnModuleInit { const result = await this.imageFileDB.cleanupDerivatives(after_ms / 1000); if (HasFailed(result)) { - this.logger.warn(result.print()); + result.print(this.logger); } if (result > 0) this.logger.log(`Cleaned up ${result} derivatives`); @@ -67,7 +67,8 @@ export class ImageManagerModule implements OnModuleInit { const cleanedUp = await this.imageDB.cleanupExpired(); if (HasFailed(cleanedUp)) { - this.logger.warn(cleanedUp.print()); + cleanedUp.print(this.logger); + return; } if (cleanedUp > 0) diff --git a/backend/src/managers/ingest/ingest.consumer.ts b/backend/src/managers/ingest/ingest.consumer.ts new file mode 100644 index 0000000..d258cb1 --- /dev/null +++ b/backend/src/managers/ingest/ingest.consumer.ts @@ -0,0 +1,161 @@ +import { + InjectQueue, + OnQueueError, + OnQueueFailed, + Process, + Processor +} from '@nestjs/bull'; +import { Logger } from '@nestjs/common'; +import type { Job, Queue } from 'bull'; +import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum'; +import { + FileType, + ImageFileType, + SupportedFileTypeCategory +} from 'picsur-shared/dist/dto/mimes.dto'; +import { + AsyncFailable, + Fail, + FT, + HasFailed, + IsFailure, + ThrowIfFailed +} from 'picsur-shared/dist/types'; +import { ParseFileType } from 'picsur-shared/dist/util/parse-mime'; +import { ImageDBService } from '../../collections/image-db/image-db.service'; +import { ImageFileDBService } from '../../collections/image-db/image-file-db.service'; +import { EImageBackend } from '../../database/entities/images/image.entity'; +import { ImageConverterService } from '../image/image-converter.service'; +import { ImageResult } from '../image/imageresult'; + +interface ImageIngestJobData { + imageID: string; + storeOriginal: boolean; +} +export type ImageIngestQueue = Queue; +export type ImageIngestJob = Job; + +@Processor('image-ingest') +export class IngestConsumer { + private readonly logger = new Logger(IngestConsumer.name); + private i = 0; + + constructor( + @InjectQueue('image-ingest') private readonly ingestQueue: Queue, + private readonly imagesService: ImageDBService, + private readonly imageFilesService: ImageFileDBService, + private readonly imageConverter: ImageConverterService, + ) { + this.logger.log('Ingest consumer started'); + this.logger.error('Ingest consumer started'); + } + + // @Process('group') + // async processJob(job: Job) { + // console.log('Received', job.data); + // await new Promise((resolve) => setTimeout(resolve, 4000)); + // console.log('Done'); + // return 'big chungus'; + // } + + @Process('image') + async processImage(job: ImageIngestJob): Promise { + const { imageID, storeOriginal } = job.data; + + job.failedReason = 'Not implemented'; + + if (this.i === 0) { + throw Fail(FT.Internal, undefined, 'oops'); + } + + // Already start the query for the image, we only need it when returning + const imagePromise = this.imagesService.findOne(imageID, undefined); + + this.logger.verbose( + `Ingesting image ${imageID} and store original: ${storeOriginal}`, + ); + + const ingestFile = ThrowIfFailed( + await this.imageFilesService.getFile(imageID, ImageEntryVariant.INGEST), + ); + + const ingestFiletype = ThrowIfFailed(ParseFileType(ingestFile.filetype)); + + const processed = ThrowIfFailed( + await this.process(ingestFile.data, ingestFiletype), + ); + + const masterPromise = this.imageFilesService.setFile( + imageID, + ImageEntryVariant.MASTER, + processed.image, + processed.filetype, + ); + + const originalPromise = storeOriginal + ? this.imageFilesService.migrateFile( + imageID, + ImageEntryVariant.INGEST, + ImageEntryVariant.ORIGINAL, + ) + : this.imageFilesService.deleteFile(imageID, ImageEntryVariant.INGEST); + + const results = await Promise.all([masterPromise, originalPromise]); + results.map((r) => ThrowIfFailed(r)); + + const image = ThrowIfFailed(await imagePromise); + + this.logger.verbose(`Ingested image ${imageID}`); + + return image; + } + + public async process( + image: Buffer, + filetype: FileType, + ): AsyncFailable { + if (filetype.category === SupportedFileTypeCategory.Image) { + return await this.processStill(image, filetype); + } else if (filetype.category === SupportedFileTypeCategory.Animation) { + return await this.processAnimation(image, filetype); + } else { + return Fail(FT.SysValidation, 'Unsupported mime type'); + } + } + + private async processStill( + image: Buffer, + filetype: FileType, + ): AsyncFailable { + const outputFileType = ParseFileType(ImageFileType.QOI); + if (HasFailed(outputFileType)) return outputFileType; + + return this.imageConverter.convert(image, filetype, outputFileType, {}); + } + + private async processAnimation( + image: Buffer, + filetype: FileType, + ): AsyncFailable { + // Webps and gifs are stored as is for now + return { + image: image, + filetype: filetype.identifier, + }; + } + + @OnQueueError() + async handleError(error: any) { + if (IsFailure(error)) error.print(this.logger); + else this.logger.error(error); + } + + @OnQueueFailed() + async handleFailed(job: Job, error: any) { + if (IsFailure(error)) + error.print(this.logger, { + prefix: `[JOB ${job.id}]`, + }); + else this.logger.error(error); + } +} diff --git a/backend/src/managers/ingest/ingest.module.ts b/backend/src/managers/ingest/ingest.module.ts new file mode 100644 index 0000000..b75adec --- /dev/null +++ b/backend/src/managers/ingest/ingest.module.ts @@ -0,0 +1,21 @@ +import { BullModule } from '@nestjs/bull'; +import { Module } from '@nestjs/common'; +import { ImageDBModule } from '../../collections/image-db/image-db.module'; +import { PreferenceDbModule } from '../../collections/preference-db/preference-db.module'; +import { ImageManagerModule } from '../image/image.module'; +import { IngestConsumer } from './ingest.consumer'; +import { IngestService } from './ingest.service'; + +@Module({ + imports: [ + ImageDBModule, + ImageManagerModule, + PreferenceDbModule, + BullModule.registerQueue({ + name: 'image-ingest', + }), + ], + providers: [IngestConsumer, IngestService], + exports: [BullModule, IngestService], +}) +export class IngestManagerModule {} diff --git a/backend/src/managers/ingest/ingest.service.ts b/backend/src/managers/ingest/ingest.service.ts new file mode 100644 index 0000000..dd0bae2 --- /dev/null +++ b/backend/src/managers/ingest/ingest.service.ts @@ -0,0 +1,126 @@ +import { InjectQueue } from '@nestjs/bull'; +import { Injectable, Logger } from '@nestjs/common'; +import { fileTypeFromBuffer, FileTypeResult } from 'file-type'; +import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum'; +import { + AnimFileType, + FileType, + ImageFileType, + Mime2FileType +} from 'picsur-shared/dist/dto/mimes.dto'; +import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.enum'; +import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types'; +import { ParseFileType } from 'picsur-shared/dist/util/parse-mime'; +import { IsQOI } from 'qoi-img'; +import { ImageDBService } from '../../collections/image-db/image-db.service'; +import { ImageFileDBService } from '../../collections/image-db/image-file-db.service'; +import { UsrPreferenceDbService } from '../../collections/preference-db/usr-preference-db.service'; +import { EImageBackend } from '../../database/entities/images/image.entity'; +import { WebPInfo } from '../image/webpinfo/webpinfo'; +import type { ImageIngestJob, ImageIngestQueue } from './ingest.consumer'; + +@Injectable() +export class IngestService { + private readonly logger = new Logger(IngestService.name); + + constructor( + @InjectQueue('image-ingest') private readonly ingestQueue: ImageIngestQueue, + private readonly imagesService: ImageDBService, + private readonly imageFilesService: ImageFileDBService, + private readonly userPref: UsrPreferenceDbService, + ) {} + + public async uploadJob( + userid: string, + filename: string, + image: Buffer, + withDeleteKey: boolean, + ): AsyncFailable { + const fileType = await this.getFileTypeFromBuffer(image); + if (HasFailed(fileType)) return fileType; + + // Check if need to save orignal + const keepOriginal = await this.userPref.getBooleanPreference( + userid, + UsrPreference.KeepOriginal, + ); + if (HasFailed(keepOriginal)) return keepOriginal; + + // Strip extension from filename + const name = (() => { + const index = filename.lastIndexOf('.'); + if (index === -1) return filename; + return filename.substring(0, index); + })(); + + // Save unprocessed image to be processed by worker + const imageEntity = await this.imagesService.create( + userid, + name, + withDeleteKey, + ); + if (HasFailed(imageEntity)) return imageEntity; + + const imageFileEntity = await this.imageFilesService.setFile( + imageEntity.id, + ImageEntryVariant.INGEST, + image, + fileType.identifier, + ); + if (HasFailed(imageFileEntity)) return imageFileEntity; + + const job = await this.ingestQueue.add('image', { + imageID: imageEntity.id, + storeOriginal: keepOriginal, + }); + if (!job.id) return Fail(FT.Internal, undefined, 'Failed to queue job'); + + return job; + } + + public async uploadPromise( + userid: string, + filename: string, + image: Buffer, + withDeleteKey: boolean, + ): AsyncFailable { + const job = await this.uploadJob(userid, filename, image, withDeleteKey); + if (HasFailed(job)) return job; + + try { + const imageEntity: EImageBackend = await job.finished(); + return imageEntity; + } catch (e) { + return Fail(FT.Internal, 'Failed to process image', e); + } + } + + private async getFileTypeFromBuffer(image: Buffer): AsyncFailable { + const filetypeResult: FileTypeResult | undefined = await fileTypeFromBuffer( + image, + ); + + let mime: string | undefined; + if (filetypeResult === undefined) { + if (IsQOI(image)) mime = 'image/x-qoi'; + } else { + mime = filetypeResult.mime; + } + + if (mime === undefined) mime = 'other/unknown'; + + let filetype: string | undefined; + if (mime === 'image/webp') { + const header = await WebPInfo.from(image); + if (header.summary.isAnimated) filetype = AnimFileType.WEBP; + else filetype = ImageFileType.WEBP; + } + if (filetype === undefined) { + const parsed = Mime2FileType(mime); + if (HasFailed(parsed)) return parsed; + filetype = parsed; + } + + return ParseFileType(filetype); + } +} diff --git a/backend/src/routes/api/experiment/experiment.controller.ts b/backend/src/routes/api/experiment/experiment.controller.ts index 6c4cb6b..3305815 100644 --- a/backend/src/routes/api/experiment/experiment.controller.ts +++ b/backend/src/routes/api/experiment/experiment.controller.ts @@ -1,5 +1,5 @@ import { InjectQueue } from '@nestjs/bull'; -import { Controller, Get } from '@nestjs/common'; +import { Controller, Get, Logger } from '@nestjs/common'; import type { Queue } from 'bull'; import { NoPermissions } from '../../../decorators/permissions.decorator'; import { ReturnsAnything } from '../../../decorators/returns.decorator'; @@ -7,16 +7,28 @@ import { ReturnsAnything } from '../../../decorators/returns.decorator'; @Controller('api/experiment') @NoPermissions() export class ExperimentController { - constructor( + private readonly logger = new Logger(ExperimentController.name); + constructor( @InjectQueue('image-ingest') private readonly ingestQueue: Queue, - ) {} + ) { + this.logger.log('experiment consumer started'); + this.logger.error('experiment consumer started'); + console.log(this.logger); + } @Get() @ReturnsAnything() async testRoute(): Promise { - this.ingestQueue.add({ foo: Buffer.from("aaaaaheleool") }); + console.log('Create job'); + const job = await this.ingestQueue.add({ + foo: Buffer.from('aaaaaheleool'), + }); + console.log('Job created', job.id); + const result = await job.finished(); + + console.log('Job finished', result); return 'ok'; } diff --git a/backend/src/routes/api/experiment/experiment.module.ts b/backend/src/routes/api/experiment/experiment.module.ts index e6fc381..8fed456 100644 --- a/backend/src/routes/api/experiment/experiment.module.ts +++ b/backend/src/routes/api/experiment/experiment.module.ts @@ -1,12 +1,13 @@ import { Module } from '@nestjs/common'; -import { ConsumersModule } from '../../../consumers/consumers.module'; +import { PicsurLoggerModule } from '../../../logger/logger.module'; +import { IngestManagerModule } from '../../../managers/ingest/ingest.module'; import { ExperimentController } from './experiment.controller'; // This is comletely useless module, but is used for testing // TODO: remove when out of beta @Module({ - imports: [ConsumersModule], + imports: [IngestManagerModule, PicsurLoggerModule], controllers: [ExperimentController] }) export class ExperimentModule {} diff --git a/backend/src/routes/image/image-manage.controller.ts b/backend/src/routes/image/image-manage.controller.ts index be9422a..4e086b3 100644 --- a/backend/src/routes/image/image-manage.controller.ts +++ b/backend/src/routes/image/image-manage.controller.ts @@ -29,15 +29,19 @@ import { RequiredPermissions } from '../../decorators/permissions.decorator'; import { ReqUserID } from '../../decorators/request-user.decorator'; -import { Returns } from '../../decorators/returns.decorator'; +import { Returns, ReturnsAnything } from '../../decorators/returns.decorator'; import { ImageManagerService } from '../../managers/image/image.service'; +import { IngestService } from '../../managers/ingest/ingest.service'; import { GetNextAsync } from '../../util/iterator'; @Controller('api/image') @RequiredPermissions(Permission.ImageUpload) export class ImageManageController { private readonly logger = new Logger(ImageManageController.name); - constructor(private readonly imagesService: ImageManagerService) {} + constructor( + private readonly imagesService: ImageManagerService, + private readonly ingestService: IngestService, + ) {} @Post('upload') @Returns(ImageUploadResponse) @@ -54,10 +58,10 @@ export class ImageManageController { buffer = await file.toBuffer(); } catch (e) { throw Fail(FT.Internal, e); - }; + } const image = ThrowIfFailed( - await this.imagesService.upload( + await this.ingestService.uploadPromise( userid, file.filename, buffer, @@ -68,6 +72,33 @@ export class ImageManageController { return image; } + @Post('upload/bulk') + @ReturnsAnything() + @Throttle(20) + async uploadImages( + @PostFiles() multipart: FileIterator, + @ReqUserID() userid: string, + @HasPermission(Permission.ImageDeleteKey) withDeleteKey: boolean, + ): Promise { + let ids: string[] = []; + for await (const file of multipart) { + const buffer = await file.toBuffer(); + const filename = file.filename; + + // const id = ThrowIfFailed( + // await this.ingressDB.uploadFile(filename, buffer), + // ); + // ids.push(id); + } + if (ids.length === 0) { + throw Fail(FT.BadRequest, 'No files uploaded'); + } + + console.log(ids); + + return; + } + @Post('list') @RequiredPermissions(Permission.ImageManage) @Returns(ImageListResponse) diff --git a/backend/src/routes/image/image.module.ts b/backend/src/routes/image/image.module.ts index dd61c51..fb43c69 100644 --- a/backend/src/routes/image/image.module.ts +++ b/backend/src/routes/image/image.module.ts @@ -1,12 +1,20 @@ import { Module } from '@nestjs/common'; +import { IngestFileDbModule } from '../../collections/ingest-file-db/ingest-file-db.module'; import { UserDbModule } from '../../collections/user-db/user-db.module'; import { DecoratorsModule } from '../../decorators/decorators.module'; import { ImageManagerModule } from '../../managers/image/image.module'; +import { IngestManagerModule } from '../../managers/ingest/ingest.module'; import { ImageManageController } from './image-manage.controller'; import { ImageController } from './image.controller'; @Module({ - imports: [ImageManagerModule, UserDbModule, DecoratorsModule], + imports: [ + ImageManagerModule, + UserDbModule, + IngestFileDbModule, + IngestManagerModule, + DecoratorsModule, + ], controllers: [ImageController, ImageManageController], }) export class ImageModule {} diff --git a/frontend/src/app/routes/settings/sharex/settings-sharex.component.ts b/frontend/src/app/routes/settings/sharex/settings-sharex.component.ts index cf8992e..2e106e8 100644 --- a/frontend/src/app/routes/settings/sharex/settings-sharex.component.ts +++ b/frontend/src/app/routes/settings/sharex/settings-sharex.component.ts @@ -64,7 +64,7 @@ export class SettingsShareXComponent implements OnInit { const ext = FileType2Ext(this.selectedFormat); if (HasFailed(ext)) { - this.logger.error(ext.print()); + ext.print(this.logger); } const sharexConfig = BuildShareX( diff --git a/frontend/src/app/util/error-manager/error.service.ts b/frontend/src/app/util/error-manager/error.service.ts index b6bbcb9..93e3e26 100644 --- a/frontend/src/app/util/error-manager/error.service.ts +++ b/frontend/src/app/util/error-manager/error.service.ts @@ -15,11 +15,7 @@ export class ErrorService { ) {} public showFailure(error: Failure, logger: Logger): void { - if (error.isImportant()) { - logger.error(error.print()); - } else { - logger.warn(error.print()); - } + error.print(logger); this.snackbar.showSnackBar( error.getReason(), diff --git a/shared/src/dto/image-entry-variant.enum.ts b/shared/src/dto/image-entry-variant.enum.ts index e9c77b6..dea38dd 100644 --- a/shared/src/dto/image-entry-variant.enum.ts +++ b/shared/src/dto/image-entry-variant.enum.ts @@ -1,4 +1,5 @@ export enum ImageEntryVariant { ORIGINAL = 'original', MASTER = 'master', + INGEST = 'ingest', } diff --git a/shared/src/types/failable.ts b/shared/src/types/failable.ts index 1260876..eb521a6 100644 --- a/shared/src/types/failable.ts +++ b/shared/src/types/failable.ts @@ -21,6 +21,12 @@ export enum FT { Network = 'network', } +interface ILogger { + error: (message: string) => void; + warn: (message: string) => void; + debug: (message: string) => void; +} + interface FTProp { important: boolean; code: number; @@ -142,10 +148,42 @@ export class Failure { return FTProps[this.type].important; } - print(): string { - return `${this.getName()}: ${this.getReason()}\n(${ - this.debugMessage - })\n${this.getStack()}`; + print( + logger: ILogger, + options?: { + notImportant?: boolean; + prefix?: string; + }, + ): void { + const message = this.getReason(); + const logmessage = + message + (this.getDebugMessage() ? ' - ' + this.getDebugMessage() : ''); + + const prefix = options?.prefix ? options.prefix + ' ' : ''; + const logline = `${prefix}${this.getName()}: ${logmessage}`; + + if (this.isImportant() && options?.notImportant !== true) { + logger.error(logline); + const stack = this.getStack(); + if (stack) { + logger.debug(stack); + } + } else { + logger.warn(logline); + } + } + + toString(): string { + return ( + `${this.getName()}: ${this.getReason()} - (${this.debugMessage})` + + (this.isImportant() ? '\n' + this.stack : '') + ); + } + + toError(): Error { + const error = new Error(); + (error as any).message = this; + return error; } static deserialize(data: any): Failure { @@ -251,10 +289,10 @@ export function ThrowIfFailed(failable: Failable): V { export function FallbackIfFailed( failable: Failable, fallback: V, - logger?: { warn: (...args: any) => any }, + logger?: ILogger, ): V { if (HasFailed(failable)) { - if (logger) logger.warn(failable.print()); + if (logger) failable.print(logger, { notImportant: true }); return fallback; }