cmon, why the logger gotta be like this

backend/src/app.module.ts

@@ -7,12 +7,12 @@ import { IncomingMessage, ServerResponse } from 'http';
 import { BullConfigService } from './config/early/bull.config.service';
 import { EarlyConfigModule } from './config/early/early-config.module';
 import { ServeStaticConfigService } from './config/early/serve-static.config.service';
-import { ConsumersModule } from './consumers/consumers.module';
 import { DatabaseModule } from './database/database.module';
 import { PicsurLayersModule } from './layers/PicsurLayers.module';
 import { PicsurLoggerModule } from './logger/logger.module';
 import { AuthManagerModule } from './managers/auth/auth.module';
 import { DemoManagerModule } from './managers/demo/demo.module';
+import { IngestManagerModule } from './managers/ingest/ingest.module';
 import { UsageManagerModule } from './managers/usage/usage.module';
 import { PicsurRoutesModule } from './routes/routes.module';
 
@@ -59,7 +59,7 @@ const imageCorsOverride = (
     DemoManagerModule,
     PicsurRoutesModule,
     PicsurLayersModule,
-    ConsumersModule,
+    IngestManagerModule,
   ],
 })
 export class AppModule implements NestModule {

backend/src/collections/image-db/image-file-db.service.ts

@@ -1,7 +1,7 @@
 import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
-import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types';
+import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
 import { LessThan, Repository } from 'typeorm';
 import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
 import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
@@ -57,6 +57,40 @@ export class ImageFileDBService {
     }
   }
 
+  public async migrateFile(
+    imageId: string,
+    sourceVariant: ImageEntryVariant,
+    targetVariant: ImageEntryVariant,
+  ): AsyncFailable<EImageFileBackend> {
+    try {
+      const sourceFile = await this.getFile(imageId, sourceVariant);
+      if (HasFailed(sourceFile)) return sourceFile;
+
+      sourceFile.variant = targetVariant;
+      return await this.imageFileRepo.save(sourceFile);
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
+  public async deleteFile(
+    imageId: string,
+    variant: ImageEntryVariant,
+  ): AsyncFailable<EImageFileBackend> {
+    try {
+      const found = await this.imageFileRepo.findOne({
+        where: { image_id: imageId, variant: variant },
+      });
+
+      if (!found) return Fail(FT.NotFound, 'Image not found');
+
+      await this.imageFileRepo.delete({ image_id: imageId, variant: variant });
+      return found;
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
   // This is useful because you dont have to pull the whole image file
   public async getFileTypes(
     imageId: string,

backend/src/collections/ingest-file-db/ingest-file-db.module.ts

@@ -0,0 +1,11 @@
+import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
+import { EIngestFileBackend } from '../../database/entities/ingest-file.entity';
+import { IngestFileDbService } from './ingest-file-db.service';
+
+@Module({
+  imports: [TypeOrmModule.forFeature([EIngestFileBackend])],
+  providers: [IngestFileDbService],
+  exports: [IngestFileDbService],
+})
+export class IngestFileDbModule {}

backend/src/collections/ingest-file-db/ingest-file-db.service.ts

@@ -0,0 +1,32 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types';
+import { Repository } from 'typeorm';
+import { EIngestFileBackend } from '../../database/entities/ingest-file.entity';
+
+@Injectable()
+export class IngestFileDbService {
+  private readonly logger = new Logger(IngestFileDbService.name);
+
+  constructor(
+    @InjectRepository(EIngestFileBackend)
+    private readonly ingressFileRepo: Repository<EIngestFileBackend>,
+  ) {}
+
+  public async uploadFile(
+    filename: string,
+    file: Buffer,
+  ): AsyncFailable<string> {
+    const ingressFile = new EIngestFileBackend();
+    ingressFile.filename = filename;
+    ingressFile.data = file;
+
+    try {
+      await this.ingressFileRepo.save(ingressFile);
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+
+    return ingressFile.id;
+  }
+}

backend/src/collections/ingress-file-db/ingress-file-db.module.ts

@@ -1,11 +0,0 @@
-import { Module } from '@nestjs/common';
-import { TypeOrmModule } from '@nestjs/typeorm';
-import { EIngressFileBackend } from '../../database/entities/ingress-file.entity';
-import { IngressFileDbService } from './ingress-file-db.service';
-
-@Module({
-  imports: [TypeOrmModule.forFeature([EIngressFileBackend])],
-  providers: [IngressFileDbService],
-  exports: [IngressFileDbService],
-})
-export class IngressFileDbModule {}

backend/src/collections/ingress-file-db/ingress-file-db.service.ts

@@ -1,14 +0,0 @@
-import { Injectable, Logger } from '@nestjs/common';
-import { InjectRepository } from '@nestjs/typeorm';
-import { Repository } from 'typeorm';
-import { EIngressFileBackend } from '../../database/entities/ingress-file.entity';
-
-@Injectable()
-export class IngressFileDbService {
-  private readonly logger = new Logger(IngressFileDbService.name);
-
-  constructor(
-    @InjectRepository(EIngressFileBackend)
-    private readonly ingressFileRepo: Repository<EIngressFileBackend>,
-  ) {}
-}

backend/src/config/early/bull.config.service.ts

@@ -17,7 +17,16 @@ export class BullConfigService implements SharedBullConfigurationFactory {
       },
       defaultJobOptions: {
         attempts: 3,
-        removeOnFail: true,
+        backoff: {
+          delay: 500,
+          type: 'fixed',
+        },
+        removeOnFail: {
+          age: 1000 * 60 * 60 * 24 * 7, // 7 days
+        },
+        removeOnComplete: {
+          age: 1000 * 60 * 60 * 24 * 7, // 7 days
+        },
       },
     };
     return options;

backend/src/config/late/info.config.service.ts

@@ -14,7 +14,7 @@ export class InfoConfigService {
       SysPreference.HostOverride,
     );
     if (HasFailed(hostname)) {
-      this.logger.warn(hostname.print());
+      hostname.print(this.logger);
       return undefined;
     }
 

backend/src/consumers/consumers.module.ts

@@ -1,14 +0,0 @@
-import { BullModule } from '@nestjs/bull';
-import { Module } from '@nestjs/common';
-import { IngestConsumer } from './ingest.consumer';
-
-@Module({
-  imports: [
-    BullModule.registerQueue({
-      name: 'image-ingest',
-    }),
-  ],
-  providers: [IngestConsumer],
-  exports: [BullModule],
-})
-export class ConsumersModule {}

backend/src/consumers/ingest.consumer.ts

@@ -1,18 +0,0 @@
-import { OnQueueError, Process, Processor } from '@nestjs/bull';
-import { Logger } from '@nestjs/common';
-import type { Job } from 'bull';
-
-@Processor('image-ingest')
-export class IngestConsumer {
-  private readonly logger = new Logger(IngestConsumer.name);
-
-  @Process()
-  async processJob(job: Job) {
-    console.log('processJob', job);
-  }
-
-  @OnQueueError()
-  async handleError(error: any) {
-    this.logger.error(error);
-  }
-}

backend/src/database/entities/index.ts

@@ -2,7 +2,7 @@ import { EApiKeyBackend } from './apikey.entity';
 import { EImageDerivativeBackend } from './images/image-derivative.entity';
 import { EImageFileBackend } from './images/image-file.entity';
 import { EImageBackend } from './images/image.entity';
-import { EIngressFileBackend } from './ingress-file.entity';
+import { EIngestFileBackend } from './ingest-file.entity';
 import { ESysPreferenceBackend } from './system/sys-preference.entity';
 import { ESystemStateBackend } from './system/system-state.entity';
 import { EUsrPreferenceBackend } from './system/usr-preference.entity';
@@ -19,5 +19,5 @@ export const EntityList = [
   EUsrPreferenceBackend,
   EApiKeyBackend,
   ESystemStateBackend,
-  EIngressFileBackend,
+  EIngestFileBackend,
 ];

backend/src/database/entities/ingest-file.entity.ts

@@ -1,7 +1,7 @@
 import { Column, Entity, PrimaryGeneratedColumn } from 'typeorm';
 
 @Entity()
-export class EIngressFileBackend {
+export class EIngestFileBackend {
   @PrimaryGeneratedColumn('uuid')
   id: string;
 

backend/src/decorators/multipart/postfiles.pipe.ts

@@ -26,7 +26,7 @@ export class MultiPartPipe implements PipeTransform {
   ) {
     const filesLimit = typeof data === 'number' ? data : undefined;
 
-    if (!request.isMultipart()) throw Fail(FT.UsrValidation, 'Invalid file');
+    if (!request.isMultipart()) throw Fail(FT.UsrValidation, 'Invalid files');
 
     const files = request.files({
       limits: this.multipartConfigService.getLimits(filesLimit),

backend/src/layers/exception/exception.filter.ts

@@ -6,7 +6,7 @@ import {
   Logger,
   MethodNotAllowedException,
   NotFoundException,
-  UnauthorizedException,
+  UnauthorizedException
 } from '@nestjs/common';
 import { FastifyReply, FastifyRequest } from 'fastify';
 import { ApiErrorResponse } from 'picsur-shared/dist/dto/api/api.dto';
@@ -14,7 +14,7 @@ import {
   Fail,
   Failure,
   FT,
-  IsFailure,
+  IsFailure
 } from 'picsur-shared/dist/types/failable';
 
 // This will catch any exception that is made in any request
@@ -39,23 +39,7 @@ export class MainExceptionFilter implements ExceptionFilter {
     const status = exception.getCode();
     const type = exception.getType();
 
-    const message = exception.getReason();
-    const logmessage =
-      message +
-      (exception.getDebugMessage() ? ' - ' + exception.getDebugMessage() : '');
-
-    if (exception.isImportant()) {
-      MainExceptionFilter.logger.error(
-        `${traceString} ${exception.getName()}: ${logmessage}`,
-      );
-      if (exception.getStack()) {
-        MainExceptionFilter.logger.debug(exception.getStack());
-      }
-    } else {
-      MainExceptionFilter.logger.warn(
-        `${traceString} ${exception.getName()}: ${logmessage}`,
-      );
-    }
+    exception.print(MainExceptionFilter.logger, { prefix: traceString });
 
     const toSend: ApiErrorResponse = {
       success: false,
@@ -65,7 +49,7 @@ export class MainExceptionFilter implements ExceptionFilter {
 
       data: {
         type,
-        message,
+        message: exception.getReason(),
       },
     };
 

backend/src/main.ts

@@ -4,7 +4,7 @@ import fastifyReplyFrom from '@fastify/reply-from';
 import { NestFactory } from '@nestjs/core';
 import {
   FastifyAdapter,
-  NestFastifyApplication,
+  NestFastifyApplication
 } from '@nestjs/platform-fastify';
 import { AppModule } from './app.module';
 import { HostConfigService } from './config/early/host.config.service';
@@ -39,14 +39,17 @@ async function bootstrap() {
     fastifyAdapter,
     {
       bufferLogs: isProduction,
+      autoFlushLogs: true,
     },
   );
 
   // Configure logger
-  app.useLogger(app.get(PicsurLoggerService));
-
+  const logger = app.get(PicsurLoggerService)
+  app.useLogger(logger);
   app.flushLogs();
 
+  console.log(logger);
+
   app.useGlobalFilters(app.get(MainExceptionFilter));
   app.useGlobalInterceptors(app.get(SuccessInterceptor));
   app.useGlobalPipes(app.get(ZodValidationPipe));

backend/src/managers/image/image-converter.service.ts

@@ -3,7 +3,7 @@ import ms from 'ms';
 import { ImageRequestParams } from 'picsur-shared/dist/dto/api/image.dto';
 import {
   FileType,
-  SupportedFileTypeCategory,
+  SupportedFileTypeCategory
 } from 'picsur-shared/dist/dto/mimes.dto';
 import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
 import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
@@ -122,16 +122,4 @@ export class ImageConverterService {
       filetype: targetFiletype.identifier,
     };
   }
-
-  private async convertAnimation(
-    image: Buffer,
-    targetFiletype: FileType,
-    options: ImageRequestParams,
-  ): AsyncFailable<ImageResult> {
-    // Apng and gif are stored as is for now
-    return {
-      image: image,
-      filetype: targetFiletype.identifier,
-    };
-  }
 }

backend/src/managers/image/image-processor.service.ts

@@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common';
 import {
   FileType,
   ImageFileType,
-  SupportedFileTypeCategory,
+  SupportedFileTypeCategory
 } from 'picsur-shared/dist/dto/mimes.dto';
 
 import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';

backend/src/managers/image/image.module.ts

@@ -19,7 +19,7 @@ import { ImageManagerService } from './image.service';
     ImageProcessorService,
     ImageConverterService,
   ],
-  exports: [ImageManagerService],
+  exports: [ImageManagerService, ImageConverterService],
 })
 export class ImageManagerModule implements OnModuleInit {
   private readonly logger = new Logger(ImageManagerModule.name);
@@ -31,7 +31,7 @@ export class ImageManagerModule implements OnModuleInit {
   ) {}
 
   async onModuleInit() {
-    await this.imageManagerCron()
+    await this.imageManagerCron();
   }
 
   @Interval(1000 * 60)
@@ -57,7 +57,7 @@ export class ImageManagerModule implements OnModuleInit {
 
     const result = await this.imageFileDB.cleanupDerivatives(after_ms / 1000);
     if (HasFailed(result)) {
-      this.logger.warn(result.print());
+      result.print(this.logger);
     }
 
     if (result > 0) this.logger.log(`Cleaned up ${result} derivatives`);
@@ -67,7 +67,8 @@ export class ImageManagerModule implements OnModuleInit {
     const cleanedUp = await this.imageDB.cleanupExpired();
 
     if (HasFailed(cleanedUp)) {
-      this.logger.warn(cleanedUp.print());
+      cleanedUp.print(this.logger);
+      return;
     }
 
     if (cleanedUp > 0)

backend/src/managers/ingest/ingest.consumer.ts

@@ -0,0 +1,161 @@
+import {
+  InjectQueue,
+  OnQueueError,
+  OnQueueFailed,
+  Process,
+  Processor
+} from '@nestjs/bull';
+import { Logger } from '@nestjs/common';
+import type { Job, Queue } from 'bull';
+import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
+import {
+  FileType,
+  ImageFileType,
+  SupportedFileTypeCategory
+} from 'picsur-shared/dist/dto/mimes.dto';
+import {
+  AsyncFailable,
+  Fail,
+  FT,
+  HasFailed,
+  IsFailure,
+  ThrowIfFailed
+} from 'picsur-shared/dist/types';
+import { ParseFileType } from 'picsur-shared/dist/util/parse-mime';
+import { ImageDBService } from '../../collections/image-db/image-db.service';
+import { ImageFileDBService } from '../../collections/image-db/image-file-db.service';
+import { EImageBackend } from '../../database/entities/images/image.entity';
+import { ImageConverterService } from '../image/image-converter.service';
+import { ImageResult } from '../image/imageresult';
+
+interface ImageIngestJobData {
+  imageID: string;
+  storeOriginal: boolean;
+}
+export type ImageIngestQueue = Queue<ImageIngestJobData>;
+export type ImageIngestJob = Job<ImageIngestJobData>;
+
+@Processor('image-ingest')
+export class IngestConsumer {
+  private readonly logger = new Logger(IngestConsumer.name);
+  private i = 0;
+
+  constructor(
+    @InjectQueue('image-ingest') private readonly ingestQueue: Queue,
+    private readonly imagesService: ImageDBService,
+    private readonly imageFilesService: ImageFileDBService,
+    private readonly imageConverter: ImageConverterService,
+  ) {
+    this.logger.log('Ingest consumer started');
+    this.logger.error('Ingest consumer started');
+  }
+
+  // @Process('group')
+  // async processJob(job: Job<GroupIngestJob>) {
+  //   console.log('Received', job.data);
+  //   await new Promise((resolve) => setTimeout(resolve, 4000));
+  //   console.log('Done');
+  //   return 'big chungus';
+  // }
+
+  @Process('image')
+  async processImage(job: ImageIngestJob): Promise<EImageBackend> {
+    const { imageID, storeOriginal } = job.data;
+
+    job.failedReason = 'Not implemented';
+
+    if (this.i === 0) {
+      throw Fail(FT.Internal, undefined, 'oops');
+    }
+
+    // Already start the query for the image, we only need it when returning
+    const imagePromise = this.imagesService.findOne(imageID, undefined);
+
+    this.logger.verbose(
+      `Ingesting image ${imageID} and store original: ${storeOriginal}`,
+    );
+
+    const ingestFile = ThrowIfFailed(
+      await this.imageFilesService.getFile(imageID, ImageEntryVariant.INGEST),
+    );
+
+    const ingestFiletype = ThrowIfFailed(ParseFileType(ingestFile.filetype));
+
+    const processed = ThrowIfFailed(
+      await this.process(ingestFile.data, ingestFiletype),
+    );
+
+    const masterPromise = this.imageFilesService.setFile(
+      imageID,
+      ImageEntryVariant.MASTER,
+      processed.image,
+      processed.filetype,
+    );
+
+    const originalPromise = storeOriginal
+      ? this.imageFilesService.migrateFile(
+          imageID,
+          ImageEntryVariant.INGEST,
+          ImageEntryVariant.ORIGINAL,
+        )
+      : this.imageFilesService.deleteFile(imageID, ImageEntryVariant.INGEST);
+
+    const results = await Promise.all([masterPromise, originalPromise]);
+    results.map((r) => ThrowIfFailed(r));
+
+    const image = ThrowIfFailed(await imagePromise);
+
+    this.logger.verbose(`Ingested image ${imageID}`);
+
+    return image;
+  }
+
+  public async process(
+    image: Buffer,
+    filetype: FileType,
+  ): AsyncFailable<ImageResult> {
+    if (filetype.category === SupportedFileTypeCategory.Image) {
+      return await this.processStill(image, filetype);
+    } else if (filetype.category === SupportedFileTypeCategory.Animation) {
+      return await this.processAnimation(image, filetype);
+    } else {
+      return Fail(FT.SysValidation, 'Unsupported mime type');
+    }
+  }
+
+  private async processStill(
+    image: Buffer,
+    filetype: FileType,
+  ): AsyncFailable<ImageResult> {
+    const outputFileType = ParseFileType(ImageFileType.QOI);
+    if (HasFailed(outputFileType)) return outputFileType;
+
+    return this.imageConverter.convert(image, filetype, outputFileType, {});
+  }
+
+  private async processAnimation(
+    image: Buffer,
+    filetype: FileType,
+  ): AsyncFailable<ImageResult> {
+    // Webps and gifs are stored as is for now
+    return {
+      image: image,
+      filetype: filetype.identifier,
+    };
+  }
+
+  @OnQueueError()
+  async handleError(error: any) {
+    if (IsFailure(error)) error.print(this.logger);
+    else this.logger.error(error);
+  }
+
+  @OnQueueFailed()
+  async handleFailed(job: Job, error: any) {
+    if (IsFailure(error))
+      error.print(this.logger, {
+        prefix: `[JOB ${job.id}]`,
+      });
+    else this.logger.error(error);
+  }
+}

backend/src/managers/ingest/ingest.module.ts

@@ -0,0 +1,21 @@
+import { BullModule } from '@nestjs/bull';
+import { Module } from '@nestjs/common';
+import { ImageDBModule } from '../../collections/image-db/image-db.module';
+import { PreferenceDbModule } from '../../collections/preference-db/preference-db.module';
+import { ImageManagerModule } from '../image/image.module';
+import { IngestConsumer } from './ingest.consumer';
+import { IngestService } from './ingest.service';
+
+@Module({
+  imports: [
+    ImageDBModule,
+    ImageManagerModule,
+    PreferenceDbModule,
+    BullModule.registerQueue({
+      name: 'image-ingest',
+    }),
+  ],
+  providers: [IngestConsumer, IngestService],
+  exports: [BullModule, IngestService],
+})
+export class IngestManagerModule {}

backend/src/managers/ingest/ingest.service.ts

@@ -0,0 +1,126 @@
+import { InjectQueue } from '@nestjs/bull';
+import { Injectable, Logger } from '@nestjs/common';
+import { fileTypeFromBuffer, FileTypeResult } from 'file-type';
+import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
+import {
+  AnimFileType,
+  FileType,
+  ImageFileType,
+  Mime2FileType
+} from 'picsur-shared/dist/dto/mimes.dto';
+import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.enum';
+import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
+import { ParseFileType } from 'picsur-shared/dist/util/parse-mime';
+import { IsQOI } from 'qoi-img';
+import { ImageDBService } from '../../collections/image-db/image-db.service';
+import { ImageFileDBService } from '../../collections/image-db/image-file-db.service';
+import { UsrPreferenceDbService } from '../../collections/preference-db/usr-preference-db.service';
+import { EImageBackend } from '../../database/entities/images/image.entity';
+import { WebPInfo } from '../image/webpinfo/webpinfo';
+import type { ImageIngestJob, ImageIngestQueue } from './ingest.consumer';
+
+@Injectable()
+export class IngestService {
+  private readonly logger = new Logger(IngestService.name);
+
+  constructor(
+    @InjectQueue('image-ingest') private readonly ingestQueue: ImageIngestQueue,
+    private readonly imagesService: ImageDBService,
+    private readonly imageFilesService: ImageFileDBService,
+    private readonly userPref: UsrPreferenceDbService,
+  ) {}
+
+  public async uploadJob(
+    userid: string,
+    filename: string,
+    image: Buffer,
+    withDeleteKey: boolean,
+  ): AsyncFailable<ImageIngestJob> {
+    const fileType = await this.getFileTypeFromBuffer(image);
+    if (HasFailed(fileType)) return fileType;
+
+    // Check if need to save orignal
+    const keepOriginal = await this.userPref.getBooleanPreference(
+      userid,
+      UsrPreference.KeepOriginal,
+    );
+    if (HasFailed(keepOriginal)) return keepOriginal;
+
+    // Strip extension from filename
+    const name = (() => {
+      const index = filename.lastIndexOf('.');
+      if (index === -1) return filename;
+      return filename.substring(0, index);
+    })();
+
+    // Save unprocessed image to be processed by worker
+    const imageEntity = await this.imagesService.create(
+      userid,
+      name,
+      withDeleteKey,
+    );
+    if (HasFailed(imageEntity)) return imageEntity;
+
+    const imageFileEntity = await this.imageFilesService.setFile(
+      imageEntity.id,
+      ImageEntryVariant.INGEST,
+      image,
+      fileType.identifier,
+    );
+    if (HasFailed(imageFileEntity)) return imageFileEntity;
+
+    const job = await this.ingestQueue.add('image', {
+      imageID: imageEntity.id,
+      storeOriginal: keepOriginal,
+    });
+    if (!job.id) return Fail(FT.Internal, undefined, 'Failed to queue job');
+
+    return job;
+  }
+
+  public async uploadPromise(
+    userid: string,
+    filename: string,
+    image: Buffer,
+    withDeleteKey: boolean,
+  ): AsyncFailable<EImageBackend> {
+    const job = await this.uploadJob(userid, filename, image, withDeleteKey);
+    if (HasFailed(job)) return job;
+
+    try {
+      const imageEntity: EImageBackend = await job.finished();
+      return imageEntity;
+    } catch (e) {
+      return Fail(FT.Internal, 'Failed to process image', e);
+    }
+  }
+
+  private async getFileTypeFromBuffer(image: Buffer): AsyncFailable<FileType> {
+    const filetypeResult: FileTypeResult | undefined = await fileTypeFromBuffer(
+      image,
+    );
+
+    let mime: string | undefined;
+    if (filetypeResult === undefined) {
+      if (IsQOI(image)) mime = 'image/x-qoi';
+    } else {
+      mime = filetypeResult.mime;
+    }
+
+    if (mime === undefined) mime = 'other/unknown';
+
+    let filetype: string | undefined;
+    if (mime === 'image/webp') {
+      const header = await WebPInfo.from(image);
+      if (header.summary.isAnimated) filetype = AnimFileType.WEBP;
+      else filetype = ImageFileType.WEBP;
+    }
+    if (filetype === undefined) {
+      const parsed = Mime2FileType(mime);
+      if (HasFailed(parsed)) return parsed;
+      filetype = parsed;
+    }
+
+    return ParseFileType(filetype);
+  }
+}

backend/src/routes/api/experiment/experiment.controller.ts

@@ -1,5 +1,5 @@
 import { InjectQueue } from '@nestjs/bull';
-import { Controller, Get } from '@nestjs/common';
+import { Controller, Get, Logger } from '@nestjs/common';
 import type { Queue } from 'bull';
 import { NoPermissions } from '../../../decorators/permissions.decorator';
 import { ReturnsAnything } from '../../../decorators/returns.decorator';
@@ -7,16 +7,28 @@ import { ReturnsAnything } from '../../../decorators/returns.decorator';
 @Controller('api/experiment')
 @NoPermissions()
 export class ExperimentController {
-  constructor(
+  private readonly logger = new Logger(ExperimentController.name);
 
+  constructor(
     @InjectQueue('image-ingest') private readonly ingestQueue: Queue,
-  ) {}
+  ) {
+    this.logger.log('experiment consumer started');
+    this.logger.error('experiment consumer started');
+    console.log(this.logger);
+  }
 
   @Get()
   @ReturnsAnything()
   async testRoute(): Promise<any> {
-    this.ingestQueue.add({ foo: Buffer.from("aaaaaheleool") });
+    console.log('Create job');
+    const job = await this.ingestQueue.add({
+      foo: Buffer.from('aaaaaheleool'),
+    });
+    console.log('Job created', job.id);
 
+    const result = await job.finished();
+
+    console.log('Job finished', result);
 
     return 'ok';
   }

backend/src/routes/api/experiment/experiment.module.ts

@@ -1,12 +1,13 @@
 import { Module } from '@nestjs/common';
-import { ConsumersModule } from '../../../consumers/consumers.module';
+import { PicsurLoggerModule } from '../../../logger/logger.module';
+import { IngestManagerModule } from '../../../managers/ingest/ingest.module';
 import { ExperimentController } from './experiment.controller';
 
 // This is comletely useless module, but is used for testing
 // TODO: remove when out of beta
 
 @Module({
-  imports: [ConsumersModule],
+  imports: [IngestManagerModule, PicsurLoggerModule],
   controllers: [ExperimentController]
 })
 export class ExperimentModule {}

backend/src/routes/image/image-manage.controller.ts

@@ -29,15 +29,19 @@ import {
   RequiredPermissions
 } from '../../decorators/permissions.decorator';
 import { ReqUserID } from '../../decorators/request-user.decorator';
-import { Returns } from '../../decorators/returns.decorator';
+import { Returns, ReturnsAnything } from '../../decorators/returns.decorator';
 import { ImageManagerService } from '../../managers/image/image.service';
+import { IngestService } from '../../managers/ingest/ingest.service';
 import { GetNextAsync } from '../../util/iterator';
 @Controller('api/image')
 @RequiredPermissions(Permission.ImageUpload)
 export class ImageManageController {
   private readonly logger = new Logger(ImageManageController.name);
 
-  constructor(private readonly imagesService: ImageManagerService) {}
+  constructor(
+    private readonly imagesService: ImageManagerService,
+    private readonly ingestService: IngestService,
+  ) {}
 
   @Post('upload')
   @Returns(ImageUploadResponse)
@@ -54,10 +58,10 @@ export class ImageManageController {
       buffer = await file.toBuffer();
     } catch (e) {
       throw Fail(FT.Internal, e);
-    };
+    }
 
     const image = ThrowIfFailed(
-      await this.imagesService.upload(
+      await this.ingestService.uploadPromise(
         userid,
         file.filename,
         buffer,
@@ -68,6 +72,33 @@ export class ImageManageController {
     return image;
   }
 
+  @Post('upload/bulk')
+  @ReturnsAnything()
+  @Throttle(20)
+  async uploadImages(
+    @PostFiles() multipart: FileIterator,
+    @ReqUserID() userid: string,
+    @HasPermission(Permission.ImageDeleteKey) withDeleteKey: boolean,
+  ): Promise<any> {
+    let ids: string[] = [];
+    for await (const file of multipart) {
+      const buffer = await file.toBuffer();
+      const filename = file.filename;
+
+      // const id = ThrowIfFailed(
+      //   await this.ingressDB.uploadFile(filename, buffer),
+      // );
+      // ids.push(id);
+    }
+    if (ids.length === 0) {
+      throw Fail(FT.BadRequest, 'No files uploaded');
+    }
+
+    console.log(ids);
+
+    return;
+  }
+
   @Post('list')
   @RequiredPermissions(Permission.ImageManage)
   @Returns(ImageListResponse)

backend/src/routes/image/image.module.ts

@@ -1,12 +1,20 @@
 import { Module } from '@nestjs/common';
+import { IngestFileDbModule } from '../../collections/ingest-file-db/ingest-file-db.module';
 import { UserDbModule } from '../../collections/user-db/user-db.module';
 import { DecoratorsModule } from '../../decorators/decorators.module';
 import { ImageManagerModule } from '../../managers/image/image.module';
+import { IngestManagerModule } from '../../managers/ingest/ingest.module';
 import { ImageManageController } from './image-manage.controller';
 import { ImageController } from './image.controller';
 
 @Module({
-  imports: [ImageManagerModule, UserDbModule, DecoratorsModule],
+  imports: [
+    ImageManagerModule,
+    UserDbModule,
+    IngestFileDbModule,
+    IngestManagerModule,
+    DecoratorsModule,
+  ],
   controllers: [ImageController, ImageManageController],
 })
 export class ImageModule {}

frontend/src/app/routes/settings/sharex/settings-sharex.component.ts

@@ -64,7 +64,7 @@ export class SettingsShareXComponent implements OnInit {
 
     const ext = FileType2Ext(this.selectedFormat);
     if (HasFailed(ext)) {
-      this.logger.error(ext.print());
+      ext.print(this.logger);
     }
 
     const sharexConfig = BuildShareX(

frontend/src/app/util/error-manager/error.service.ts

@@ -15,11 +15,7 @@ export class ErrorService {
   ) {}
 
   public showFailure(error: Failure, logger: Logger): void {
-    if (error.isImportant()) {
-      logger.error(error.print());
-    } else {
-      logger.warn(error.print());
-    }
+    error.print(logger);
 
     this.snackbar.showSnackBar(
       error.getReason(),

shared/src/dto/image-entry-variant.enum.ts

@@ -1,4 +1,5 @@
 export enum ImageEntryVariant {
   ORIGINAL = 'original',
   MASTER = 'master',
+  INGEST = 'ingest',
 }

shared/src/types/failable.ts

@@ -21,6 +21,12 @@ export enum FT {
   Network = 'network',
 }
 
+interface ILogger {
+  error: (message: string) => void;
+  warn: (message: string) => void;
+  debug: (message: string) => void;
+}
+
 interface FTProp {
   important: boolean;
   code: number;
@@ -142,10 +148,42 @@ export class Failure {
     return FTProps[this.type].important;
   }
 
-  print(): string {
-    return `${this.getName()}: ${this.getReason()}\n(${
-      this.debugMessage
-    })\n${this.getStack()}`;
+  print(
+    logger: ILogger,
+    options?: {
+      notImportant?: boolean;
+      prefix?: string;
+    },
+  ): void {
+    const message = this.getReason();
+    const logmessage =
+      message + (this.getDebugMessage() ? ' - ' + this.getDebugMessage() : '');
+
+    const prefix = options?.prefix ? options.prefix + ' ' : '';
+    const logline = `${prefix}${this.getName()}: ${logmessage}`;
+
+    if (this.isImportant() && options?.notImportant !== true) {
+      logger.error(logline);
+      const stack = this.getStack();
+      if (stack) {
+        logger.debug(stack);
+      }
+    } else {
+      logger.warn(logline);
+    }
+  }
+
+  toString(): string {
+    return (
+      `${this.getName()}: ${this.getReason()} - (${this.debugMessage})` +
+      (this.isImportant() ? '\n' + this.stack : '')
+    );
+  }
+
+  toError(): Error {
+    const error = new Error();
+    (error as any).message = this;
+    return error;
   }
 
   static deserialize(data: any): Failure {
@@ -251,10 +289,10 @@ export function ThrowIfFailed<V>(failable: Failable<V>): V {
 export function FallbackIfFailed<V>(
   failable: Failable<V>,
   fallback: V,
-  logger?: { warn: (...args: any) => any },
+  logger?: ILogger,
 ): V {
   if (HasFailed(failable)) {
-    if (logger) logger.warn(failable.print());
+    if (logger) failable.print(logger, { notImportant: true });
     return fallback;
   }