[TEMP] working on stuff

backend/package.json

@@ -77,11 +77,13 @@
     "@types/passport-strategy": "^0.2.38",
     "@types/semver": "^7.5.8",
     "@types/supertest": "^6.0.2",
+    "@types/uuid": "^10.0.0",
     "prettier": "^3.3.3",
     "source-map-support": "^0.5.21",
     "ts-loader": "^9.5.1",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
-    "typescript": "~5.5.4"
+    "typescript": "~5.5.4",
+    "uuid": "^11.0.2"
   }
 }

backend/src/app.module.ts

@@ -3,13 +3,15 @@ import {
   MiddlewareConsumer,
   Module,
   NestModule,
-  OnModuleInit,
+  OnApplicationBootstrap,
+  OnApplicationShutdown
 } from '@nestjs/common';
 import { ScheduleModule } from '@nestjs/schedule';
 import { ServeStaticModule } from '@nestjs/serve-static';
 import cors from 'cors';
 import { IncomingMessage, ServerResponse } from 'http';
 import semver from 'semver';
+import { FileDBModule } from './collections/file-db/file-db.module.js';
 import { EarlyConfigModule } from './config/early/early-config.module.js';
 import { ServeStaticConfigService } from './config/early/serve-static.config.service.js';
 import { DatabaseModule } from './database/database.module.js';
@@ -66,6 +68,7 @@ const imageCacheSet = (
     }),
     ScheduleModule.forRoot(),
     DatabaseModule,
+    FileDBModule,
     AuthManagerModule,
     UsageManagerModule,
     DemoManagerModule,
@@ -73,7 +76,7 @@ const imageCacheSet = (
     PicsurLayersModule,
   ],
 })
-export class AppModule implements NestModule, OnModuleInit {
+export class AppModule implements NestModule, OnApplicationBootstrap, OnApplicationShutdown {
   private readonly logger = new Logger(AppModule.name);
 
   configure(consumer: MiddlewareConsumer) {
@@ -83,7 +86,7 @@ export class AppModule implements NestModule, OnModuleInit {
       .forRoutes('i/(.*)');
   }
 
-  onModuleInit() {
+  onApplicationBootstrap() {
     const nodeVersion = process.version;
     if (!supportedNodeVersions.some((v) => semver.satisfies(nodeVersion, v))) {
       this.logger.error(
@@ -95,4 +98,8 @@ export class AppModule implements NestModule, OnModuleInit {
       );
     }
   }
+
+  onApplicationShutdown() {
+    this.logger.warn(`Shutting down`);
+  }
 }

backend/src/collections/file-db/file-db-s3.service.ts

@@ -5,20 +5,21 @@ import {
   GetObjectCommand,
   ListBucketsCommand,
   PutObjectCommand,
-  S3Client,
+  S3Client
 } from '@aws-sdk/client-s3';
 import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
-import { getStreamAsBuffer } from 'get-stream';
 
-import { AsyncFailable, Fail, FT } from 'picsur-shared/types/failable';
+import { getStreamAsBuffer } from 'get-stream';
+import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/types/failable';
 import { Readable } from 'stream';
 import { S3ConfigService } from '../../config/early/s3.config.service.js';
+import { FileDB } from './file-db.interface.js';
 
 @Injectable()
-export class FileS3Service implements OnModuleInit {
+export class FileS3Service implements OnModuleInit, FileDB {
   private readonly logger = new Logger(FileS3Service.name);
 
-  private S3: Promise<S3Client> = this.loadS3();
+  private S3: S3Client | null = null;
 
   constructor(private readonly s3config: S3ConfigService) {}
 
@@ -26,7 +27,98 @@ export class FileS3Service implements OnModuleInit {
     this.loadS3();
   }
 
-  private async loadS3(): Promise<S3Client> {
+  public async putFile(key: string, data: Buffer | Readable): AsyncFailable<string> {
+    const S3 = await this.getS3();
+    if (HasFailed(S3)) return S3;
+
+    const request = new PutObjectCommand({
+      Bucket: this.s3config.getS3Bucket(),
+      Key: key,
+      Body: data,
+    });
+
+    try {
+      await S3.send(request);
+      return key;
+    } catch (e) {
+      return Fail(FT.S3, e);
+    }
+  }
+
+  public async getFileStream(key: string): AsyncFailable<Readable> {
+    const S3 = await this.getS3();
+    if (HasFailed(S3)) return S3;
+
+    const request = new GetObjectCommand({
+      Bucket: this.s3config.getS3Bucket(),
+      Key: key,
+    });
+
+    try {
+      const result = await S3.send(request);
+      if (!result.Body) return Fail(FT.NotFound, 'File not found');
+
+      return result.Body as Readable;
+    } catch (e) {
+      return Fail(FT.S3, e);
+    }
+  }
+
+  public async getFileBlob(key: string): AsyncFailable<Buffer> {
+    const stream = await this.getFileStream(key);
+    if (HasFailed(stream)) return stream;
+
+    try {
+      return await getStreamAsBuffer(stream);
+    } catch (e) {
+      return Fail(FT.S3, e);
+    }
+  }
+
+  public async deleteFile(key: string): AsyncFailable<true> {
+    const S3 = await this.getS3();
+    if (HasFailed(S3)) return S3;
+
+    const request = new DeleteObjectCommand({
+      Bucket: this.s3config.getS3Bucket(),
+      Key: key,
+    });
+
+    try {
+      await S3.send(request);
+      return true;
+    } catch (e) {
+      return Fail(FT.S3, e);
+    }
+  }
+
+  public async deleteFiles(keys: string[]): AsyncFailable<true> {
+    const S3 = await this.getS3();
+    if (HasFailed(S3)) return S3;
+
+    const request = new DeleteObjectsCommand({
+      Bucket: this.s3config.getS3Bucket(),
+      Delete: {
+        Objects: keys.map((key) => ({ Key: key })),
+      },
+    });
+
+    try {
+      await S3.send(request);
+      return true;
+    } catch (e) {
+      return Fail(FT.S3, e);
+    }
+  }
+
+  private async getS3(): AsyncFailable<S3Client> {
+    if (this.S3) return this.S3;
+    await this.loadS3();
+    if (this.S3) return this.S3;
+    return Fail(FT.S3, 'S3 not loaded');
+  }
+
+  private async loadS3(): Promise<void> {
     const S3 = new S3Client(this.s3config.getS3Config());
 
     try {
@@ -43,81 +135,14 @@ export class FileS3Service implements OnModuleInit {
       } else {
         this.logger.verbose(`Using existing S3 Bucket ${bucket}`);
       }
+
+      this.S3 = S3;
     } catch (e) {
       this.logger.error(e);
-    }
-    return S3;
-  }
-
-  public async putFile(key: string, data: Buffer): AsyncFailable<string> {
-    const S3 = await this.S3;
-
-    const request = new PutObjectCommand({
-      Bucket: this.s3config.getS3Bucket(),
-      Key: key,
-      Body: data,
-    });
-
-    try {
-      await S3.send(request);
-      return key;
-    } catch (e) {
-      return Fail(FT.Database, e);
-    }
-  }
-
-  public async getFile(key: string): AsyncFailable<Buffer> {
-    const S3 = await this.S3;
-
-    const request = new GetObjectCommand({
-      Bucket: this.s3config.getS3Bucket(),
-      Key: key,
-    });
-
-    try {
-      const result = await S3.send(request);
-      if (!result.Body) return Fail(FT.NotFound, 'File not found');
-
-      if (result.Body instanceof Blob) {
-        return Buffer.from(await result.Body.arrayBuffer());
-      }
-      return await getStreamAsBuffer(result.Body as Readable);
-    } catch (e) {
-      return Fail(FT.Database, e);
-    }
-  }
-
-  public async deleteFile(key: string): AsyncFailable<true> {
-    const S3 = await this.S3;
-
-    const request = new DeleteObjectCommand({
-      Bucket: this.s3config.getS3Bucket(),
-      Key: key,
-    });
-
-    try {
-      await S3.send(request);
-      return true;
-    } catch (e) {
-      return Fail(FT.Database, e);
-    }
-  }
-
-  public async deleteFiles(keys: string[]): AsyncFailable<true> {
-    const S3 = await this.S3;
-
-    const request = new DeleteObjectsCommand({
-      Bucket: this.s3config.getS3Bucket(),
-      Delete: {
-        Objects: keys.map((key) => ({ Key: key })),
-      },
-    });
-
-    try {
-      await S3.send(request);
-      return true;
-    } catch (e) {
-      return Fail(FT.Database, e);
+      this.logger.warn(
+        'There was an error setting up S3, are you sure you have set up an S3 instance and configured it correctly?\n' +
+          'Please check https://github.com/caramelfur/picsur for up to date documentation.',
+      );
     }
   }
 }

backend/src/collections/file-db/file-db.interface.ts

@@ -0,0 +1,10 @@
+import { AsyncFailable } from 'picsur-shared/types/failable';
+import { Readable } from 'stream';
+
+export interface FileDB {
+  putFile(key: string, data: Buffer | Readable): AsyncFailable<string>;
+  getFileBlob(key: string): AsyncFailable<Buffer>;
+  getFileStream(key: string): AsyncFailable<Readable>;
+  deleteFile(key: string): AsyncFailable<true>;
+  deleteFiles(keys: string[]): AsyncFailable<true>;
+}

backend/src/collections/file-db/file-db.module.ts

@@ -1,10 +1,10 @@
 import { Module } from '@nestjs/common';
 import { EarlyConfigModule } from '../../config/early/early-config.module.js';
-import { FileS3Service } from './file-s3.service.js';
+import { FileS3Service } from './file-db-s3.service.js';
 
 @Module({
   imports: [EarlyConfigModule],
   providers: [FileS3Service],
   exports: [FileS3Service],
 })
-export class FileS3Module {}
+export class FileDBModule {}

backend/src/collections/image-db/image-db.module.ts

@@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
 import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity.js';
 import { EImageFileBackend } from '../../database/entities/images/image-file.entity.js';
 import { EImageBackend } from '../../database/entities/images/image.entity.js';
+import { FileDBModule } from '../file-db/file-db.module.js';
 import { ImageDBService } from './image-db.service.js';
 import { ImageFileDBService } from './image-file-db.service.js';
 
@@ -13,6 +14,7 @@ import { ImageFileDBService } from './image-file-db.service.js';
       EImageFileBackend,
       EImageDerivativeBackend,
     ]),
+    FileDBModule
   ],
   providers: [ImageDBService, ImageFileDBService],
   exports: [ImageDBService, ImageFileDBService],

backend/src/collections/image-db/image-file-db.service.ts

@@ -2,14 +2,16 @@ import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
 import {
-    AsyncFailable,
-    Fail,
-    FT,
-    HasFailed,
+  AsyncFailable,
+  Fail,
+  FT,
+  HasFailed,
 } from 'picsur-shared/dist/types/failable';
-import { LessThan, Repository } from 'typeorm';
+import { In, IsNull, LessThan, Repository } from 'typeorm';
+import { v4 as uuidv4 } from 'uuid';
 import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity.js';
 import { EImageFileBackend } from '../../database/entities/images/image-file.entity.js';
+import { FileS3Service } from '../file-db/file-db-s3.service.js';
 
 const A_DAY_IN_SECONDS = 24 * 60 * 60;
 
@@ -21,24 +23,61 @@ export class ImageFileDBService {
 
     @InjectRepository(EImageDerivativeBackend)
     private readonly imageDerivativeRepo: Repository<EImageDerivativeBackend>,
+
+    private readonly s3Service: FileS3Service,
   ) {}
 
+  public async getFileData(
+    file: EImageFileBackend | EImageDerivativeBackend,
+  ): AsyncFailable<Buffer> {
+    if (file.data !== null) {
+      // Migrate files from old format to s3
+      const data = file.data;
+
+      const s3result = await this.s3Service.putFile(file.fileKey, data);
+      if (HasFailed(s3result)) return s3result;
+
+      file.data = null;
+      let repoResult: EImageFileBackend | EImageDerivativeBackend;
+      if (file instanceof EImageFileBackend) {
+        repoResult = await this.imageFileRepo.save(file);
+      } else if (file instanceof EImageDerivativeBackend) {
+        repoResult = await this.imageDerivativeRepo.save(file);
+      } else {
+        return Fail(FT.SysValidation, 'Invalid file type');
+      }
+      if (HasFailed(repoResult)) return repoResult;
+
+      return data;
+    }
+
+    const result = await this.s3Service.getFile(file.fileKey);
+    if (HasFailed(result)) return result;
+
+    return result;
+  }
+
   public async setFile(
     imageId: string,
     variant: ImageEntryVariant,
     file: Buffer,
     filetype: string,
   ): AsyncFailable<true> {
+    const s3key = uuidv4();
+
     const imageFile = new EImageFileBackend();
     imageFile.image_id = imageId;
     imageFile.variant = variant;
     imageFile.filetype = filetype;
-    imageFile.data = file;
+    imageFile.fileKey = s3key;
 
     try {
       await this.imageFileRepo.upsert(imageFile, {
         conflictPaths: ['image_id', 'variant'],
       });
+
+      const s3result = await this.s3Service.putFile(s3key, file);
+      if (HasFailed(s3result)) return s3result;
     } catch (e) {
       return Fail(FT.Database, e);
     }
@@ -89,6 +128,9 @@ export class ImageFileDBService {
 
       if (!found) return Fail(FT.NotFound, 'Image not found');
 
+      const s3result = await this.s3Service.deleteFile(found.fileKey);
+      if (HasFailed(s3result)) return s3result;
+
       await this.imageFileRepo.delete({ image_id: imageId, variant: variant });
       return found;
     } catch (e) {
@@ -125,15 +167,22 @@ export class ImageFileDBService {
     filetype: string,
     file: Buffer,
   ): AsyncFailable<EImageDerivativeBackend> {
+    const s3key = uuidv4();
+
     const imageDerivative = new EImageDerivativeBackend();
     imageDerivative.image_id = imageId;
     imageDerivative.key = key;
     imageDerivative.filetype = filetype;
-    imageDerivative.data = file;
+    imageDerivative.fileKey = s3key;
     imageDerivative.last_read = new Date();
 
     try {
-      return await this.imageDerivativeRepo.save(imageDerivative);
+      const result = await this.imageDerivativeRepo.save(imageDerivative);
+
+      const s3result = await this.s3Service.putFile(s3key, file);
+      if (HasFailed(s3result)) return s3result;
+
+      return result;
     } catch (e) {
       return Fail(FT.Database, e);
     }
@@ -176,4 +225,49 @@ export class ImageFileDBService {
       return Fail(FT.Database, e);
     }
   }
+
+  public async cleanupOrphanedDerivatives(): AsyncFailable<number> {
+    return this.cleanupRepoWithFilekey(this.imageDerivativeRepo);
+  }
+
+  public async cleanupOrphanedFiles(): AsyncFailable<number> {
+    return this.cleanupRepoWithFilekey(this.imageFileRepo);
+  }
+
+  // Go over all image files in the db, and any that are not linked to an image are deleted from s3 and the db
+  private async cleanupRepoWithFilekey(
+    repo: Repository<{ image_id: string | null; fileKey: string }>,
+  ): AsyncFailable<number> {
+    try {
+      let remaining = Infinity;
+      let processed = 0;
+      
+      while (remaining > 0) {
+        const orphaned = await repo.findAndCount({
+          where: {
+            image_id: IsNull(),
+          },
+          select: ['fileKey'],
+          take: 100,
+        });
+        if (orphaned[1] === 0) break;
+        remaining = orphaned[1] - orphaned[0].length;
+
+        const keys = orphaned[0].map((d) => d.fileKey);
+
+        const s3result = await this.s3Service.deleteFiles(keys);
+        if (HasFailed(s3result)) return s3result;
+
+        const result = await repo.delete({
+          fileKey: In(keys),
+        });
+
+        processed += result.affected ?? 0;
+      }
+
+      return processed;
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
 }

backend/src/config/early/early-config.module.ts

@@ -4,7 +4,6 @@ import { AuthConfigService } from './auth.config.service.js';
 import { EarlyJwtConfigService } from './early-jwt.config.service.js';
 import { HostConfigService } from './host.config.service.js';
 import { MultipartConfigService } from './multipart.config.service.js';
-import { RedisConfigService } from './redis.config.service.js';
 import { S3ConfigService } from './s3.config.service.js';
 import { ServeStaticConfigService } from './serve-static.config.service.js';
 import { TypeOrmConfigService } from './type-orm.config.service.js';
@@ -23,7 +22,6 @@ import { TypeOrmConfigService } from './type-orm.config.service.js';
     HostConfigService,
     AuthConfigService,
     MultipartConfigService,
-    RedisConfigService,
     S3ConfigService,
   ],
   exports: [
@@ -34,7 +32,6 @@ import { TypeOrmConfigService } from './type-orm.config.service.js';
     HostConfigService,
     AuthConfigService,
     MultipartConfigService,
-    RedisConfigService,
     S3ConfigService,
   ],
 })

backend/src/config/early/redis.config.service.ts

@@ -1,20 +0,0 @@
-import { Injectable, Logger } from '@nestjs/common';
-import { ConfigService } from '@nestjs/config';
-import { ParseString } from 'picsur-shared/dist/util/parse-simple';
-import { EnvPrefix } from '../config.static.js';
-
-@Injectable()
-export class RedisConfigService {
-  private readonly logger = new Logger(RedisConfigService.name);
-
-  constructor(private readonly configService: ConfigService) {
-    this.logger.log('Redis URL: ' + this.getRedisUrl());
-  }
-
-  public getRedisUrl(): string {
-    return ParseString(
-      this.configService.get(`${EnvPrefix}REDIS_URL`),
-      'redis://localhost:6379',
-    );
-  }
-}

backend/src/config/early/s3.config.service.ts

@@ -3,12 +3,16 @@ import { Injectable, Logger } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { ParseString } from 'picsur-shared/dist/util/parse-simple';
 import { EnvPrefix } from '../config.static.js';
+import { StorageConfigService } from './storage.config.service.js';
 
 @Injectable()
 export class S3ConfigService {
   private readonly logger = new Logger(S3ConfigService.name);
 
-  constructor(private readonly configService: ConfigService) {
+  constructor(
+    private readonly configService: ConfigService,
+    private readonly storageConfigService: StorageConfigService,
+  ) {
     if (this.getS3Endpoint())
       this.logger.log('Custom S3 Endpoint: ' + this.getS3Endpoint());
 

backend/src/config/early/storage.config.service.ts

@@ -0,0 +1,44 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { ParseString } from 'picsur-shared/dist/util/parse-simple';
+import { EnvPrefix } from '../config.static.js';
+import { HostConfigService } from './host.config.service.js';
+
+export enum StorageTarget {
+  DATABASE = 'DATABASE',
+  LOCAL = 'LOCAL',
+  S3 = 'S3',
+}
+
+@Injectable()
+export class StorageConfigService {
+  private readonly logger = new Logger(StorageConfigService.name);
+
+  constructor(
+    private readonly configService: ConfigService,
+    private readonly hostConfigService: HostConfigService,
+  ) {
+    this.logger.log('Storage Target: ' + this.getStorageTarget());
+
+    this.logger.log('Local Storage Path: ' + this.getLocalStoragePath());
+  }
+
+  public getStorageTarget(): StorageTarget {
+    const target = ParseString(
+      this.configService.get(`${EnvPrefix}STORAGE_TARGET`),
+      StorageTarget.S3,
+    ) as StorageTarget;
+    // Ensure the location is valid
+    if (Object.values(StorageTarget).includes(target)) {
+      return target as StorageTarget;
+    }
+    return StorageTarget.DATABASE;
+  }
+
+  public getLocalStoragePath(): string {
+    return ParseString(
+      this.configService.get(`${EnvPrefix}LOCAL_STORAGE_PATH`),
+      this.hostConfigService.isProduction() ? '/data' : './data',
+    );
+  }
+}

backend/src/database/entities/images/image-derivative.entity.ts

@@ -1,41 +1,46 @@
 import {
-    Column,
-    Entity,
-    Index,
-    JoinColumn,
-    ManyToOne,
-    PrimaryGeneratedColumn,
-    Unique,
+  Column,
+  Entity,
+  Index,
+  JoinColumn,
+  ManyToOne,
+  PrimaryColumn,
+  Unique
 } from 'typeorm';
 import { EImageBackend } from './image.entity.js';
 
 @Entity()
 @Unique(['image_id', 'key'])
 export class EImageDerivativeBackend {
-  @PrimaryGeneratedColumn('uuid')
-  private _id?: string;
+  @PrimaryColumn({ type: 'uuid', nullable: false, name: '_id' })
+  @Index()
+  fileKey: string;
 
-  // We do a little trickery
+  // == Reference to parent image
   @Index()
   @ManyToOne(() => EImageBackend, (image) => image.derivatives, {
-    nullable: false,
-    onDelete: 'CASCADE',
+    nullable: true,
+    onDelete: 'SET NULL',
   })
   @JoinColumn({ name: 'image_id' })
   private _image?: any;
 
   @Column({
     name: 'image_id',
+    nullable: true,
   })
-  image_id: string;
+  image_id: string | null;
 
+  // == Derivative options hash
   @Index()
   @Column({ nullable: false })
   key: string;
 
+  // == Filetype of the derivative
   @Column({ nullable: false })
   filetype: string;
 
+  // == Last time the derivative was read
   @Column({
     type: 'timestamptz',
     name: 'last_read',
@@ -43,7 +48,7 @@ export class EImageDerivativeBackend {
   })
   last_read: Date;
 
-  // Binary data
-  @Column({ type: 'bytea', nullable: false })
-  data: Buffer;
+  // == Binary data
+  @Column({ type: 'bytea', nullable: true })
+  data: Buffer | null;
 }

backend/src/database/entities/images/image-file.entity.ts

@@ -1,43 +1,48 @@
 import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
 import {
-    Column,
-    Entity,
-    Index,
-    JoinColumn,
-    ManyToOne,
-    PrimaryGeneratedColumn,
-    Unique,
+  Column,
+  Entity,
+  Index,
+  JoinColumn,
+  ManyToOne,
+  PrimaryColumn,
+  Unique
 } from 'typeorm';
 import { EImageBackend } from './image.entity.js';
 
 @Entity()
 @Unique(['image_id', 'variant'])
 export class EImageFileBackend {
-  @PrimaryGeneratedColumn('uuid')
-  private _id?: string;
+  @PrimaryColumn({ type: 'uuid', nullable: false, name: '_id' })
+  @Index()
+  fileKey: string;
 
-  // We do a little trickery
+  // == Reference to parent image
   @Index()
   @ManyToOne(() => EImageBackend, (image) => image.files, {
-    nullable: false,
-    onDelete: 'CASCADE',
+    // TODO: is this smart? idea is that we keep it here so we know to delete it in s3
+    nullable: true,
+    onDelete: 'SET NULL',
   })
   @JoinColumn({ name: 'image_id' })
   private _image?: any;
 
   @Column({
     name: 'image_id',
+    nullable: true,
   })
-  image_id: string;
+  image_id: string | null;
 
+  // == File variant
   @Index()
   @Column({ nullable: false, enum: ImageEntryVariant })
   variant: ImageEntryVariant;
 
+  // == Filetype of the derivative
   @Column({ nullable: false })
   filetype: string;
 
-  // Binary data
-  @Column({ type: 'bytea', nullable: false })
-  data: Buffer;
+  // == Binary data
+  @Column({ type: 'bytea', nullable: true })
+  data: Buffer | null;
 }

backend/src/main.ts

@@ -3,8 +3,8 @@ import multipart from '@fastify/multipart';
 import fastifyReplyFrom from '@fastify/reply-from';
 import { NestFactory } from '@nestjs/core';
 import {
-    FastifyAdapter,
-    NestFastifyApplication,
+  FastifyAdapter,
+  NestFastifyApplication,
 } from '@nestjs/platform-fastify';
 import { AppModule } from './app.module.js';
 import { HostConfigService } from './config/early/host.config.service.js';
@@ -43,6 +43,8 @@ async function bootstrap() {
     },
   );
 
+  app.enableShutdownHooks();
+
   // Configure logger
   app.useLogger(app.get(PicsurLoggerService));
   app.flushLogs();

backend/src/managers/image/image-manager.module.ts

@@ -39,6 +39,7 @@ export class ImageManagerModule implements OnModuleInit {
     await this.cleanupDerivatives();
     await this.cleanupExpired();
     await this.cleanupOrphanedFiles();
+    // TODO: Auto migrate all images to S3
   }
 
   private async cleanupDerivatives() {
@@ -79,23 +80,23 @@ export class ImageManagerModule implements OnModuleInit {
   }
 
   private async cleanupOrphanedFiles() {
-    // const cleanedUpDerivatives =
-    //   await this.imageFileDB.cleanupOrphanedDerivatives();
+    const cleanedUpDerivatives =
+      await this.imageFileDB.cleanupOrphanedDerivatives();
 
-    // if (HasFailed(cleanedUpDerivatives)) {
-    //   cleanedUpDerivatives.print(this.logger);
-    //   return;
-    // }
+    if (HasFailed(cleanedUpDerivatives)) {
+      cleanedUpDerivatives.print(this.logger);
+      return;
+    }
 
-    // const cleanedUpFiles = await this.imageFileDB.cleanupOrphanedFiles();
-    // if (HasFailed(cleanedUpFiles)) {
-    //   cleanedUpFiles.print(this.logger);
-    //   return;
-    // }
+    const cleanedUpFiles = await this.imageFileDB.cleanupOrphanedFiles();
+    if (HasFailed(cleanedUpFiles)) {
+      cleanedUpFiles.print(this.logger);
+      return;
+    }
 
-    // if (cleanedUpDerivatives > 0 || cleanedUpFiles > 0)
-    //   this.logger.log(
-    //     `Cleaned up ${cleanedUpDerivatives} orphaned derivatives and ${cleanedUpFiles} orphaned files`,
-    //   );
+    if (cleanedUpDerivatives > 0 || cleanedUpFiles > 0)
+      this.logger.log(
+        `Cleaned up ${cleanedUpDerivatives} orphaned derivatives and ${cleanedUpFiles} orphaned files`,
+      );
   }
 }

backend/src/managers/image/image.service.ts

@@ -4,18 +4,18 @@ import { fileTypeFromBuffer, FileTypeResult } from 'file-type';
 import { ImageRequestParams } from 'picsur-shared/dist/dto/api/image.dto';
 import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
 import {
-    AnimFileType,
-    FileType,
-    ImageFileType,
-    Mime2FileType,
+  AnimFileType,
+  FileType,
+  ImageFileType,
+  Mime2FileType,
 } from 'picsur-shared/dist/dto/mimes.dto';
 import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
 import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.enum';
 import {
-    AsyncFailable,
-    Fail,
-    FT,
-    HasFailed,
+  AsyncFailable,
+  Fail,
+  FT,
+  HasFailed,
 } from 'picsur-shared/dist/types/failable';
 import { FindResult } from 'picsur-shared/dist/types/find-result';
 import { ParseFileType } from 'picsur-shared/dist/util/parse-mime';
@@ -62,11 +62,13 @@ export class ImageManagerService {
     userid: string | undefined,
     options: Partial<Pick<EImageBackend, 'file_name' | 'expires_at'>>,
   ): AsyncFailable<EImageBackend> {
-    if (options.expires_at !== undefined && options.expires_at !== null) {
-      if (options.expires_at < new Date()) {
-        return Fail(FT.UsrValidation, 'Expiration date must be in the future');
-      }
-    }
+    if (
+      options.expires_at !== undefined &&
+      options.expires_at !== null &&
+      options.expires_at < new Date()
+    )
+      return Fail(FT.UsrValidation, 'Expiration date must be in the future');
+
     return await this.imagesService.update(id, userid, options);
   }
 
@@ -119,13 +121,24 @@ export class ImageManagerService {
     );
     if (HasFailed(imageEntity)) return imageEntity;
 
+    const onFail = async () => {
+      const result = await this.imagesService.delete(
+        [imageEntity.id],
+        undefined,
+      );
+      if (HasFailed(result)) result.print(this.logger);
+    };
+
     const imageFileEntity = await this.imageFilesService.setFile(
       imageEntity.id,
       ImageEntryVariant.MASTER,
       processResult.image,
       processResult.filetype,
     );
-    if (HasFailed(imageFileEntity)) return imageFileEntity;
+    if (HasFailed(imageFileEntity)) {
+      await onFail();
+      return imageFileEntity;
+    }
 
     if (keepOriginal) {
       const originalFileEntity = await this.imageFilesService.setFile(
@@ -134,7 +147,10 @@ export class ImageManagerService {
         image,
         fileType.identifier,
       );
-      if (HasFailed(originalFileEntity)) return originalFileEntity;
+      if (HasFailed(originalFileEntity)) {
+        await onFail();
+        return originalFileEntity;
+      }
     }
 
     return imageEntity;
@@ -167,9 +183,12 @@ export class ImageManagerService {
         const sourceFileType = ParseFileType(masterImage.filetype);
         if (HasFailed(sourceFileType)) return sourceFileType;
 
+        const data = await this.imageFilesService.getFileData(masterImage);
+        if (HasFailed(data)) return data;
+
         const startTime = Date.now();
         const convertResult = await this.convertService.convert(
-          masterImage.data,
+          data,
           sourceFileType,
           targetFileType,
           allow_editing ? options : {},
@@ -239,6 +258,12 @@ export class ImageManagerService {
     };
   }
 
+  public async getFileData(
+    file: EImageFileBackend | EImageDerivativeBackend,
+  ): AsyncFailable<Buffer> {
+    return this.imageFilesService.getFileData(file);
+  }
+
   // Util stuff ==================================================================
 
   private async getFileTypeFromBuffer(image: Buffer): AsyncFailable<FileType> {

backend/src/routes/image/image-manage.controller.ts

@@ -1,37 +1,37 @@
 import {
-    Body,
-    Controller,
-    Get,
-    Logger,
-    Param,
-    Post,
-    Res,
+  Body,
+  Controller,
+  Get,
+  Logger,
+  Param,
+  Post,
+  Res,
 } from '@nestjs/common';
 import type { FastifyReply } from 'fastify';
 import {
-    ImageDeleteRequest,
-    ImageDeleteResponse,
-    ImageDeleteWithKeyRequest,
-    ImageDeleteWithKeyResponse,
-    ImageListRequest,
-    ImageListResponse,
-    ImageUpdateRequest,
-    ImageUpdateResponse,
-    ImageUploadResponse,
+  ImageDeleteRequest,
+  ImageDeleteResponse,
+  ImageDeleteWithKeyRequest,
+  ImageDeleteWithKeyResponse,
+  ImageListRequest,
+  ImageListResponse,
+  ImageUpdateRequest,
+  ImageUpdateResponse,
+  ImageUploadResponse,
 } from 'picsur-shared/dist/dto/api/image-manage.dto';
 import { Permission } from 'picsur-shared/dist/dto/permissions.enum';
 import {
-    FT,
-    Fail,
-    HasFailed,
-    ThrowIfFailed,
+  FT,
+  Fail,
+  HasFailed,
+  ThrowIfFailed,
 } from 'picsur-shared/dist/types/failable';
 import { EasyThrottle } from '../../decorators/easy-throttle.decorator.js';
 import { PostFiles } from '../../decorators/multipart/multipart.decorator.js';
 import type { FileIterator } from '../../decorators/multipart/postfiles.pipe.js';
 import {
-    HasPermission,
-    RequiredPermissions,
+  HasPermission,
+  RequiredPermissions,
 } from '../../decorators/permissions.decorator.js';
 import { ReqUserID } from '../../decorators/request-user.decorator.js';
 import { Returns } from '../../decorators/returns.decorator.js';
@@ -97,14 +97,14 @@ export class ImageManageController {
   @RequiredPermissions(Permission.ImageManage)
   @Returns(ImageUpdateResponse)
   async updateImage(
-    @Body() body: ImageUpdateRequest,
+    @Body() options: ImageUpdateRequest,
     @ReqUserID() userid: string,
     @HasPermission(Permission.ImageAdmin) isImageAdmin: boolean,
   ): Promise<ImageUpdateResponse> {
     const user_id = isImageAdmin ? undefined : userid;
 
     const image = ThrowIfFailed(
-      await this.imagesService.update(body.id, user_id, body),
+      await this.imagesService.update(options.id, user_id, options),
     );
 
     return image;

backend/src/routes/image/image.controller.ts

@@ -2,17 +2,19 @@ import { Controller, Get, Head, Logger, Query, Res } from '@nestjs/common';
 import { SkipThrottle } from '@nestjs/throttler';
 import type { FastifyReply } from 'fastify';
 import {
-    ImageMetaResponse,
-    ImageRequestParams,
+  ImageMetaResponse,
+  ImageRequestParams,
 } from 'picsur-shared/dist/dto/api/image.dto';
 import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
 import { FileType2Mime } from 'picsur-shared/dist/dto/mimes.dto';
 import {
-    FT,
-    IsFailure,
-    ThrowIfFailed,
+  FT,
+  IsFailure,
+  ThrowIfFailed,
 } from 'picsur-shared/dist/types/failable';
 import { UserDbService } from '../../collections/user-db/user-db.service.js';
+import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity.js';
+import { EImageFileBackend } from '../../database/entities/images/image-file.entity.js';
 import { ImageFullIdParam } from '../../decorators/image-id/image-full-id.decorator.js';
 import { ImageIdParam } from '../../decorators/image-id/image-id.decorator.js';
 import { RequiredPermissions } from '../../decorators/permissions.decorator.js';
@@ -61,25 +63,23 @@ export class ImageController {
     @Query() params: ImageRequestParams,
   ): Promise<Buffer> {
     try {
+      let image: EImageFileBackend | EImageDerivativeBackend;
       if (fullid.variant === ImageEntryVariant.ORIGINAL) {
-        const image = ThrowIfFailed(
-          await this.imagesService.getOriginal(fullid.id),
+        image = ThrowIfFailed(await this.imagesService.getOriginal(fullid.id));
+      } else {
+        image = ThrowIfFailed(
+          await this.imagesService.getConverted(
+            fullid.id,
+            fullid.filetype,
+            params,
+          ),
         );
-
-        res.type(ThrowIfFailed(FileType2Mime(image.filetype)));
-        return image.data;
       }
 
-      const image = ThrowIfFailed(
-        await this.imagesService.getConverted(
-          fullid.id,
-          fullid.filetype,
-          params,
-        ),
-      );
+      const data = ThrowIfFailed(await this.imagesService.getFileData(image));
 
       res.type(ThrowIfFailed(FileType2Mime(image.filetype)));
-      return image.data;
+      return data;
     } catch (e) {
       if (!IsFailure(e) || e.getType() !== FT.NotFound) throw e;
 

frontend/src/app/services/api/api.service.ts

@@ -3,7 +3,7 @@ import { WINDOW } from '@ng-web-apis/common';
 import axios, {
   AxiosRequestConfig,
   AxiosResponse,
-  AxiosResponseHeaders,
+  AxiosResponseHeaders
 } from 'axios';
 import { ApiResponseSchema } from 'picsur-shared/dist/dto/api/api.dto';
 import { FileType2Ext } from 'picsur-shared/dist/dto/mimes.dto';
@@ -244,15 +244,13 @@ export class ApiService {
             uploadProgress.next((e.loaded / (e.total ?? 1000000)) * 100);
           },
           signal: abortController.signal,
+          validateStatus: () => true,
           ...options,
         });
 
         uploadProgress.complete();
         downloadProgress.complete();
 
-        if (result.status < 200 || result.status >= 300) {
-          return Fail(FT.Network, 'Recieved a non-ok response');
-        }
         return result;
       } catch (e) {
         return Fail(FT.Network, e);

pnpm-lock.yaml

@@ -204,6 +204,9 @@ importers:
       '@types/supertest':
         specifier: ^6.0.2
         version: 6.0.2
+      '@types/uuid':
+        specifier: ^10.0.0
+        version: 10.0.0
       prettier:
         specifier: ^3.3.3
         version: 3.3.3
@@ -222,6 +225,9 @@ importers:
       typescript:
         specifier: ~5.5.4
         version: 5.5.4
+      uuid:
+        specifier: ^11.0.2
+        version: 11.0.2
 
   frontend:
     devDependencies:
@@ -2867,6 +2873,9 @@ packages:
   '@types/supertest@6.0.2':
     resolution: {integrity: sha512-137ypx2lk/wTQbW6An6safu9hXmajAifU/s7szAHLN/FeIm5w7yR0Wkl9fdJMRSHwOn4HLAI0DaB2TOORuhPDg==}
 
+  '@types/uuid@10.0.0':
+    resolution: {integrity: sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==}
+
   '@types/validator@13.12.2':
     resolution: {integrity: sha512-6SlHBzUW8Jhf3liqrGGXyTJSIFe4nqlJ5A5KaMZ2l/vbM3Wh3KSybots/wfWVzNLK4D1NZluDlSQIbIEPx6oyA==}
 
@@ -6093,6 +6102,10 @@ packages:
     resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==}
     hasBin: true
 
+  uuid@11.0.2:
+    resolution: {integrity: sha512-14FfcOJmqdjbBPdDjFQyk/SdT4NySW4eM0zcG+HqbHP5jzuH56xO3J1DGhgs/cEMCfwYi3HQI1gnTO62iaG+tQ==}
+    hasBin: true
+
   uuid@8.3.2:
     resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==}
     hasBin: true
@@ -9539,6 +9552,8 @@ snapshots:
       '@types/methods': 1.1.4
       '@types/superagent': 8.1.9
 
+  '@types/uuid@10.0.0': {}
+
   '@types/validator@13.12.2': {}
 
   '@types/wrap-ansi@3.0.0': {}
@@ -12973,6 +12988,8 @@ snapshots:
 
   uuid@10.0.0: {}
 
+  uuid@11.0.2: {}
+
   uuid@8.3.2: {}
 
   uuid@9.0.1: {}

shared/src/types/failable.ts

@@ -7,6 +7,7 @@
 export enum FT {
   Unknown = 'unknown',
   Database = 'database',
+  S3 = 's3',
   SysValidation = 'sysvalidation',
   UsrValidation = 'usrvalidation',
   BadRequest = 'badrequest',
@@ -51,6 +52,11 @@ const FTProps: {
     code: 500,
     message: 'A database error occurred',
   },
+  [FT.S3]: {
+    important: true,
+    code: 500,
+    message: 'An S3 error occurred',
+  },
   [FT.Network]: {
     important: true,
     code: 500,