diff --git a/backend/package.json b/backend/package.json index b766917..e324c0d 100644 --- a/backend/package.json +++ b/backend/package.json @@ -77,11 +77,13 @@ "@types/passport-strategy": "^0.2.38", "@types/semver": "^7.5.8", "@types/supertest": "^6.0.2", + "@types/uuid": "^10.0.0", "prettier": "^3.3.3", "source-map-support": "^0.5.21", "ts-loader": "^9.5.1", "ts-node": "^10.9.2", "tsconfig-paths": "^4.2.0", - "typescript": "~5.5.4" + "typescript": "~5.5.4", + "uuid": "^11.0.2" } } diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index a9b1795..0f16549 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -3,13 +3,15 @@ import { MiddlewareConsumer, Module, NestModule, - OnModuleInit, + OnApplicationBootstrap, + OnApplicationShutdown } from '@nestjs/common'; import { ScheduleModule } from '@nestjs/schedule'; import { ServeStaticModule } from '@nestjs/serve-static'; import cors from 'cors'; import { IncomingMessage, ServerResponse } from 'http'; import semver from 'semver'; +import { FileDBModule } from './collections/file-db/file-db.module.js'; import { EarlyConfigModule } from './config/early/early-config.module.js'; import { ServeStaticConfigService } from './config/early/serve-static.config.service.js'; import { DatabaseModule } from './database/database.module.js'; @@ -66,6 +68,7 @@ const imageCacheSet = ( }), ScheduleModule.forRoot(), DatabaseModule, + FileDBModule, AuthManagerModule, UsageManagerModule, DemoManagerModule, @@ -73,7 +76,7 @@ const imageCacheSet = ( PicsurLayersModule, ], }) -export class AppModule implements NestModule, OnModuleInit { +export class AppModule implements NestModule, OnApplicationBootstrap, OnApplicationShutdown { private readonly logger = new Logger(AppModule.name); configure(consumer: MiddlewareConsumer) { @@ -83,7 +86,7 @@ export class AppModule implements NestModule, OnModuleInit { .forRoutes('i/(.*)'); } - onModuleInit() { + onApplicationBootstrap() { const nodeVersion = process.version; if (!supportedNodeVersions.some((v) => semver.satisfies(nodeVersion, v))) { this.logger.error( @@ -95,4 +98,8 @@ export class AppModule implements NestModule, OnModuleInit { ); } } + + onApplicationShutdown() { + this.logger.warn(`Shutting down`); + } } diff --git a/backend/src/collections/file-s3/file-s3.service.ts b/backend/src/collections/file-db/file-db-s3.service.ts similarity index 58% rename from backend/src/collections/file-s3/file-s3.service.ts rename to backend/src/collections/file-db/file-db-s3.service.ts index c81c1a0..c9736df 100644 --- a/backend/src/collections/file-s3/file-s3.service.ts +++ b/backend/src/collections/file-db/file-db-s3.service.ts @@ -5,20 +5,21 @@ import { GetObjectCommand, ListBucketsCommand, PutObjectCommand, - S3Client, + S3Client } from '@aws-sdk/client-s3'; import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; -import { getStreamAsBuffer } from 'get-stream'; -import { AsyncFailable, Fail, FT } from 'picsur-shared/types/failable'; +import { getStreamAsBuffer } from 'get-stream'; +import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/types/failable'; import { Readable } from 'stream'; import { S3ConfigService } from '../../config/early/s3.config.service.js'; +import { FileDB } from './file-db.interface.js'; @Injectable() -export class FileS3Service implements OnModuleInit { +export class FileS3Service implements OnModuleInit, FileDB { private readonly logger = new Logger(FileS3Service.name); - private S3: Promise = this.loadS3(); + private S3: S3Client | null = null; constructor(private readonly s3config: S3ConfigService) {} @@ -26,7 +27,98 @@ export class FileS3Service implements OnModuleInit { this.loadS3(); } - private async loadS3(): Promise { + public async putFile(key: string, data: Buffer | Readable): AsyncFailable { + const S3 = await this.getS3(); + if (HasFailed(S3)) return S3; + + const request = new PutObjectCommand({ + Bucket: this.s3config.getS3Bucket(), + Key: key, + Body: data, + }); + + try { + await S3.send(request); + return key; + } catch (e) { + return Fail(FT.S3, e); + } + } + + public async getFileStream(key: string): AsyncFailable { + const S3 = await this.getS3(); + if (HasFailed(S3)) return S3; + + const request = new GetObjectCommand({ + Bucket: this.s3config.getS3Bucket(), + Key: key, + }); + + try { + const result = await S3.send(request); + if (!result.Body) return Fail(FT.NotFound, 'File not found'); + + return result.Body as Readable; + } catch (e) { + return Fail(FT.S3, e); + } + } + + public async getFileBlob(key: string): AsyncFailable { + const stream = await this.getFileStream(key); + if (HasFailed(stream)) return stream; + + try { + return await getStreamAsBuffer(stream); + } catch (e) { + return Fail(FT.S3, e); + } + } + + public async deleteFile(key: string): AsyncFailable { + const S3 = await this.getS3(); + if (HasFailed(S3)) return S3; + + const request = new DeleteObjectCommand({ + Bucket: this.s3config.getS3Bucket(), + Key: key, + }); + + try { + await S3.send(request); + return true; + } catch (e) { + return Fail(FT.S3, e); + } + } + + public async deleteFiles(keys: string[]): AsyncFailable { + const S3 = await this.getS3(); + if (HasFailed(S3)) return S3; + + const request = new DeleteObjectsCommand({ + Bucket: this.s3config.getS3Bucket(), + Delete: { + Objects: keys.map((key) => ({ Key: key })), + }, + }); + + try { + await S3.send(request); + return true; + } catch (e) { + return Fail(FT.S3, e); + } + } + + private async getS3(): AsyncFailable { + if (this.S3) return this.S3; + await this.loadS3(); + if (this.S3) return this.S3; + return Fail(FT.S3, 'S3 not loaded'); + } + + private async loadS3(): Promise { const S3 = new S3Client(this.s3config.getS3Config()); try { @@ -43,81 +135,14 @@ export class FileS3Service implements OnModuleInit { } else { this.logger.verbose(`Using existing S3 Bucket ${bucket}`); } + + this.S3 = S3; } catch (e) { this.logger.error(e); - } - return S3; - } - - public async putFile(key: string, data: Buffer): AsyncFailable { - const S3 = await this.S3; - - const request = new PutObjectCommand({ - Bucket: this.s3config.getS3Bucket(), - Key: key, - Body: data, - }); - - try { - await S3.send(request); - return key; - } catch (e) { - return Fail(FT.Database, e); - } - } - - public async getFile(key: string): AsyncFailable { - const S3 = await this.S3; - - const request = new GetObjectCommand({ - Bucket: this.s3config.getS3Bucket(), - Key: key, - }); - - try { - const result = await S3.send(request); - if (!result.Body) return Fail(FT.NotFound, 'File not found'); - - if (result.Body instanceof Blob) { - return Buffer.from(await result.Body.arrayBuffer()); - } - return await getStreamAsBuffer(result.Body as Readable); - } catch (e) { - return Fail(FT.Database, e); - } - } - - public async deleteFile(key: string): AsyncFailable { - const S3 = await this.S3; - - const request = new DeleteObjectCommand({ - Bucket: this.s3config.getS3Bucket(), - Key: key, - }); - - try { - await S3.send(request); - return true; - } catch (e) { - return Fail(FT.Database, e); - } - } - - public async deleteFiles(keys: string[]): AsyncFailable { - const S3 = await this.S3; - - const request = new DeleteObjectsCommand({ - Bucket: this.s3config.getS3Bucket(), - Delete: { - Objects: keys.map((key) => ({ Key: key })), - }, - }); - - try { - await S3.send(request); - return true; - } catch (e) { - return Fail(FT.Database, e); + this.logger.warn( + 'There was an error setting up S3, are you sure you have set up an S3 instance and configured it correctly?\n' + + 'Please check https://github.com/caramelfur/picsur for up to date documentation.', + ); } } } diff --git a/backend/src/collections/file-db/file-db.interface.ts b/backend/src/collections/file-db/file-db.interface.ts new file mode 100644 index 0000000..3dbfcd2 --- /dev/null +++ b/backend/src/collections/file-db/file-db.interface.ts @@ -0,0 +1,10 @@ +import { AsyncFailable } from 'picsur-shared/types/failable'; +import { Readable } from 'stream'; + +export interface FileDB { + putFile(key: string, data: Buffer | Readable): AsyncFailable; + getFileBlob(key: string): AsyncFailable; + getFileStream(key: string): AsyncFailable; + deleteFile(key: string): AsyncFailable; + deleteFiles(keys: string[]): AsyncFailable; +} diff --git a/backend/src/collections/file-s3/file-s3.module.ts b/backend/src/collections/file-db/file-db.module.ts similarity index 72% rename from backend/src/collections/file-s3/file-s3.module.ts rename to backend/src/collections/file-db/file-db.module.ts index ee1af7b..914af55 100644 --- a/backend/src/collections/file-s3/file-s3.module.ts +++ b/backend/src/collections/file-db/file-db.module.ts @@ -1,10 +1,10 @@ import { Module } from '@nestjs/common'; import { EarlyConfigModule } from '../../config/early/early-config.module.js'; -import { FileS3Service } from './file-s3.service.js'; +import { FileS3Service } from './file-db-s3.service.js'; @Module({ imports: [EarlyConfigModule], providers: [FileS3Service], exports: [FileS3Service], }) -export class FileS3Module {} +export class FileDBModule {} diff --git a/backend/src/collections/image-db/image-db.module.ts b/backend/src/collections/image-db/image-db.module.ts index 4322c1a..944b963 100644 --- a/backend/src/collections/image-db/image-db.module.ts +++ b/backend/src/collections/image-db/image-db.module.ts @@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity.js'; import { EImageFileBackend } from '../../database/entities/images/image-file.entity.js'; import { EImageBackend } from '../../database/entities/images/image.entity.js'; +import { FileDBModule } from '../file-db/file-db.module.js'; import { ImageDBService } from './image-db.service.js'; import { ImageFileDBService } from './image-file-db.service.js'; @@ -13,6 +14,7 @@ import { ImageFileDBService } from './image-file-db.service.js'; EImageFileBackend, EImageDerivativeBackend, ]), + FileDBModule ], providers: [ImageDBService, ImageFileDBService], exports: [ImageDBService, ImageFileDBService], diff --git a/backend/src/collections/image-db/image-file-db.service.ts b/backend/src/collections/image-db/image-file-db.service.ts index ab1d135..45a0a1e 100644 --- a/backend/src/collections/image-db/image-file-db.service.ts +++ b/backend/src/collections/image-db/image-file-db.service.ts @@ -2,14 +2,16 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum'; import { - AsyncFailable, - Fail, - FT, - HasFailed, + AsyncFailable, + Fail, + FT, + HasFailed, } from 'picsur-shared/dist/types/failable'; -import { LessThan, Repository } from 'typeorm'; +import { In, IsNull, LessThan, Repository } from 'typeorm'; +import { v4 as uuidv4 } from 'uuid'; import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity.js'; import { EImageFileBackend } from '../../database/entities/images/image-file.entity.js'; +import { FileS3Service } from '../file-db/file-db-s3.service.js'; const A_DAY_IN_SECONDS = 24 * 60 * 60; @@ -21,24 +23,61 @@ export class ImageFileDBService { @InjectRepository(EImageDerivativeBackend) private readonly imageDerivativeRepo: Repository, + + private readonly s3Service: FileS3Service, ) {} + public async getFileData( + file: EImageFileBackend | EImageDerivativeBackend, + ): AsyncFailable { + if (file.data !== null) { + // Migrate files from old format to s3 + const data = file.data; + + const s3result = await this.s3Service.putFile(file.fileKey, data); + if (HasFailed(s3result)) return s3result; + + file.data = null; + let repoResult: EImageFileBackend | EImageDerivativeBackend; + if (file instanceof EImageFileBackend) { + repoResult = await this.imageFileRepo.save(file); + } else if (file instanceof EImageDerivativeBackend) { + repoResult = await this.imageDerivativeRepo.save(file); + } else { + return Fail(FT.SysValidation, 'Invalid file type'); + } + if (HasFailed(repoResult)) return repoResult; + + return data; + } + + const result = await this.s3Service.getFile(file.fileKey); + if (HasFailed(result)) return result; + + return result; + } + public async setFile( imageId: string, variant: ImageEntryVariant, file: Buffer, filetype: string, ): AsyncFailable { + const s3key = uuidv4(); + const imageFile = new EImageFileBackend(); imageFile.image_id = imageId; imageFile.variant = variant; imageFile.filetype = filetype; - imageFile.data = file; + imageFile.fileKey = s3key; try { await this.imageFileRepo.upsert(imageFile, { conflictPaths: ['image_id', 'variant'], }); + + const s3result = await this.s3Service.putFile(s3key, file); + if (HasFailed(s3result)) return s3result; } catch (e) { return Fail(FT.Database, e); } @@ -89,6 +128,9 @@ export class ImageFileDBService { if (!found) return Fail(FT.NotFound, 'Image not found'); + const s3result = await this.s3Service.deleteFile(found.fileKey); + if (HasFailed(s3result)) return s3result; + await this.imageFileRepo.delete({ image_id: imageId, variant: variant }); return found; } catch (e) { @@ -125,15 +167,22 @@ export class ImageFileDBService { filetype: string, file: Buffer, ): AsyncFailable { + const s3key = uuidv4(); + const imageDerivative = new EImageDerivativeBackend(); imageDerivative.image_id = imageId; imageDerivative.key = key; imageDerivative.filetype = filetype; - imageDerivative.data = file; + imageDerivative.fileKey = s3key; imageDerivative.last_read = new Date(); try { - return await this.imageDerivativeRepo.save(imageDerivative); + const result = await this.imageDerivativeRepo.save(imageDerivative); + + const s3result = await this.s3Service.putFile(s3key, file); + if (HasFailed(s3result)) return s3result; + + return result; } catch (e) { return Fail(FT.Database, e); } @@ -176,4 +225,49 @@ export class ImageFileDBService { return Fail(FT.Database, e); } } + + public async cleanupOrphanedDerivatives(): AsyncFailable { + return this.cleanupRepoWithFilekey(this.imageDerivativeRepo); + } + + public async cleanupOrphanedFiles(): AsyncFailable { + return this.cleanupRepoWithFilekey(this.imageFileRepo); + } + + // Go over all image files in the db, and any that are not linked to an image are deleted from s3 and the db + private async cleanupRepoWithFilekey( + repo: Repository<{ image_id: string | null; fileKey: string }>, + ): AsyncFailable { + try { + let remaining = Infinity; + let processed = 0; + + while (remaining > 0) { + const orphaned = await repo.findAndCount({ + where: { + image_id: IsNull(), + }, + select: ['fileKey'], + take: 100, + }); + if (orphaned[1] === 0) break; + remaining = orphaned[1] - orphaned[0].length; + + const keys = orphaned[0].map((d) => d.fileKey); + + const s3result = await this.s3Service.deleteFiles(keys); + if (HasFailed(s3result)) return s3result; + + const result = await repo.delete({ + fileKey: In(keys), + }); + + processed += result.affected ?? 0; + } + + return processed; + } catch (e) { + return Fail(FT.Database, e); + } + } } diff --git a/backend/src/config/early/early-config.module.ts b/backend/src/config/early/early-config.module.ts index fb59526..c47aabd 100644 --- a/backend/src/config/early/early-config.module.ts +++ b/backend/src/config/early/early-config.module.ts @@ -4,7 +4,6 @@ import { AuthConfigService } from './auth.config.service.js'; import { EarlyJwtConfigService } from './early-jwt.config.service.js'; import { HostConfigService } from './host.config.service.js'; import { MultipartConfigService } from './multipart.config.service.js'; -import { RedisConfigService } from './redis.config.service.js'; import { S3ConfigService } from './s3.config.service.js'; import { ServeStaticConfigService } from './serve-static.config.service.js'; import { TypeOrmConfigService } from './type-orm.config.service.js'; @@ -23,7 +22,6 @@ import { TypeOrmConfigService } from './type-orm.config.service.js'; HostConfigService, AuthConfigService, MultipartConfigService, - RedisConfigService, S3ConfigService, ], exports: [ @@ -34,7 +32,6 @@ import { TypeOrmConfigService } from './type-orm.config.service.js'; HostConfigService, AuthConfigService, MultipartConfigService, - RedisConfigService, S3ConfigService, ], }) diff --git a/backend/src/config/early/redis.config.service.ts b/backend/src/config/early/redis.config.service.ts deleted file mode 100644 index 91f49cd..0000000 --- a/backend/src/config/early/redis.config.service.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { ParseString } from 'picsur-shared/dist/util/parse-simple'; -import { EnvPrefix } from '../config.static.js'; - -@Injectable() -export class RedisConfigService { - private readonly logger = new Logger(RedisConfigService.name); - - constructor(private readonly configService: ConfigService) { - this.logger.log('Redis URL: ' + this.getRedisUrl()); - } - - public getRedisUrl(): string { - return ParseString( - this.configService.get(`${EnvPrefix}REDIS_URL`), - 'redis://localhost:6379', - ); - } -} diff --git a/backend/src/config/early/s3.config.service.ts b/backend/src/config/early/s3.config.service.ts index c37d46d..4e5ac2e 100644 --- a/backend/src/config/early/s3.config.service.ts +++ b/backend/src/config/early/s3.config.service.ts @@ -3,12 +3,16 @@ import { Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { ParseString } from 'picsur-shared/dist/util/parse-simple'; import { EnvPrefix } from '../config.static.js'; +import { StorageConfigService } from './storage.config.service.js'; @Injectable() export class S3ConfigService { private readonly logger = new Logger(S3ConfigService.name); - constructor(private readonly configService: ConfigService) { + constructor( + private readonly configService: ConfigService, + private readonly storageConfigService: StorageConfigService, + ) { if (this.getS3Endpoint()) this.logger.log('Custom S3 Endpoint: ' + this.getS3Endpoint()); diff --git a/backend/src/config/early/storage.config.service.ts b/backend/src/config/early/storage.config.service.ts new file mode 100644 index 0000000..04066cd --- /dev/null +++ b/backend/src/config/early/storage.config.service.ts @@ -0,0 +1,44 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { ParseString } from 'picsur-shared/dist/util/parse-simple'; +import { EnvPrefix } from '../config.static.js'; +import { HostConfigService } from './host.config.service.js'; + +export enum StorageTarget { + DATABASE = 'DATABASE', + LOCAL = 'LOCAL', + S3 = 'S3', +} + +@Injectable() +export class StorageConfigService { + private readonly logger = new Logger(StorageConfigService.name); + + constructor( + private readonly configService: ConfigService, + private readonly hostConfigService: HostConfigService, + ) { + this.logger.log('Storage Target: ' + this.getStorageTarget()); + + this.logger.log('Local Storage Path: ' + this.getLocalStoragePath()); + } + + public getStorageTarget(): StorageTarget { + const target = ParseString( + this.configService.get(`${EnvPrefix}STORAGE_TARGET`), + StorageTarget.S3, + ) as StorageTarget; + // Ensure the location is valid + if (Object.values(StorageTarget).includes(target)) { + return target as StorageTarget; + } + return StorageTarget.DATABASE; + } + + public getLocalStoragePath(): string { + return ParseString( + this.configService.get(`${EnvPrefix}LOCAL_STORAGE_PATH`), + this.hostConfigService.isProduction() ? '/data' : './data', + ); + } +} diff --git a/backend/src/database/entities/images/image-derivative.entity.ts b/backend/src/database/entities/images/image-derivative.entity.ts index 32e5ce7..ce93aa0 100644 --- a/backend/src/database/entities/images/image-derivative.entity.ts +++ b/backend/src/database/entities/images/image-derivative.entity.ts @@ -1,41 +1,46 @@ import { - Column, - Entity, - Index, - JoinColumn, - ManyToOne, - PrimaryGeneratedColumn, - Unique, + Column, + Entity, + Index, + JoinColumn, + ManyToOne, + PrimaryColumn, + Unique } from 'typeorm'; import { EImageBackend } from './image.entity.js'; @Entity() @Unique(['image_id', 'key']) export class EImageDerivativeBackend { - @PrimaryGeneratedColumn('uuid') - private _id?: string; + @PrimaryColumn({ type: 'uuid', nullable: false, name: '_id' }) + @Index() + fileKey: string; - // We do a little trickery + // == Reference to parent image @Index() @ManyToOne(() => EImageBackend, (image) => image.derivatives, { - nullable: false, - onDelete: 'CASCADE', + nullable: true, + onDelete: 'SET NULL', }) @JoinColumn({ name: 'image_id' }) private _image?: any; @Column({ name: 'image_id', + nullable: true, }) - image_id: string; + image_id: string | null; + // == Derivative options hash @Index() @Column({ nullable: false }) key: string; + // == Filetype of the derivative @Column({ nullable: false }) filetype: string; + // == Last time the derivative was read @Column({ type: 'timestamptz', name: 'last_read', @@ -43,7 +48,7 @@ export class EImageDerivativeBackend { }) last_read: Date; - // Binary data - @Column({ type: 'bytea', nullable: false }) - data: Buffer; + // == Binary data + @Column({ type: 'bytea', nullable: true }) + data: Buffer | null; } diff --git a/backend/src/database/entities/images/image-file.entity.ts b/backend/src/database/entities/images/image-file.entity.ts index 37abe04..01b525a 100644 --- a/backend/src/database/entities/images/image-file.entity.ts +++ b/backend/src/database/entities/images/image-file.entity.ts @@ -1,43 +1,48 @@ import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum'; import { - Column, - Entity, - Index, - JoinColumn, - ManyToOne, - PrimaryGeneratedColumn, - Unique, + Column, + Entity, + Index, + JoinColumn, + ManyToOne, + PrimaryColumn, + Unique } from 'typeorm'; import { EImageBackend } from './image.entity.js'; @Entity() @Unique(['image_id', 'variant']) export class EImageFileBackend { - @PrimaryGeneratedColumn('uuid') - private _id?: string; + @PrimaryColumn({ type: 'uuid', nullable: false, name: '_id' }) + @Index() + fileKey: string; - // We do a little trickery + // == Reference to parent image @Index() @ManyToOne(() => EImageBackend, (image) => image.files, { - nullable: false, - onDelete: 'CASCADE', + // TODO: is this smart? idea is that we keep it here so we know to delete it in s3 + nullable: true, + onDelete: 'SET NULL', }) @JoinColumn({ name: 'image_id' }) private _image?: any; @Column({ name: 'image_id', + nullable: true, }) - image_id: string; + image_id: string | null; + // == File variant @Index() @Column({ nullable: false, enum: ImageEntryVariant }) variant: ImageEntryVariant; + // == Filetype of the derivative @Column({ nullable: false }) filetype: string; - // Binary data - @Column({ type: 'bytea', nullable: false }) - data: Buffer; + // == Binary data + @Column({ type: 'bytea', nullable: true }) + data: Buffer | null; } diff --git a/backend/src/main.ts b/backend/src/main.ts index 685ecff..7ad336e 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -3,8 +3,8 @@ import multipart from '@fastify/multipart'; import fastifyReplyFrom from '@fastify/reply-from'; import { NestFactory } from '@nestjs/core'; import { - FastifyAdapter, - NestFastifyApplication, + FastifyAdapter, + NestFastifyApplication, } from '@nestjs/platform-fastify'; import { AppModule } from './app.module.js'; import { HostConfigService } from './config/early/host.config.service.js'; @@ -43,6 +43,8 @@ async function bootstrap() { }, ); + app.enableShutdownHooks(); + // Configure logger app.useLogger(app.get(PicsurLoggerService)); app.flushLogs(); diff --git a/backend/src/managers/image/image-manager.module.ts b/backend/src/managers/image/image-manager.module.ts index d7a9c91..42a38d8 100644 --- a/backend/src/managers/image/image-manager.module.ts +++ b/backend/src/managers/image/image-manager.module.ts @@ -39,6 +39,7 @@ export class ImageManagerModule implements OnModuleInit { await this.cleanupDerivatives(); await this.cleanupExpired(); await this.cleanupOrphanedFiles(); + // TODO: Auto migrate all images to S3 } private async cleanupDerivatives() { @@ -79,23 +80,23 @@ export class ImageManagerModule implements OnModuleInit { } private async cleanupOrphanedFiles() { - // const cleanedUpDerivatives = - // await this.imageFileDB.cleanupOrphanedDerivatives(); + const cleanedUpDerivatives = + await this.imageFileDB.cleanupOrphanedDerivatives(); - // if (HasFailed(cleanedUpDerivatives)) { - // cleanedUpDerivatives.print(this.logger); - // return; - // } + if (HasFailed(cleanedUpDerivatives)) { + cleanedUpDerivatives.print(this.logger); + return; + } - // const cleanedUpFiles = await this.imageFileDB.cleanupOrphanedFiles(); - // if (HasFailed(cleanedUpFiles)) { - // cleanedUpFiles.print(this.logger); - // return; - // } + const cleanedUpFiles = await this.imageFileDB.cleanupOrphanedFiles(); + if (HasFailed(cleanedUpFiles)) { + cleanedUpFiles.print(this.logger); + return; + } - // if (cleanedUpDerivatives > 0 || cleanedUpFiles > 0) - // this.logger.log( - // `Cleaned up ${cleanedUpDerivatives} orphaned derivatives and ${cleanedUpFiles} orphaned files`, - // ); + if (cleanedUpDerivatives > 0 || cleanedUpFiles > 0) + this.logger.log( + `Cleaned up ${cleanedUpDerivatives} orphaned derivatives and ${cleanedUpFiles} orphaned files`, + ); } } diff --git a/backend/src/managers/image/image.service.ts b/backend/src/managers/image/image.service.ts index 342ba2b..68f8fc0 100644 --- a/backend/src/managers/image/image.service.ts +++ b/backend/src/managers/image/image.service.ts @@ -4,18 +4,18 @@ import { fileTypeFromBuffer, FileTypeResult } from 'file-type'; import { ImageRequestParams } from 'picsur-shared/dist/dto/api/image.dto'; import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum'; import { - AnimFileType, - FileType, - ImageFileType, - Mime2FileType, + AnimFileType, + FileType, + ImageFileType, + Mime2FileType, } from 'picsur-shared/dist/dto/mimes.dto'; import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum'; import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.enum'; import { - AsyncFailable, - Fail, - FT, - HasFailed, + AsyncFailable, + Fail, + FT, + HasFailed, } from 'picsur-shared/dist/types/failable'; import { FindResult } from 'picsur-shared/dist/types/find-result'; import { ParseFileType } from 'picsur-shared/dist/util/parse-mime'; @@ -62,11 +62,13 @@ export class ImageManagerService { userid: string | undefined, options: Partial>, ): AsyncFailable { - if (options.expires_at !== undefined && options.expires_at !== null) { - if (options.expires_at < new Date()) { - return Fail(FT.UsrValidation, 'Expiration date must be in the future'); - } - } + if ( + options.expires_at !== undefined && + options.expires_at !== null && + options.expires_at < new Date() + ) + return Fail(FT.UsrValidation, 'Expiration date must be in the future'); + return await this.imagesService.update(id, userid, options); } @@ -119,13 +121,24 @@ export class ImageManagerService { ); if (HasFailed(imageEntity)) return imageEntity; + const onFail = async () => { + const result = await this.imagesService.delete( + [imageEntity.id], + undefined, + ); + if (HasFailed(result)) result.print(this.logger); + }; + const imageFileEntity = await this.imageFilesService.setFile( imageEntity.id, ImageEntryVariant.MASTER, processResult.image, processResult.filetype, ); - if (HasFailed(imageFileEntity)) return imageFileEntity; + if (HasFailed(imageFileEntity)) { + await onFail(); + return imageFileEntity; + } if (keepOriginal) { const originalFileEntity = await this.imageFilesService.setFile( @@ -134,7 +147,10 @@ export class ImageManagerService { image, fileType.identifier, ); - if (HasFailed(originalFileEntity)) return originalFileEntity; + if (HasFailed(originalFileEntity)) { + await onFail(); + return originalFileEntity; + } } return imageEntity; @@ -167,9 +183,12 @@ export class ImageManagerService { const sourceFileType = ParseFileType(masterImage.filetype); if (HasFailed(sourceFileType)) return sourceFileType; + const data = await this.imageFilesService.getFileData(masterImage); + if (HasFailed(data)) return data; + const startTime = Date.now(); const convertResult = await this.convertService.convert( - masterImage.data, + data, sourceFileType, targetFileType, allow_editing ? options : {}, @@ -239,6 +258,12 @@ export class ImageManagerService { }; } + public async getFileData( + file: EImageFileBackend | EImageDerivativeBackend, + ): AsyncFailable { + return this.imageFilesService.getFileData(file); + } + // Util stuff ================================================================== private async getFileTypeFromBuffer(image: Buffer): AsyncFailable { diff --git a/backend/src/routes/image/image-manage.controller.ts b/backend/src/routes/image/image-manage.controller.ts index 5e9d705..7c89be5 100644 --- a/backend/src/routes/image/image-manage.controller.ts +++ b/backend/src/routes/image/image-manage.controller.ts @@ -1,37 +1,37 @@ import { - Body, - Controller, - Get, - Logger, - Param, - Post, - Res, + Body, + Controller, + Get, + Logger, + Param, + Post, + Res, } from '@nestjs/common'; import type { FastifyReply } from 'fastify'; import { - ImageDeleteRequest, - ImageDeleteResponse, - ImageDeleteWithKeyRequest, - ImageDeleteWithKeyResponse, - ImageListRequest, - ImageListResponse, - ImageUpdateRequest, - ImageUpdateResponse, - ImageUploadResponse, + ImageDeleteRequest, + ImageDeleteResponse, + ImageDeleteWithKeyRequest, + ImageDeleteWithKeyResponse, + ImageListRequest, + ImageListResponse, + ImageUpdateRequest, + ImageUpdateResponse, + ImageUploadResponse, } from 'picsur-shared/dist/dto/api/image-manage.dto'; import { Permission } from 'picsur-shared/dist/dto/permissions.enum'; import { - FT, - Fail, - HasFailed, - ThrowIfFailed, + FT, + Fail, + HasFailed, + ThrowIfFailed, } from 'picsur-shared/dist/types/failable'; import { EasyThrottle } from '../../decorators/easy-throttle.decorator.js'; import { PostFiles } from '../../decorators/multipart/multipart.decorator.js'; import type { FileIterator } from '../../decorators/multipart/postfiles.pipe.js'; import { - HasPermission, - RequiredPermissions, + HasPermission, + RequiredPermissions, } from '../../decorators/permissions.decorator.js'; import { ReqUserID } from '../../decorators/request-user.decorator.js'; import { Returns } from '../../decorators/returns.decorator.js'; @@ -97,14 +97,14 @@ export class ImageManageController { @RequiredPermissions(Permission.ImageManage) @Returns(ImageUpdateResponse) async updateImage( - @Body() body: ImageUpdateRequest, + @Body() options: ImageUpdateRequest, @ReqUserID() userid: string, @HasPermission(Permission.ImageAdmin) isImageAdmin: boolean, ): Promise { const user_id = isImageAdmin ? undefined : userid; const image = ThrowIfFailed( - await this.imagesService.update(body.id, user_id, body), + await this.imagesService.update(options.id, user_id, options), ); return image; diff --git a/backend/src/routes/image/image.controller.ts b/backend/src/routes/image/image.controller.ts index a5ecd54..6e213f6 100644 --- a/backend/src/routes/image/image.controller.ts +++ b/backend/src/routes/image/image.controller.ts @@ -2,17 +2,19 @@ import { Controller, Get, Head, Logger, Query, Res } from '@nestjs/common'; import { SkipThrottle } from '@nestjs/throttler'; import type { FastifyReply } from 'fastify'; import { - ImageMetaResponse, - ImageRequestParams, + ImageMetaResponse, + ImageRequestParams, } from 'picsur-shared/dist/dto/api/image.dto'; import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum'; import { FileType2Mime } from 'picsur-shared/dist/dto/mimes.dto'; import { - FT, - IsFailure, - ThrowIfFailed, + FT, + IsFailure, + ThrowIfFailed, } from 'picsur-shared/dist/types/failable'; import { UserDbService } from '../../collections/user-db/user-db.service.js'; +import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity.js'; +import { EImageFileBackend } from '../../database/entities/images/image-file.entity.js'; import { ImageFullIdParam } from '../../decorators/image-id/image-full-id.decorator.js'; import { ImageIdParam } from '../../decorators/image-id/image-id.decorator.js'; import { RequiredPermissions } from '../../decorators/permissions.decorator.js'; @@ -61,25 +63,23 @@ export class ImageController { @Query() params: ImageRequestParams, ): Promise { try { + let image: EImageFileBackend | EImageDerivativeBackend; if (fullid.variant === ImageEntryVariant.ORIGINAL) { - const image = ThrowIfFailed( - await this.imagesService.getOriginal(fullid.id), + image = ThrowIfFailed(await this.imagesService.getOriginal(fullid.id)); + } else { + image = ThrowIfFailed( + await this.imagesService.getConverted( + fullid.id, + fullid.filetype, + params, + ), ); - - res.type(ThrowIfFailed(FileType2Mime(image.filetype))); - return image.data; } - const image = ThrowIfFailed( - await this.imagesService.getConverted( - fullid.id, - fullid.filetype, - params, - ), - ); + const data = ThrowIfFailed(await this.imagesService.getFileData(image)); res.type(ThrowIfFailed(FileType2Mime(image.filetype))); - return image.data; + return data; } catch (e) { if (!IsFailure(e) || e.getType() !== FT.NotFound) throw e; diff --git a/frontend/src/app/services/api/api.service.ts b/frontend/src/app/services/api/api.service.ts index 7a75f1a..fe2ddd2 100644 --- a/frontend/src/app/services/api/api.service.ts +++ b/frontend/src/app/services/api/api.service.ts @@ -3,7 +3,7 @@ import { WINDOW } from '@ng-web-apis/common'; import axios, { AxiosRequestConfig, AxiosResponse, - AxiosResponseHeaders, + AxiosResponseHeaders } from 'axios'; import { ApiResponseSchema } from 'picsur-shared/dist/dto/api/api.dto'; import { FileType2Ext } from 'picsur-shared/dist/dto/mimes.dto'; @@ -244,15 +244,13 @@ export class ApiService { uploadProgress.next((e.loaded / (e.total ?? 1000000)) * 100); }, signal: abortController.signal, + validateStatus: () => true, ...options, }); uploadProgress.complete(); downloadProgress.complete(); - if (result.status < 200 || result.status >= 300) { - return Fail(FT.Network, 'Recieved a non-ok response'); - } return result; } catch (e) { return Fail(FT.Network, e); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 65b3ca7..c3c92d0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -204,6 +204,9 @@ importers: '@types/supertest': specifier: ^6.0.2 version: 6.0.2 + '@types/uuid': + specifier: ^10.0.0 + version: 10.0.0 prettier: specifier: ^3.3.3 version: 3.3.3 @@ -222,6 +225,9 @@ importers: typescript: specifier: ~5.5.4 version: 5.5.4 + uuid: + specifier: ^11.0.2 + version: 11.0.2 frontend: devDependencies: @@ -2867,6 +2873,9 @@ packages: '@types/supertest@6.0.2': resolution: {integrity: sha512-137ypx2lk/wTQbW6An6safu9hXmajAifU/s7szAHLN/FeIm5w7yR0Wkl9fdJMRSHwOn4HLAI0DaB2TOORuhPDg==} + '@types/uuid@10.0.0': + resolution: {integrity: sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==} + '@types/validator@13.12.2': resolution: {integrity: sha512-6SlHBzUW8Jhf3liqrGGXyTJSIFe4nqlJ5A5KaMZ2l/vbM3Wh3KSybots/wfWVzNLK4D1NZluDlSQIbIEPx6oyA==} @@ -6093,6 +6102,10 @@ packages: resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==} hasBin: true + uuid@11.0.2: + resolution: {integrity: sha512-14FfcOJmqdjbBPdDjFQyk/SdT4NySW4eM0zcG+HqbHP5jzuH56xO3J1DGhgs/cEMCfwYi3HQI1gnTO62iaG+tQ==} + hasBin: true + uuid@8.3.2: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} hasBin: true @@ -9539,6 +9552,8 @@ snapshots: '@types/methods': 1.1.4 '@types/superagent': 8.1.9 + '@types/uuid@10.0.0': {} + '@types/validator@13.12.2': {} '@types/wrap-ansi@3.0.0': {} @@ -12973,6 +12988,8 @@ snapshots: uuid@10.0.0: {} + uuid@11.0.2: {} + uuid@8.3.2: {} uuid@9.0.1: {} diff --git a/shared/src/types/failable.ts b/shared/src/types/failable.ts index eb521a6..e07f77e 100644 --- a/shared/src/types/failable.ts +++ b/shared/src/types/failable.ts @@ -7,6 +7,7 @@ export enum FT { Unknown = 'unknown', Database = 'database', + S3 = 's3', SysValidation = 'sysvalidation', UsrValidation = 'usrvalidation', BadRequest = 'badrequest', @@ -51,6 +52,11 @@ const FTProps: { code: 500, message: 'A database error occurred', }, + [FT.S3]: { + important: true, + code: 500, + message: 'An S3 error occurred', + }, [FT.Network]: { important: true, code: 500,