diff --git a/backend/package.json b/backend/package.json index 898224f..f0bb271 100644 --- a/backend/package.json +++ b/backend/package.json @@ -78,10 +78,11 @@ "@types/semver": "^7.3.12", "@types/sharp": "^0.31.0", "@types/supertest": "^2.0.12", - "@typescript-eslint/eslint-plugin": "^5.55.0", - "@typescript-eslint/parser": "^5.55.0", - "eslint": "^8.36.0", - "eslint-config-prettier": "^8.7.0", + "@types/uuid": "^9.0.0", + "@typescript-eslint/eslint-plugin": "^5.47.0", + "@typescript-eslint/parser": "^5.47.0", + "eslint": "^8.30.0", + "eslint-config-prettier": "^8.5.0", "eslint-plugin-prettier": "^4.2.1", "prettier": "^2.8.4", "source-map-support": "^0.5.21", diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index f15375a..393dcc6 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -1,9 +1,10 @@ -import { Logger, MiddlewareConsumer, Module, NestModule, OnModuleInit } from '@nestjs/common'; +import { Logger, MiddlewareConsumer, Module, NestModule, OnApplicationBootstrap, OnApplicationShutdown } from '@nestjs/common'; import { ScheduleModule } from '@nestjs/schedule'; import { ServeStaticModule } from '@nestjs/serve-static'; import cors from 'cors'; import { IncomingMessage, ServerResponse } from 'http'; import semver from 'semver'; +import { FileS3Module } from './collections/file-s3/file-s3.module'; import { EarlyConfigModule } from './config/early/early-config.module'; import { ServeStaticConfigService } from './config/early/serve-static.config.service'; import { DatabaseModule } from './database/database.module'; @@ -49,6 +50,7 @@ const imageCorsOverride = ( }), ScheduleModule.forRoot(), DatabaseModule, + FileS3Module, AuthManagerModule, UsageManagerModule, DemoManagerModule, @@ -56,7 +58,7 @@ const imageCorsOverride = ( PicsurLayersModule, ], }) -export class AppModule implements NestModule, OnModuleInit { +export class AppModule implements NestModule, OnApplicationBootstrap, OnApplicationShutdown { private readonly logger = new Logger(AppModule.name); configure(consumer: MiddlewareConsumer) { @@ -64,7 +66,7 @@ export class AppModule implements NestModule, OnModuleInit { consumer.apply(imageCorsConfig, imageCorsOverride).forRoutes('/i'); } - onModuleInit() { + onApplicationBootstrap() { const nodeVersion = process.version; if (!supportedNodeVersions.some((v) => semver.satisfies(nodeVersion, v))) { this.logger.error( @@ -76,4 +78,8 @@ export class AppModule implements NestModule, OnModuleInit { ); } } + + onApplicationShutdown() { + this.logger.warn(`Shutting down`); + } } diff --git a/backend/src/collections/file-s3/file-s3.service.ts b/backend/src/collections/file-s3/file-s3.service.ts index c94ecbc..057698e 100644 --- a/backend/src/collections/file-s3/file-s3.service.ts +++ b/backend/src/collections/file-s3/file-s3.service.ts @@ -5,11 +5,15 @@ import { GetObjectCommand, ListBucketsCommand, PutObjectCommand, - S3Client, + S3Client } from '@aws-sdk/client-s3'; import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; import { buffer as streamToBuffer } from 'get-stream'; -import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types'; +import { + AsyncFailable, + Fail, FT, + HasFailed +} from 'picsur-shared/dist/types'; import { Readable } from 'stream'; import { S3ConfigService } from '../../config/early/s3.config.service'; @@ -17,7 +21,7 @@ import { S3ConfigService } from '../../config/early/s3.config.service'; export class FileS3Service implements OnModuleInit { private readonly logger = new Logger(FileS3Service.name); - private S3: Promise = this.loadS3(); + private S3: S3Client | null = null; constructor(private readonly s3config: S3ConfigService) {} @@ -26,7 +30,8 @@ export class FileS3Service implements OnModuleInit { } public async putFile(key: string, data: Buffer): AsyncFailable { - const S3 = await this.S3; + const S3 = await this.getS3(); + if (HasFailed(S3)) return S3; const request = new PutObjectCommand({ Bucket: this.s3config.getS3Bucket(), @@ -38,12 +43,13 @@ export class FileS3Service implements OnModuleInit { await S3.send(request); return key; } catch (e) { - return Fail(FT.Database, e); + return Fail(FT.S3, e); } } public async getFile(key: string): AsyncFailable { - const S3 = await this.S3; + const S3 = await this.getS3(); + if (HasFailed(S3)) return S3; const request = new GetObjectCommand({ Bucket: this.s3config.getS3Bucket(), @@ -59,12 +65,13 @@ export class FileS3Service implements OnModuleInit { } return streamToBuffer(result.Body as Readable); } catch (e) { - return Fail(FT.Database, e); + return Fail(FT.S3, e); } } public async deleteFile(key: string): AsyncFailable { - const S3 = await this.S3; + const S3 = await this.getS3(); + if (HasFailed(S3)) return S3; const request = new DeleteObjectCommand({ Bucket: this.s3config.getS3Bucket(), @@ -75,12 +82,13 @@ export class FileS3Service implements OnModuleInit { await S3.send(request); return true; } catch (e) { - return Fail(FT.Database, e); + return Fail(FT.S3, e); } } public async deleteFiles(keys: string[]): AsyncFailable { - const S3 = await this.S3; + const S3 = await this.getS3(); + if (HasFailed(S3)) return S3; const request = new DeleteObjectsCommand({ Bucket: this.s3config.getS3Bucket(), @@ -93,11 +101,18 @@ export class FileS3Service implements OnModuleInit { await S3.send(request); return true; } catch (e) { - return Fail(FT.Database, e); + return Fail(FT.S3, e); } } - private async loadS3(): Promise { + private async getS3(): AsyncFailable { + if (this.S3) return this.S3; + await this.loadS3(); + if (this.S3) return this.S3; + return Fail(FT.S3, 'S3 not loaded'); + } + + private async loadS3(): Promise { const S3 = new S3Client(this.s3config.getS3Config()); try { @@ -114,9 +129,14 @@ export class FileS3Service implements OnModuleInit { } else { this.logger.verbose(`Using existing S3 Bucket ${bucket}`); } + + this.S3 = S3; } catch (e) { this.logger.error(e); + this.logger.warn( + 'There was an error setting up S3, are you sure you have set up an S3 instance and configured it correctly?\n' + + 'Please check https://github.com/rubikscraft/picsur for up to date documentation.', + ); } - return S3; } } diff --git a/backend/src/collections/image-db/image-db.module.ts b/backend/src/collections/image-db/image-db.module.ts index c19ed1d..622aad7 100644 --- a/backend/src/collections/image-db/image-db.module.ts +++ b/backend/src/collections/image-db/image-db.module.ts @@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity'; import { EImageFileBackend } from '../../database/entities/images/image-file.entity'; import { EImageBackend } from '../../database/entities/images/image.entity'; +import { FileS3Module } from '../file-s3/file-s3.module'; import { ImageDBService } from './image-db.service'; import { ImageFileDBService } from './image-file-db.service'; @@ -13,6 +14,7 @@ import { ImageFileDBService } from './image-file-db.service'; EImageFileBackend, EImageDerivativeBackend, ]), + FileS3Module ], providers: [ImageDBService, ImageFileDBService], exports: [ImageDBService, ImageFileDBService], diff --git a/backend/src/collections/image-db/image-file-db.service.ts b/backend/src/collections/image-db/image-file-db.service.ts index 7991a39..a2680ff 100644 --- a/backend/src/collections/image-db/image-file-db.service.ts +++ b/backend/src/collections/image-db/image-file-db.service.ts @@ -2,9 +2,11 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum'; import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types'; -import { LessThan, Repository } from 'typeorm'; +import { In, IsNull, LessThan, Repository } from 'typeorm'; +import { v4 as uuidv4 } from 'uuid'; import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity'; import { EImageFileBackend } from '../../database/entities/images/image-file.entity'; +import { FileS3Service } from '../file-s3/file-s3.service'; const A_DAY_IN_SECONDS = 24 * 60 * 60; @@ -16,24 +18,61 @@ export class ImageFileDBService { @InjectRepository(EImageDerivativeBackend) private readonly imageDerivativeRepo: Repository, + + private readonly s3Service: FileS3Service, ) {} + public async getFileData( + file: EImageFileBackend | EImageDerivativeBackend, + ): AsyncFailable { + if (file.data !== null) { + // Migrate files from old format to s3 + const data = file.data; + + const s3result = await this.s3Service.putFile(file.fileKey, data); + if (HasFailed(s3result)) return s3result; + + file.data = null; + let repoResult: EImageFileBackend | EImageDerivativeBackend; + if (file instanceof EImageFileBackend) { + repoResult = await this.imageFileRepo.save(file); + } else if (file instanceof EImageDerivativeBackend) { + repoResult = await this.imageDerivativeRepo.save(file); + } else { + return Fail(FT.SysValidation, 'Invalid file type'); + } + if (HasFailed(repoResult)) return repoResult; + + return data; + } + + const result = await this.s3Service.getFile(file.fileKey); + if (HasFailed(result)) return result; + + return result; + } + public async setFile( imageId: string, variant: ImageEntryVariant, file: Buffer, filetype: string, ): AsyncFailable { + const s3key = uuidv4(); + const imageFile = new EImageFileBackend(); imageFile.image_id = imageId; imageFile.variant = variant; imageFile.filetype = filetype; - imageFile.data = file; + imageFile.fileKey = s3key; try { await this.imageFileRepo.upsert(imageFile, { conflictPaths: ['image_id', 'variant'], }); + + const s3result = await this.s3Service.putFile(s3key, file); + if (HasFailed(s3result)) return s3result; } catch (e) { return Fail(FT.Database, e); } @@ -84,6 +123,9 @@ export class ImageFileDBService { if (!found) return Fail(FT.NotFound, 'Image not found'); + const s3result = await this.s3Service.deleteFile(found.fileKey); + if (HasFailed(s3result)) return s3result; + await this.imageFileRepo.delete({ image_id: imageId, variant: variant }); return found; } catch (e) { @@ -120,15 +162,22 @@ export class ImageFileDBService { filetype: string, file: Buffer, ): AsyncFailable { + const s3key = uuidv4(); + const imageDerivative = new EImageDerivativeBackend(); imageDerivative.image_id = imageId; imageDerivative.key = key; imageDerivative.filetype = filetype; - imageDerivative.data = file; + imageDerivative.fileKey = s3key; imageDerivative.last_read = new Date(); try { - return await this.imageDerivativeRepo.save(imageDerivative); + const result = await this.imageDerivativeRepo.save(imageDerivative); + + const s3result = await this.s3Service.putFile(s3key, file); + if (HasFailed(s3result)) return s3result; + + return result; } catch (e) { return Fail(FT.Database, e); } @@ -171,4 +220,49 @@ export class ImageFileDBService { return Fail(FT.Database, e); } } + + public async cleanupOrphanedDerivatives(): AsyncFailable { + return this.cleanupRepoWithFilekey(this.imageDerivativeRepo); + } + + public async cleanupOrphanedFiles(): AsyncFailable { + return this.cleanupRepoWithFilekey(this.imageFileRepo); + } + + // Go over all image files in the db, and any that are not linked to an image are deleted from s3 and the db + private async cleanupRepoWithFilekey( + repo: Repository<{ image_id: string | null; fileKey: string }>, + ): AsyncFailable { + try { + let remaining = Infinity; + let processed = 0; + + while (remaining > 0) { + const orphaned = await repo.findAndCount({ + where: { + image_id: IsNull(), + }, + select: ['fileKey'], + take: 100, + }); + if (orphaned[1] === 0) break; + remaining = orphaned[1] - orphaned[0].length; + + const keys = orphaned[0].map((d) => d.fileKey); + + const s3result = await this.s3Service.deleteFiles(keys); + if (HasFailed(s3result)) return s3result; + + const result = await repo.delete({ + fileKey: In(keys), + }); + + processed += result.affected ?? 0; + } + + return processed; + } catch (e) { + return Fail(FT.Database, e); + } + } } diff --git a/backend/src/database/entities/images/image-derivative.entity.ts b/backend/src/database/entities/images/image-derivative.entity.ts index 51ca6d3..fb5b209 100644 --- a/backend/src/database/entities/images/image-derivative.entity.ts +++ b/backend/src/database/entities/images/image-derivative.entity.ts @@ -3,39 +3,42 @@ import { Entity, Index, JoinColumn, - ManyToOne, - PrimaryGeneratedColumn, - Unique, + ManyToOne, PrimaryColumn, Unique } from 'typeorm'; import { EImageBackend } from './image.entity'; @Entity() @Unique(['image_id', 'key']) export class EImageDerivativeBackend { - @PrimaryGeneratedColumn('uuid') - private _id?: string; + @PrimaryColumn({ type: 'uuid', nullable: false, name: '_id' }) + @Index() + fileKey: string; - // We do a little trickery + // == Reference to parent image @Index() @ManyToOne(() => EImageBackend, (image) => image.derivatives, { - nullable: false, - onDelete: 'CASCADE', + nullable: true, + onDelete: 'SET NULL', }) @JoinColumn({ name: 'image_id' }) private _image?: any; @Column({ name: 'image_id', + nullable: true, }) - image_id: string; + image_id: string | null; + // == Derivative options hash @Index() @Column({ nullable: false }) key: string; + // == Filetype of the derivative @Column({ nullable: false }) filetype: string; + // == Last time the derivative was read @Column({ type: 'timestamptz', name: 'last_read', @@ -43,7 +46,7 @@ export class EImageDerivativeBackend { }) last_read: Date; - // Binary data - @Column({ type: 'bytea', nullable: false }) - data: Buffer; + // == Binary data + @Column({ type: 'bytea', nullable: true }) + data: Buffer | null; } diff --git a/backend/src/database/entities/images/image-file.entity.ts b/backend/src/database/entities/images/image-file.entity.ts index f4ab13b..fc57ab9 100644 --- a/backend/src/database/entities/images/image-file.entity.ts +++ b/backend/src/database/entities/images/image-file.entity.ts @@ -5,39 +5,42 @@ import { Index, JoinColumn, ManyToOne, - PrimaryGeneratedColumn, - Unique, + PrimaryColumn, Unique } from 'typeorm'; import { EImageBackend } from './image.entity'; @Entity() @Unique(['image_id', 'variant']) export class EImageFileBackend { - @PrimaryGeneratedColumn('uuid') - private _id?: string; + @PrimaryColumn({ type: 'uuid', nullable: false, name: '_id' }) + @Index() + fileKey: string; - // We do a little trickery + // == Reference to parent image @Index() @ManyToOne(() => EImageBackend, (image) => image.files, { - nullable: false, - onDelete: 'CASCADE', + nullable: true, + onDelete: 'SET NULL', }) @JoinColumn({ name: 'image_id' }) private _image?: any; @Column({ name: 'image_id', + nullable: true, }) - image_id: string; + image_id: string | null; + // == File variant @Index() @Column({ nullable: false, enum: ImageEntryVariant }) variant: ImageEntryVariant; + // == Filetype of the derivative @Column({ nullable: false }) filetype: string; - // Binary data - @Column({ type: 'bytea', nullable: false }) - data: Buffer; + // == Binary data + @Column({ type: 'bytea', nullable: true }) + data: Buffer | null; } diff --git a/backend/src/database/migrations/1672247794308-V_0_6_0_a.ts b/backend/src/database/migrations/1672247794308-V_0_6_0_a.ts new file mode 100644 index 0000000..707d677 --- /dev/null +++ b/backend/src/database/migrations/1672247794308-V_0_6_0_a.ts @@ -0,0 +1,44 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class V060A1672247794308 implements MigrationInterface { + name = 'V060A1672247794308' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db"`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "FK_37055605f39b3f8847232d604f8"`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a"`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "_id" DROP DEFAULT`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "image_id" DROP NOT NULL`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "data" DROP NOT NULL`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75"`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "_id" DROP DEFAULT`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "image_id" DROP NOT NULL`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "data" DROP NOT NULL`); + await queryRunner.query(`CREATE INDEX "IDX_95953be58a506e5de46feec618" ON "e_image_file_backend" ("_id") `); + await queryRunner.query(`CREATE INDEX "IDX_ff1ecff935b8d7bdcea8908781" ON "e_image_derivative_backend" ("_id") `); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a" UNIQUE ("image_id", "variant")`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75" UNIQUE ("image_id", "key")`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE SET NULL ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "FK_37055605f39b3f8847232d604f8" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE SET NULL ON UPDATE NO ACTION`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "FK_37055605f39b3f8847232d604f8"`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db"`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75"`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a"`); + await queryRunner.query(`DROP INDEX "public"."IDX_ff1ecff935b8d7bdcea8908781"`); + await queryRunner.query(`DROP INDEX "public"."IDX_95953be58a506e5de46feec618"`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "data" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "image_id" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "_id" SET DEFAULT uuid_generate_v4()`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75" UNIQUE ("image_id", "key")`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "data" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "image_id" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "_id" SET DEFAULT uuid_generate_v4()`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a" UNIQUE ("image_id", "variant")`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "FK_37055605f39b3f8847232d604f8" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + } + +} diff --git a/backend/src/database/migrations/index.ts b/backend/src/database/migrations/index.ts index f75529d..de91ecf 100644 --- a/backend/src/database/migrations/index.ts +++ b/backend/src/database/migrations/index.ts @@ -5,6 +5,7 @@ import { V040B1662485374471 } from './1662485374471-V_0_4_0_b'; import { V040C1662535484200 } from './1662535484200-V_0_4_0_c'; import { V040D1662728275448 } from './1662728275448-V_0_4_0_d'; import { V050A1672154027079 } from './1672154027079-V_0_5_0_a'; +import { V060A1672247794308 } from './1672247794308-V_0_6_0_a'; export const MigrationList: Function[] = [ V030A1661692206479, @@ -14,4 +15,5 @@ export const MigrationList: Function[] = [ V040C1662535484200, V040D1662728275448, V050A1672154027079, + V060A1672247794308 ]; diff --git a/backend/src/main.ts b/backend/src/main.ts index 4a531b2..4ae09e1 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -4,7 +4,7 @@ import fastifyReplyFrom from '@fastify/reply-from'; import { NestFactory } from '@nestjs/core'; import { FastifyAdapter, - NestFastifyApplication, + NestFastifyApplication } from '@nestjs/platform-fastify'; import { AppModule } from './app.module'; import { HostConfigService } from './config/early/host.config.service'; @@ -43,6 +43,8 @@ async function bootstrap() { }, ); + app.enableShutdownHooks(); + // Configure logger app.useLogger(app.get(PicsurLoggerService)); app.flushLogs(); diff --git a/backend/src/managers/image/image-manager.module.ts b/backend/src/managers/image/image-manager.module.ts index d9d9aaa..38e026a 100644 --- a/backend/src/managers/image/image-manager.module.ts +++ b/backend/src/managers/image/image-manager.module.ts @@ -39,6 +39,7 @@ export class ImageManagerModule implements OnModuleInit { await this.cleanupDerivatives(); await this.cleanupExpired(); await this.cleanupOrphanedFiles(); + // TODO: Auto migrate all images to S3 } private async cleanupDerivatives() { @@ -78,23 +79,23 @@ export class ImageManagerModule implements OnModuleInit { } private async cleanupOrphanedFiles() { - // const cleanedUpDerivatives = - // await this.imageFileDB.cleanupOrphanedDerivatives(); + const cleanedUpDerivatives = + await this.imageFileDB.cleanupOrphanedDerivatives(); - // if (HasFailed(cleanedUpDerivatives)) { - // cleanedUpDerivatives.print(this.logger); - // return; - // } + if (HasFailed(cleanedUpDerivatives)) { + cleanedUpDerivatives.print(this.logger); + return; + } - // const cleanedUpFiles = await this.imageFileDB.cleanupOrphanedFiles(); - // if (HasFailed(cleanedUpFiles)) { - // cleanedUpFiles.print(this.logger); - // return; - // } + const cleanedUpFiles = await this.imageFileDB.cleanupOrphanedFiles(); + if (HasFailed(cleanedUpFiles)) { + cleanedUpFiles.print(this.logger); + return; + } - // if (cleanedUpDerivatives > 0 || cleanedUpFiles > 0) - // this.logger.log( - // `Cleaned up ${cleanedUpDerivatives} orphaned derivatives and ${cleanedUpFiles} orphaned files`, - // ); + if (cleanedUpDerivatives > 0 || cleanedUpFiles > 0) + this.logger.log( + `Cleaned up ${cleanedUpDerivatives} orphaned derivatives and ${cleanedUpFiles} orphaned files`, + ); } } diff --git a/backend/src/managers/image/image.service.ts b/backend/src/managers/image/image.service.ts index 7b5f245..6c21804 100644 --- a/backend/src/managers/image/image.service.ts +++ b/backend/src/managers/image/image.service.ts @@ -7,7 +7,7 @@ import { AnimFileType, FileType, ImageFileType, - Mime2FileType, + Mime2FileType } from 'picsur-shared/dist/dto/mimes.dto'; import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum'; import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.enum'; @@ -57,11 +57,13 @@ export class ImageManagerService { userid: string | undefined, options: Partial>, ): AsyncFailable { - if (options.expires_at !== undefined && options.expires_at !== null) { - if (options.expires_at < new Date()) { - return Fail(FT.UsrValidation, 'Expiration date must be in the future'); - } - } + if ( + options.expires_at !== undefined && + options.expires_at !== null && + options.expires_at < new Date() + ) + return Fail(FT.UsrValidation, 'Expiration date must be in the future'); + return await this.imagesService.update(id, userid, options); } @@ -114,13 +116,24 @@ export class ImageManagerService { ); if (HasFailed(imageEntity)) return imageEntity; + const onFail = async () => { + const result = await this.imagesService.delete( + [imageEntity.id], + undefined, + ); + if (HasFailed(result)) result.print(this.logger); + }; + const imageFileEntity = await this.imageFilesService.setFile( imageEntity.id, ImageEntryVariant.MASTER, processResult.image, processResult.filetype, ); - if (HasFailed(imageFileEntity)) return imageFileEntity; + if (HasFailed(imageFileEntity)) { + await onFail(); + return imageFileEntity; + } if (keepOriginal) { const originalFileEntity = await this.imageFilesService.setFile( @@ -129,7 +142,10 @@ export class ImageManagerService { image, fileType.identifier, ); - if (HasFailed(originalFileEntity)) return originalFileEntity; + if (HasFailed(originalFileEntity)) { + await onFail(); + return originalFileEntity; + } } return imageEntity; @@ -162,9 +178,12 @@ export class ImageManagerService { const sourceFileType = ParseFileType(masterImage.filetype); if (HasFailed(sourceFileType)) return sourceFileType; + const data = await this.imageFilesService.getFileData(masterImage); + if (HasFailed(data)) return data; + const startTime = Date.now(); const convertResult = await this.convertService.convert( - masterImage.data, + data, sourceFileType, targetFileType, allow_editing ? options : {}, @@ -234,6 +253,12 @@ export class ImageManagerService { }; } + public async getFileData( + file: EImageFileBackend | EImageDerivativeBackend, + ): AsyncFailable { + return this.imageFilesService.getFileData(file); + } + // Util stuff ================================================================== private async getFileTypeFromBuffer(image: Buffer): AsyncFailable { diff --git a/backend/src/routes/image/image-manage.controller.ts b/backend/src/routes/image/image-manage.controller.ts index 765b069..e1de11e 100644 --- a/backend/src/routes/image/image-manage.controller.ts +++ b/backend/src/routes/image/image-manage.controller.ts @@ -5,7 +5,7 @@ import { Logger, Param, Post, - Res, + Res } from '@nestjs/common'; import { Throttle } from '@nestjs/throttler'; import type { FastifyReply } from 'fastify'; @@ -18,7 +18,7 @@ import { ImageListResponse, ImageUpdateRequest, ImageUpdateResponse, - ImageUploadResponse, + ImageUploadResponse } from 'picsur-shared/dist/dto/api/image-manage.dto'; import { Permission } from 'picsur-shared/dist/dto/permissions.enum'; import { Fail, FT, HasFailed, ThrowIfFailed } from 'picsur-shared/dist/types'; @@ -26,7 +26,7 @@ import { PostFiles } from '../../decorators/multipart/multipart.decorator'; import type { FileIterator } from '../../decorators/multipart/postfiles.pipe'; import { HasPermission, - RequiredPermissions, + RequiredPermissions } from '../../decorators/permissions.decorator'; import { ReqUserID } from '../../decorators/request-user.decorator'; import { Returns } from '../../decorators/returns.decorator'; @@ -91,14 +91,14 @@ export class ImageManageController { @RequiredPermissions(Permission.ImageManage) @Returns(ImageUpdateResponse) async updateImage( - @Body() body: ImageUpdateRequest, + @Body() options: ImageUpdateRequest, @ReqUserID() userid: string, @HasPermission(Permission.ImageAdmin) isImageAdmin: boolean, ): Promise { const user_id = isImageAdmin ? undefined : userid; const image = ThrowIfFailed( - await this.imagesService.update(body.id, user_id, body), + await this.imagesService.update(options.id, user_id, options), ); return image; diff --git a/backend/src/routes/image/image.controller.ts b/backend/src/routes/image/image.controller.ts index 6dda3af..aa83c01 100644 --- a/backend/src/routes/image/image.controller.ts +++ b/backend/src/routes/image/image.controller.ts @@ -3,12 +3,14 @@ import { SkipThrottle } from '@nestjs/throttler'; import type { FastifyReply } from 'fastify'; import { ImageMetaResponse, - ImageRequestParams, + ImageRequestParams } from 'picsur-shared/dist/dto/api/image.dto'; import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum'; import { FileType2Mime } from 'picsur-shared/dist/dto/mimes.dto'; import { FT, IsFailure, ThrowIfFailed } from 'picsur-shared/dist/types'; import { UserDbService } from '../../collections/user-db/user-db.service'; +import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity'; +import { EImageFileBackend } from '../../database/entities/images/image-file.entity'; import { ImageFullIdParam } from '../../decorators/image-id/image-full-id.decorator'; import { ImageIdParam } from '../../decorators/image-id/image-id.decorator'; import { RequiredPermissions } from '../../decorators/permissions.decorator'; @@ -57,25 +59,23 @@ export class ImageController { @Query() params: ImageRequestParams, ): Promise { try { + let image: EImageFileBackend | EImageDerivativeBackend; if (fullid.variant === ImageEntryVariant.ORIGINAL) { - const image = ThrowIfFailed( - await this.imagesService.getOriginal(fullid.id), + image = ThrowIfFailed(await this.imagesService.getOriginal(fullid.id)); + } else { + image = ThrowIfFailed( + await this.imagesService.getConverted( + fullid.id, + fullid.filetype, + params, + ), ); - - res.type(ThrowIfFailed(FileType2Mime(image.filetype))); - return image.data; } - const image = ThrowIfFailed( - await this.imagesService.getConverted( - fullid.id, - fullid.filetype, - params, - ), - ); + const data = ThrowIfFailed(await this.imagesService.getFileData(image)); res.type(ThrowIfFailed(FileType2Mime(image.filetype))); - return image.data; + return data; } catch (e) { if (!IsFailure(e) || e.getType() !== FT.NotFound) throw e; diff --git a/frontend/src/app/services/api/api.service.ts b/frontend/src/app/services/api/api.service.ts index d7f9541..4e62c43 100644 --- a/frontend/src/app/services/api/api.service.ts +++ b/frontend/src/app/services/api/api.service.ts @@ -3,7 +3,7 @@ import { WINDOW } from '@ng-web-apis/common'; import axios, { AxiosRequestConfig, AxiosResponse, - AxiosResponseHeaders, + AxiosResponseHeaders } from 'axios'; import { ApiResponseSchema } from 'picsur-shared/dist/dto/api/api.dto'; import { FileType2Ext } from 'picsur-shared/dist/dto/mimes.dto'; @@ -13,7 +13,7 @@ import { Failure, FT, HasFailed, - HasSuccess, + HasSuccess } from 'picsur-shared/dist/types'; import { ZodDtoStatic } from 'picsur-shared/dist/util/create-zod-dto'; import { ParseMime2FileType } from 'picsur-shared/dist/util/parse-mime'; @@ -243,15 +243,13 @@ export class ApiService { uploadProgress.next((e.loaded / (e.total ?? 1000000)) * 100); }, signal: abortController.signal, + validateStatus: () => true, ...options, }); uploadProgress.complete(); downloadProgress.complete(); - if (result.status < 200 || result.status >= 300) { - return Fail(FT.Network, 'Recieved a non-ok response'); - } return result; } catch (e) { return Fail(FT.Network, e); diff --git a/shared/src/types/failable.ts b/shared/src/types/failable.ts index eb521a6..e07f77e 100644 --- a/shared/src/types/failable.ts +++ b/shared/src/types/failable.ts @@ -7,6 +7,7 @@ export enum FT { Unknown = 'unknown', Database = 'database', + S3 = 's3', SysValidation = 'sysvalidation', UsrValidation = 'usrvalidation', BadRequest = 'badrequest', @@ -51,6 +52,11 @@ const FTProps: { code: 500, message: 'A database error occurred', }, + [FT.S3]: { + important: true, + code: 500, + message: 'An S3 error occurred', + }, [FT.Network]: { important: true, code: 500, diff --git a/yarn.lock b/yarn.lock index 4a86dde..8a5c5da 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5555,10 +5555,17 @@ __metadata: languageName: node linkType: hard -"@types/validator@npm:^13.7.14": - version: 13.7.14 - resolution: "@types/validator@npm:13.7.14" - checksum: 51bd82cd08aa7d8006f97357b5768a77bfca30e4823b5962e63bbf6446f46b5afe236bec1089148a15fd04cc0a748a10e2dd1a559f07163ec5e4e9fb5581896e +"@types/uuid@npm:^9.0.0": + version: 9.0.0 + resolution: "@types/uuid@npm:9.0.0" + checksum: 59ae56d9547c8758588659da2a2b4c97cce79c2aae1798c892bb29452ef08e87859dea2ec3a66bfa88d0d2153147520be2b1893be920f9f0bc9c53a3207ea6aa + languageName: node + linkType: hard + +"@types/validator@npm:^13.7.10": + version: 13.7.10 + resolution: "@types/validator@npm:13.7.10" + checksum: 7b142c08019f484d62c9f3074231f640c24311558f157dd253a60810dd0cb29e41ec64ca210a192b54f6de51f4fe016bfeb2c30f90fa49c9337ed54a9d8e02aa languageName: node linkType: hard @@ -11567,8 +11574,9 @@ __metadata: "@types/semver": ^7.3.12 "@types/sharp": ^0.31.0 "@types/supertest": ^2.0.12 - "@typescript-eslint/eslint-plugin": ^5.55.0 - "@typescript-eslint/parser": ^5.55.0 + "@types/uuid": ^9.0.0 + "@typescript-eslint/eslint-plugin": ^5.47.0 + "@typescript-eslint/parser": ^5.47.0 bcrypt: ^5.1.0 bmp-img: ^1.2.1 cors: ^2.8.5