From 03fec5f832ac36309cd52f5ffba05e9ff52d2d17 Mon Sep 17 00:00:00 2001 From: rubikscraft Date: Tue, 6 Sep 2022 19:45:17 +0200 Subject: [PATCH] add expiring images to backend --- .../collections/image-db/image-db.service.ts | 71 +++++++++++-------- .../config/early/type-orm.config.service.ts | 2 +- .../src/database/entities/apikey.entity.ts | 2 + .../entities/image-derivative.entity.ts | 25 ++++++- .../database/entities/image-file.entity.ts | 23 +++++- backend/src/database/entities/image.entity.ts | 17 ++++- .../migrations/1662485374471-V_0_4_0_b.ts | 39 ++++++++++ backend/src/database/migrations/index.ts | 2 + backend/src/main.ts | 2 +- backend/src/managers/image/image.module.ts | 17 +++++ backend/src/managers/image/image.service.ts | 8 +++ .../routes/image/image-manage.controller.ts | 19 +++++ shared/src/dto/api/image-manage.dto.ts | 18 +++++ shared/src/entities/image.entity.ts | 1 + 14 files changed, 207 insertions(+), 39 deletions(-) create mode 100644 backend/src/database/migrations/1662485374471-V_0_4_0_b.ts diff --git a/backend/src/collections/image-db/image-db.service.ts b/backend/src/collections/image-db/image-db.service.ts index b27a4b0..2e0c473 100644 --- a/backend/src/collections/image-db/image-db.service.ts +++ b/backend/src/collections/image-db/image-db.service.ts @@ -3,9 +3,7 @@ import { InjectRepository } from '@nestjs/typeorm'; import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types'; import { FindResult } from 'picsur-shared/dist/types/find-result'; import { generateRandomString } from 'picsur-shared/dist/util/random'; -import { In, Repository } from 'typeorm'; -import { EImageDerivativeBackend } from '../../database/entities/image-derivative.entity'; -import { EImageFileBackend } from '../../database/entities/image-file.entity'; +import { In, LessThan, Repository } from 'typeorm'; import { EImageBackend } from '../../database/entities/image.entity'; @Injectable() @@ -13,12 +11,6 @@ export class ImageDBService { constructor( @InjectRepository(EImageBackend) private readonly imageRepo: Repository, - - @InjectRepository(EImageFileBackend) - private readonly imageFileRepo: Repository, - - @InjectRepository(EImageDerivativeBackend) - private readonly imageDerivativeRepo: Repository, ) {} public async create( @@ -91,6 +83,31 @@ export class ImageDBService { } } + public async update( + id: string, + userid: string | undefined, + options: Partial>, + ): AsyncFailable { + try { + const found = await this.imageRepo.findOne({ + where: { id, user_id: userid }, + }); + + if (!found) return Fail(FT.NotFound, 'Image not found'); + + if (options.file_name !== undefined) found.file_name = options.file_name; + + if (options.expires_at !== undefined) + found.expires_at = options.expires_at; + + await this.imageRepo.save(found); + + return found; + } catch (e) { + return Fail(FT.Database, e); + } + } + public async delete( ids: string[], userid: string | undefined, @@ -111,16 +128,7 @@ export class ImageDBService { if (available_ids.length === 0) return Fail(FT.NotFound, 'Images not found'); - await Promise.all([ - this.imageDerivativeRepo.delete({ - image_id: In(available_ids), - }), - this.imageFileRepo.delete({ - image_id: In(available_ids), - }), - - this.imageRepo.delete({ id: In(available_ids) }), - ]); + await this.imageRepo.delete({ id: In(available_ids) }); return deletable_images; } catch (e) { @@ -139,16 +147,7 @@ export class ImageDBService { if (!found) return Fail(FT.NotFound, 'Image not found'); - await Promise.all([ - this.imageDerivativeRepo.delete({ - image_id: found.id, - }), - this.imageFileRepo.delete({ - image_id: found.id, - }), - - this.imageRepo.delete({ id: found.id }), - ]); + await this.imageRepo.delete({ id: found.id }); return found; } catch (e) { @@ -164,12 +163,22 @@ export class ImageDBService { ); try { - await this.imageDerivativeRepo.delete({}); - await this.imageFileRepo.delete({}); await this.imageRepo.delete({}); } catch (e) { return Fail(FT.Database, e); } return true; } + + public async cleanupExpired(): AsyncFailable { + try { + const res = await this.imageRepo.delete({ + expires_at: LessThan(new Date()), + }); + + return res.affected ?? 0; + } catch (e) { + return Fail(FT.Database, e); + } + } } diff --git a/backend/src/config/early/type-orm.config.service.ts b/backend/src/config/early/type-orm.config.service.ts index 8cd5177..8a639ee 100644 --- a/backend/src/config/early/type-orm.config.service.ts +++ b/backend/src/config/early/type-orm.config.service.ts @@ -52,7 +52,7 @@ export class TypeOrmConfigService implements TypeOrmOptionsFactory { const varOptions = this.getTypeOrmServerOptions(); return { type: 'postgres' as 'postgres', - synchronize: !this.hostService.isProduction(), + synchronize: false, //!this.hostService.isProduction(), migrationsRun: true, diff --git a/backend/src/database/entities/apikey.entity.ts b/backend/src/database/entities/apikey.entity.ts index 7fa72e5..c425256 100644 --- a/backend/src/database/entities/apikey.entity.ts +++ b/backend/src/database/entities/apikey.entity.ts @@ -41,11 +41,13 @@ export class EApiKeyBackend< name: string; @Column({ + type: 'timestamp', nullable: false, }) created: Date; @Column({ + type: 'timestamp', nullable: true, }) last_used: Date; diff --git a/backend/src/database/entities/image-derivative.entity.ts b/backend/src/database/entities/image-derivative.entity.ts index 7e3f699..aa4f872 100644 --- a/backend/src/database/entities/image-derivative.entity.ts +++ b/backend/src/database/entities/image-derivative.entity.ts @@ -1,4 +1,13 @@ -import { Column, Entity, Index, PrimaryGeneratedColumn, Unique } from 'typeorm'; +import { + Column, + Entity, + Index, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, + Unique, +} from 'typeorm'; +import { EImageBackend } from './image.entity'; @Entity() @Unique(['image_id', 'key']) @@ -6,8 +15,18 @@ export class EImageDerivativeBackend { @PrimaryGeneratedColumn('uuid') private _id?: string; + // We do a little trickery @Index() - @Column({ nullable: false }) + @ManyToOne(() => EImageBackend, (image) => image.derivatives, { + nullable: false, + onDelete: 'CASCADE', + }) + @JoinColumn({ name: 'image_id' }) + private _image: any; + + @Column({ + name: 'image_id', + }) image_id: string; @Index() @@ -17,7 +36,7 @@ export class EImageDerivativeBackend { @Column({ nullable: false }) filetype: string; - @Column({ name: 'last_read', nullable: false }) + @Column({ type: 'timestamp', name: 'last_read', nullable: false }) last_read: Date; // Binary data diff --git a/backend/src/database/entities/image-file.entity.ts b/backend/src/database/entities/image-file.entity.ts index 7a7fab4..94f058b 100644 --- a/backend/src/database/entities/image-file.entity.ts +++ b/backend/src/database/entities/image-file.entity.ts @@ -1,5 +1,14 @@ import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum'; -import { Column, Entity, Index, PrimaryGeneratedColumn, Unique } from 'typeorm'; +import { + Column, + Entity, + Index, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, + Unique, +} from 'typeorm'; +import { EImageBackend } from './image.entity'; @Entity() @Unique(['image_id', 'variant']) @@ -7,8 +16,18 @@ export class EImageFileBackend { @PrimaryGeneratedColumn('uuid') private _id?: string; + // We do a little trickery @Index() - @Column({ nullable: false }) + @ManyToOne(() => EImageBackend, (image) => image.files, { + nullable: false, + onDelete: 'CASCADE', + }) + @JoinColumn({ name: 'image_id' }) + private _image: any; + + @Column({ + name: 'image_id' + }) image_id: string; @Index() diff --git a/backend/src/database/entities/image.entity.ts b/backend/src/database/entities/image.entity.ts index 1fd7cb0..cbf9955 100644 --- a/backend/src/database/entities/image.entity.ts +++ b/backend/src/database/entities/image.entity.ts @@ -1,5 +1,7 @@ import { EImage } from 'picsur-shared/dist/entities/image.entity'; -import { Column, Entity, PrimaryGeneratedColumn } from 'typeorm'; +import { Column, Entity, OneToMany, PrimaryGeneratedColumn } from 'typeorm'; +import { EImageDerivativeBackend } from './image-derivative.entity'; +import { EImageFileBackend } from './image-file.entity'; @Entity() export class EImageBackend implements EImage { @@ -12,6 +14,7 @@ export class EImageBackend implements EImage { user_id: string; @Column({ + type: 'timestamp', nullable: false, }) created: Date; @@ -22,9 +25,21 @@ export class EImageBackend implements EImage { }) file_name: string; + @Column({ + type: 'timestamp', + nullable: true, + }) + expires_at: Date | null; + @Column({ nullable: true, select: false, }) delete_key?: string; + + @OneToMany(() => EImageDerivativeBackend, (derivative) => derivative.image_id) + derivatives: EImageDerivativeBackend[]; + + @OneToMany(() => EImageFileBackend, (file) => file.image_id) + files: EImageFileBackend[]; } diff --git a/backend/src/database/migrations/1662485374471-V_0_4_0_b.ts b/backend/src/database/migrations/1662485374471-V_0_4_0_b.ts new file mode 100644 index 0000000..49a0b0f --- /dev/null +++ b/backend/src/database/migrations/1662485374471-V_0_4_0_b.ts @@ -0,0 +1,39 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class V040B1662485374471 implements MigrationInterface { + name = 'V040B1662485374471' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "e_image_backend" ADD "expires_at" TIMESTAMP`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a"`); + await queryRunner.query(`DROP INDEX "public"."IDX_8055f37d3b9f52f421b94ee84d"`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "image_id" SET DATA TYPE UUID USING image_id::uuid`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75"`); + await queryRunner.query(`DROP INDEX "public"."IDX_37055605f39b3f8847232d604f"`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "image_id" SET DATA TYPE UUID USING image_id::uuid`); + await queryRunner.query(`CREATE INDEX "IDX_8055f37d3b9f52f421b94ee84d" ON "e_image_file_backend" ("image_id") `); + await queryRunner.query(`CREATE INDEX "IDX_37055605f39b3f8847232d604f" ON "e_image_derivative_backend" ("image_id") `); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a" UNIQUE ("image_id", "variant")`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75" UNIQUE ("image_id", "key")`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "FK_37055605f39b3f8847232d604f8" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "FK_37055605f39b3f8847232d604f8"`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db"`); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75"`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a"`); + await queryRunner.query(`DROP INDEX "public"."IDX_37055605f39b3f8847232d604f"`); + await queryRunner.query(`DROP INDEX "public"."IDX_8055f37d3b9f52f421b94ee84d"`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "image_id" SET DATA TYPE character varying`); + await queryRunner.query(`CREATE INDEX "IDX_37055605f39b3f8847232d604f" ON "e_image_derivative_backend" ("image_id") `); + await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75" UNIQUE ("image_id", "key")`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP COLUMN "image_id"`); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "image_id" SET DATA TYPE character varying`); + await queryRunner.query(`CREATE INDEX "IDX_8055f37d3b9f52f421b94ee84d" ON "e_image_file_backend" ("image_id") `); + await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a" UNIQUE ("image_id", "variant")`); + await queryRunner.query(`ALTER TABLE "e_image_backend" DROP COLUMN "expires_at"`); + } + +} diff --git a/backend/src/database/migrations/index.ts b/backend/src/database/migrations/index.ts index 53dd346..7ddf102 100644 --- a/backend/src/database/migrations/index.ts +++ b/backend/src/database/migrations/index.ts @@ -1,9 +1,11 @@ import { V030A1661692206479 } from './1661692206479-V_0_3_0_a'; import { V032A1662029904716 } from './1662029904716-V_0_3_2_a'; import { V040A1662314197741 } from './1662314197741-V_0_4_0_a'; +import { V040B1662485374471 } from './1662485374471-V_0_4_0_b'; export const MigrationList: Function[] = [ V030A1661692206479, V032A1662029904716, V040A1662314197741, + V040B1662485374471, ]; diff --git a/backend/src/main.ts b/backend/src/main.ts index 299b6e1..c1aacad 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -27,7 +27,7 @@ async function bootstrap() { AppModule, fastifyAdapter, { - bufferLogs: true, + bufferLogs: false, }, ); diff --git a/backend/src/managers/image/image.module.ts b/backend/src/managers/image/image.module.ts index f306374..951c7a5 100644 --- a/backend/src/managers/image/image.module.ts +++ b/backend/src/managers/image/image.module.ts @@ -3,6 +3,7 @@ import ms from 'ms'; import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum'; import { HasFailed } from 'picsur-shared/dist/types'; import { ImageDBModule } from '../../collections/image-db/image-db.module'; +import { ImageDBService } from '../../collections/image-db/image-db.service'; import { ImageFileDBService } from '../../collections/image-db/image-file-db.service'; import { PreferenceDbModule } from '../../collections/preference-db/preference-db.module'; import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service'; @@ -26,6 +27,7 @@ export class ImageManagerModule implements OnModuleInit, OnModuleDestroy { constructor( private readonly prefManager: SysPreferenceDbService, private readonly imageFileDB: ImageFileDBService, + private readonly imageDB: ImageDBService, ) {} async onModuleInit() { @@ -38,6 +40,11 @@ export class ImageManagerModule implements OnModuleInit, OnModuleDestroy { } private async imageManagerCron() { + await this.cleanupDerivatives(); + await this.cleanupExpired(); + } + + private async cleanupDerivatives() { const remove_derivatives_after = await this.prefManager.getStringPreference( SysPreference.RemoveDerivativesAfter, ); @@ -60,6 +67,16 @@ export class ImageManagerModule implements OnModuleInit, OnModuleDestroy { this.logger.log(`Cleaned up ${result} derivatives`); } + private async cleanupExpired() { + const cleanedUp = await this.imageDB.cleanupExpired(); + + if (HasFailed(cleanedUp)) { + this.logger.warn(`Failed to cleanup expired images`); + } + + this.logger.log(`Cleaned up ${cleanedUp} expired images`); + } + onModuleDestroy() { if (this.interval) clearInterval(this.interval); } diff --git a/backend/src/managers/image/image.service.ts b/backend/src/managers/image/image.service.ts index c5d49ef..23bba17 100644 --- a/backend/src/managers/image/image.service.ts +++ b/backend/src/managers/image/image.service.ts @@ -52,6 +52,14 @@ export class ImageManagerService { return await this.imagesService.findMany(count, page, userid); } + public async update( + id: string, + userid: string | undefined, + options: Partial>, + ): AsyncFailable { + return await this.imagesService.update(id, userid, options); + } + public async deleteMany( ids: string[], userid: string | undefined, diff --git a/backend/src/routes/image/image-manage.controller.ts b/backend/src/routes/image/image-manage.controller.ts index fd30adc..cece5a5 100644 --- a/backend/src/routes/image/image-manage.controller.ts +++ b/backend/src/routes/image/image-manage.controller.ts @@ -15,6 +15,8 @@ import { ImageDeleteWithKeyResponse, ImageListRequest, ImageListResponse, + ImageUpdateRequest, + ImageUpdateResponse, ImageUploadResponse, } from 'picsur-shared/dist/dto/api/image-manage.dto'; import { Permission } from 'picsur-shared/dist/dto/permissions.enum'; @@ -73,6 +75,23 @@ export class ImageManageController { return found; } + @Post('update') + @RequiredPermissions(Permission.ImageManage) + @Returns(ImageUpdateResponse) + async updateImage( + @Body() body: ImageUpdateRequest, + @ReqUserID() userid: string, + @HasPermission(Permission.ImageAdmin) isImageAdmin: boolean, + ): Promise { + const user_id = isImageAdmin ? undefined : userid; + + const image = ThrowIfFailed( + await this.imagesService.update(body.id, user_id, body), + ); + + return image; + } + @Post('delete') @RequiredPermissions(Permission.ImageManage) @Returns(ImageDeleteResponse) diff --git a/shared/src/dto/api/image-manage.dto.ts b/shared/src/dto/api/image-manage.dto.ts index 9778965..de6757c 100644 --- a/shared/src/dto/api/image-manage.dto.ts +++ b/shared/src/dto/api/image-manage.dto.ts @@ -30,6 +30,24 @@ export const ImageListResponseSchema = z.object({ }); export class ImageListResponse extends createZodDto(ImageListResponseSchema) {} +// Image update +export const ImageUpdateRequestSchema = EImageSchema.pick({ + id: true, + expires_at: true, + file_name: true, +}).partial({ + expires_at: true, + file_name: true, +}); +export class ImageUpdateRequest extends createZodDto( + ImageUpdateRequestSchema, +) {} + +export const ImageUpdateResponseSchema = EImageSchema; +export class ImageUpdateResponse extends createZodDto( + ImageUpdateResponseSchema, +) {} + // Image Delete export const ImageDeleteRequestSchema = z.object({ diff --git a/shared/src/entities/image.entity.ts b/shared/src/entities/image.entity.ts index ec4fb82..c1b556e 100644 --- a/shared/src/entities/image.entity.ts +++ b/shared/src/entities/image.entity.ts @@ -6,5 +6,6 @@ export const EImageSchema = z.object({ user_id: IsEntityID(), created: z.preprocess((data: any) => new Date(data), z.date()), file_name: z.string(), + expires_at: z.preprocess((data: any) => new Date(data), z.date()).nullable(), }); export type EImage = z.infer;