Dockery/Picsur
1
0
Fork 0
mirror of https://github.com/CaramelFur/Picsur.git synced 2026-05-07 02:17:29 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add support for s3 storage

Browse Source
This commit is contained in:
rubikscraft
2022-12-30 16:34:04 +01:00
committed by Caramel
parent ada9fd8b4b
commit 743bd56722
17 changed files with 317 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
backend/package.json
View File

@@ -78,10 +78,11 @@
"@types/semver": "^7.3.12",
"@types/sharp": "^0.31.0",
"@types/supertest": "^2.0.12",
"@typescript-eslint/eslint-plugin": "^5.55.0",
"@typescript-eslint/parser": "^5.55.0",
"eslint": "^8.36.0",
"eslint-config-prettier": "^8.7.0",
"@types/uuid": "^9.0.0",
"@typescript-eslint/eslint-plugin": "^5.47.0",
"@typescript-eslint/parser": "^5.47.0",
"eslint": "^8.30.0",
"eslint-config-prettier": "^8.5.0",
"eslint-plugin-prettier": "^4.2.1",
"prettier": "^2.8.4",
"source-map-support": "^0.5.21",

12
backend/src/app.module.ts
View File

@@ -1,9 +1,10 @@
import { Logger, MiddlewareConsumer, Module, NestModule, OnModuleInit } from '@nestjs/common';
import { Logger, MiddlewareConsumer, Module, NestModule, OnApplicationBootstrap, OnApplicationShutdown } from '@nestjs/common';
import { ScheduleModule } from '@nestjs/schedule';
import { ServeStaticModule } from '@nestjs/serve-static';
import cors from 'cors';
import { IncomingMessage, ServerResponse } from 'http';
import semver from 'semver';
import { FileS3Module } from './collections/file-s3/file-s3.module';
import { EarlyConfigModule } from './config/early/early-config.module';
import { ServeStaticConfigService } from './config/early/serve-static.config.service';
import { DatabaseModule } from './database/database.module';
@@ -49,6 +50,7 @@ const imageCorsOverride = (
}),
ScheduleModule.forRoot(),
DatabaseModule,
FileS3Module,
AuthManagerModule,
UsageManagerModule,
DemoManagerModule,
@@ -56,7 +58,7 @@ const imageCorsOverride = (
PicsurLayersModule,
],
})
export class AppModule implements NestModule, OnModuleInit {
export class AppModule implements NestModule, OnApplicationBootstrap, OnApplicationShutdown {
private readonly logger = new Logger(AppModule.name);
configure(consumer: MiddlewareConsumer) {
@@ -64,7 +66,7 @@ export class AppModule implements NestModule, OnModuleInit {
consumer.apply(imageCorsConfig, imageCorsOverride).forRoutes('/i');
}
onModuleInit() {
onApplicationBootstrap() {
const nodeVersion = process.version;
if (!supportedNodeVersions.some((v) => semver.satisfies(nodeVersion, v))) {
this.logger.error(
@@ -76,4 +78,8 @@ export class AppModule implements NestModule, OnModuleInit {
);
}
}
onApplicationShutdown() {
this.logger.warn(`Shutting down`);
}
}

46
backend/src/collections/file-s3/file-s3.service.ts
View File

@@ -5,11 +5,15 @@ import {
GetObjectCommand,
ListBucketsCommand,
PutObjectCommand,
S3Client,
S3Client
} from '@aws-sdk/client-s3';
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { buffer as streamToBuffer } from 'get-stream';
import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types';
import {
AsyncFailable,
Fail, FT,
HasFailed
} from 'picsur-shared/dist/types';
import { Readable } from 'stream';
import { S3ConfigService } from '../../config/early/s3.config.service';
@@ -17,7 +21,7 @@ import { S3ConfigService } from '../../config/early/s3.config.service';
export class FileS3Service implements OnModuleInit {
private readonly logger = new Logger(FileS3Service.name);
private S3: Promise<S3Client> = this.loadS3();
private S3: S3Client | null = null;
constructor(private readonly s3config: S3ConfigService) {}
@@ -26,7 +30,8 @@ export class FileS3Service implements OnModuleInit {
}
public async putFile(key: string, data: Buffer): AsyncFailable<string> {
const S3 = await this.S3;
const S3 = await this.getS3();
if (HasFailed(S3)) return S3;
const request = new PutObjectCommand({
Bucket: this.s3config.getS3Bucket(),
@@ -38,12 +43,13 @@ export class FileS3Service implements OnModuleInit {
await S3.send(request);
return key;
} catch (e) {
return Fail(FT.Database, e);
return Fail(FT.S3, e);
}
}
public async getFile(key: string): AsyncFailable<Buffer> {
const S3 = await this.S3;
const S3 = await this.getS3();
if (HasFailed(S3)) return S3;
const request = new GetObjectCommand({
Bucket: this.s3config.getS3Bucket(),
@@ -59,12 +65,13 @@ export class FileS3Service implements OnModuleInit {
}
return streamToBuffer(result.Body as Readable);
} catch (e) {
return Fail(FT.Database, e);
return Fail(FT.S3, e);
}
}
public async deleteFile(key: string): AsyncFailable<true> {
const S3 = await this.S3;
const S3 = await this.getS3();
if (HasFailed(S3)) return S3;
const request = new DeleteObjectCommand({
Bucket: this.s3config.getS3Bucket(),
@@ -75,12 +82,13 @@ export class FileS3Service implements OnModuleInit {
await S3.send(request);
return true;
} catch (e) {
return Fail(FT.Database, e);
return Fail(FT.S3, e);
}
}
public async deleteFiles(keys: string[]): AsyncFailable<true> {
const S3 = await this.S3;
const S3 = await this.getS3();
if (HasFailed(S3)) return S3;
const request = new DeleteObjectsCommand({
Bucket: this.s3config.getS3Bucket(),
@@ -93,11 +101,18 @@ export class FileS3Service implements OnModuleInit {
await S3.send(request);
return true;
} catch (e) {
return Fail(FT.Database, e);
return Fail(FT.S3, e);
}
}
private async loadS3(): Promise<S3Client> {
private async getS3(): AsyncFailable<S3Client> {
if (this.S3) return this.S3;
await this.loadS3();
if (this.S3) return this.S3;
return Fail(FT.S3, 'S3 not loaded');
}
private async loadS3(): Promise<void> {
const S3 = new S3Client(this.s3config.getS3Config());
try {
@@ -114,9 +129,14 @@ export class FileS3Service implements OnModuleInit {
} else {
this.logger.verbose(`Using existing S3 Bucket ${bucket}`);
}
this.S3 = S3;
} catch (e) {
this.logger.error(e);
this.logger.warn(
'There was an error setting up S3, are you sure you have set up an S3 instance and configured it correctly?\n' +
'Please check https://github.com/rubikscraft/picsur for up to date documentation.',
);
}
return S3;
}
}

2
backend/src/collections/image-db/image-db.module.ts
View File

@@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
import { EImageBackend } from '../../database/entities/images/image.entity';
import { FileS3Module } from '../file-s3/file-s3.module';
import { ImageDBService } from './image-db.service';
import { ImageFileDBService } from './image-file-db.service';
@@ -13,6 +14,7 @@ import { ImageFileDBService } from './image-file-db.service';
EImageFileBackend,
EImageDerivativeBackend,
]),
FileS3Module
],
providers: [ImageDBService, ImageFileDBService],
exports: [ImageDBService, ImageFileDBService],

102
backend/src/collections/image-db/image-file-db.service.ts
View File

@@ -2,9 +2,11 @@ import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
import { LessThan, Repository } from 'typeorm';
import { In, IsNull, LessThan, Repository } from 'typeorm';
import { v4 as uuidv4 } from 'uuid';
import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
import { FileS3Service } from '../file-s3/file-s3.service';
const A_DAY_IN_SECONDS = 24 * 60 * 60;
@@ -16,24 +18,61 @@ export class ImageFileDBService {
@InjectRepository(EImageDerivativeBackend)
private readonly imageDerivativeRepo: Repository<EImageDerivativeBackend>,
private readonly s3Service: FileS3Service,
) {}
public async getFileData(
file: EImageFileBackend | EImageDerivativeBackend,
): AsyncFailable<Buffer> {
if (file.data !== null) {
// Migrate files from old format to s3
const data = file.data;
const s3result = await this.s3Service.putFile(file.fileKey, data);
if (HasFailed(s3result)) return s3result;
file.data = null;
let repoResult: EImageFileBackend | EImageDerivativeBackend;
if (file instanceof EImageFileBackend) {
repoResult = await this.imageFileRepo.save(file);
} else if (file instanceof EImageDerivativeBackend) {
repoResult = await this.imageDerivativeRepo.save(file);
} else {
return Fail(FT.SysValidation, 'Invalid file type');
}
if (HasFailed(repoResult)) return repoResult;
return data;
}
const result = await this.s3Service.getFile(file.fileKey);
if (HasFailed(result)) return result;
return result;
}
public async setFile(
imageId: string,
variant: ImageEntryVariant,
file: Buffer,
filetype: string,
): AsyncFailable<true> {
const s3key = uuidv4();
const imageFile = new EImageFileBackend();
imageFile.image_id = imageId;
imageFile.variant = variant;
imageFile.filetype = filetype;
imageFile.data = file;
imageFile.fileKey = s3key;
try {
await this.imageFileRepo.upsert(imageFile, {
conflictPaths: ['image_id', 'variant'],
});
const s3result = await this.s3Service.putFile(s3key, file);
if (HasFailed(s3result)) return s3result;
} catch (e) {
return Fail(FT.Database, e);
}
@@ -84,6 +123,9 @@ export class ImageFileDBService {
if (!found) return Fail(FT.NotFound, 'Image not found');
const s3result = await this.s3Service.deleteFile(found.fileKey);
if (HasFailed(s3result)) return s3result;
await this.imageFileRepo.delete({ image_id: imageId, variant: variant });
return found;
} catch (e) {
@@ -120,15 +162,22 @@ export class ImageFileDBService {
filetype: string,
file: Buffer,
): AsyncFailable<EImageDerivativeBackend> {
const s3key = uuidv4();
const imageDerivative = new EImageDerivativeBackend();
imageDerivative.image_id = imageId;
imageDerivative.key = key;
imageDerivative.filetype = filetype;
imageDerivative.data = file;
imageDerivative.fileKey = s3key;
imageDerivative.last_read = new Date();
try {
return await this.imageDerivativeRepo.save(imageDerivative);
const result = await this.imageDerivativeRepo.save(imageDerivative);
const s3result = await this.s3Service.putFile(s3key, file);
if (HasFailed(s3result)) return s3result;
return result;
} catch (e) {
return Fail(FT.Database, e);
}
@@ -171,4 +220,49 @@ export class ImageFileDBService {
return Fail(FT.Database, e);
}
}
public async cleanupOrphanedDerivatives(): AsyncFailable<number> {
return this.cleanupRepoWithFilekey(this.imageDerivativeRepo);
}
public async cleanupOrphanedFiles(): AsyncFailable<number> {
return this.cleanupRepoWithFilekey(this.imageFileRepo);
}
// Go over all image files in the db, and any that are not linked to an image are deleted from s3 and the db
private async cleanupRepoWithFilekey(
repo: Repository<{ image_id: string | null; fileKey: string }>,
): AsyncFailable<number> {
try {
let remaining = Infinity;
let processed = 0;
while (remaining > 0) {
const orphaned = await repo.findAndCount({
where: {
image_id: IsNull(),
},
select: ['fileKey'],
take: 100,
});
if (orphaned[1] === 0) break;
remaining = orphaned[1] - orphaned[0].length;
const keys = orphaned[0].map((d) => d.fileKey);
const s3result = await this.s3Service.deleteFiles(keys);
if (HasFailed(s3result)) return s3result;
const result = await repo.delete({
fileKey: In(keys),
});
processed += result.affected ?? 0;
}
return processed;
} catch (e) {
return Fail(FT.Database, e);
}
}
}

27
backend/src/database/entities/images/image-derivative.entity.ts
View File

@@ -3,39 +3,42 @@ import {
Entity,
Index,
JoinColumn,
ManyToOne,
PrimaryGeneratedColumn,
Unique,
ManyToOne, PrimaryColumn, Unique
} from 'typeorm';
import { EImageBackend } from './image.entity';
@Entity()
@Unique(['image_id', 'key'])
export class EImageDerivativeBackend {
@PrimaryGeneratedColumn('uuid')
private _id?: string;
@PrimaryColumn({ type: 'uuid', nullable: false, name: '_id' })
@Index()
fileKey: string;
// We do a little trickery
// == Reference to parent image
@Index()
@ManyToOne(() => EImageBackend, (image) => image.derivatives, {
nullable: false,
onDelete: 'CASCADE',
nullable: true,
onDelete: 'SET NULL',
})
@JoinColumn({ name: 'image_id' })
private _image?: any;
@Column({
name: 'image_id',
nullable: true,
})
image_id: string;
image_id: string | null;
// == Derivative options hash
@Index()
@Column({ nullable: false })
key: string;
// == Filetype of the derivative
@Column({ nullable: false })
filetype: string;
// == Last time the derivative was read
@Column({
type: 'timestamptz',
name: 'last_read',
@@ -43,7 +46,7 @@ export class EImageDerivativeBackend {
})
last_read: Date;
// Binary data
@Column({ type: 'bytea', nullable: false })
data: Buffer;
// == Binary data
@Column({ type: 'bytea', nullable: true })
data: Buffer | null;
}

25
backend/src/database/entities/images/image-file.entity.ts
View File

@@ -5,39 +5,42 @@ import {
Index,
JoinColumn,
ManyToOne,
PrimaryGeneratedColumn,
Unique,
PrimaryColumn, Unique
} from 'typeorm';
import { EImageBackend } from './image.entity';
@Entity()
@Unique(['image_id', 'variant'])
export class EImageFileBackend {
@PrimaryGeneratedColumn('uuid')
private _id?: string;
@PrimaryColumn({ type: 'uuid', nullable: false, name: '_id' })
@Index()
fileKey: string;
// We do a little trickery
// == Reference to parent image
@Index()
@ManyToOne(() => EImageBackend, (image) => image.files, {
nullable: false,
onDelete: 'CASCADE',
nullable: true,
onDelete: 'SET NULL',
})
@JoinColumn({ name: 'image_id' })
private _image?: any;
@Column({
name: 'image_id',
nullable: true,
})
image_id: string;
image_id: string | null;
// == File variant
@Index()
@Column({ nullable: false, enum: ImageEntryVariant })
variant: ImageEntryVariant;
// == Filetype of the derivative
@Column({ nullable: false })
filetype: string;
// Binary data
@Column({ type: 'bytea', nullable: false })
data: Buffer;
// == Binary data
@Column({ type: 'bytea', nullable: true })
data: Buffer | null;
}

44
backend/src/database/migrations/1672247794308-V_0_6_0_a.ts Normal file
View File

@@ -0,0 +1,44 @@
import { MigrationInterface, QueryRunner } from "typeorm";
export class V060A1672247794308 implements MigrationInterface {
name = 'V060A1672247794308'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db"`);
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "FK_37055605f39b3f8847232d604f8"`);
await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a"`);
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "_id" DROP DEFAULT`);
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "image_id" DROP NOT NULL`);
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "data" DROP NOT NULL`);
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75"`);
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "_id" DROP DEFAULT`);
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "image_id" DROP NOT NULL`);
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "data" DROP NOT NULL`);
await queryRunner.query(`CREATE INDEX "IDX_95953be58a506e5de46feec618" ON "e_image_file_backend" ("_id") `);
await queryRunner.query(`CREATE INDEX "IDX_ff1ecff935b8d7bdcea8908781" ON "e_image_derivative_backend" ("_id") `);
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a" UNIQUE ("image_id", "variant")`);
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75" UNIQUE ("image_id", "key")`);
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE SET NULL ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "FK_37055605f39b3f8847232d604f8" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE SET NULL ON UPDATE NO ACTION`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "FK_37055605f39b3f8847232d604f8"`);
await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db"`);
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75"`);
await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a"`);
await queryRunner.query(`DROP INDEX "public"."IDX_ff1ecff935b8d7bdcea8908781"`);
await queryRunner.query(`DROP INDEX "public"."IDX_95953be58a506e5de46feec618"`);
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "data" SET NOT NULL`);
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "image_id" SET NOT NULL`);
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "_id" SET DEFAULT uuid_generate_v4()`);
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75" UNIQUE ("image_id", "key")`);
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "data" SET NOT NULL`);
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "image_id" SET NOT NULL`);
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "_id" SET DEFAULT uuid_generate_v4()`);
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a" UNIQUE ("image_id", "variant")`);
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "FK_37055605f39b3f8847232d604f8" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
}
}

2
backend/src/database/migrations/index.ts
View File

@@ -5,6 +5,7 @@ import { V040B1662485374471 } from './1662485374471-V_0_4_0_b';
import { V040C1662535484200 } from './1662535484200-V_0_4_0_c';
import { V040D1662728275448 } from './1662728275448-V_0_4_0_d';
import { V050A1672154027079 } from './1672154027079-V_0_5_0_a';
import { V060A1672247794308 } from './1672247794308-V_0_6_0_a';
export const MigrationList: Function[] = [
V030A1661692206479,
@@ -14,4 +15,5 @@ export const MigrationList: Function[] = [
V040C1662535484200,
V040D1662728275448,
V050A1672154027079,
V060A1672247794308
];

4
backend/src/main.ts
View File

@@ -4,7 +4,7 @@ import fastifyReplyFrom from '@fastify/reply-from';
import { NestFactory } from '@nestjs/core';
import {
FastifyAdapter,
NestFastifyApplication,
NestFastifyApplication
} from '@nestjs/platform-fastify';
import { AppModule } from './app.module';
import { HostConfigService } from './config/early/host.config.service';
@@ -43,6 +43,8 @@ async function bootstrap() {
},
);
app.enableShutdownHooks();
// Configure logger
app.useLogger(app.get(PicsurLoggerService));
app.flushLogs();

31
backend/src/managers/image/image-manager.module.ts
View File

@@ -39,6 +39,7 @@ export class ImageManagerModule implements OnModuleInit {
await this.cleanupDerivatives();
await this.cleanupExpired();
await this.cleanupOrphanedFiles();
// TODO: Auto migrate all images to S3
}
private async cleanupDerivatives() {
@@ -78,23 +79,23 @@ export class ImageManagerModule implements OnModuleInit {
}
private async cleanupOrphanedFiles() {
// const cleanedUpDerivatives =
// await this.imageFileDB.cleanupOrphanedDerivatives();
const cleanedUpDerivatives =
await this.imageFileDB.cleanupOrphanedDerivatives();
// if (HasFailed(cleanedUpDerivatives)) {
// cleanedUpDerivatives.print(this.logger);
// return;
// }
if (HasFailed(cleanedUpDerivatives)) {
cleanedUpDerivatives.print(this.logger);
return;
}
// const cleanedUpFiles = await this.imageFileDB.cleanupOrphanedFiles();
// if (HasFailed(cleanedUpFiles)) {
// cleanedUpFiles.print(this.logger);
// return;
// }
const cleanedUpFiles = await this.imageFileDB.cleanupOrphanedFiles();
if (HasFailed(cleanedUpFiles)) {
cleanedUpFiles.print(this.logger);
return;
}
// if (cleanedUpDerivatives > 0 || cleanedUpFiles > 0)
// this.logger.log(
// `Cleaned up ${cleanedUpDerivatives} orphaned derivatives and ${cleanedUpFiles} orphaned files`,
// );
if (cleanedUpDerivatives > 0 || cleanedUpFiles > 0)
this.logger.log(
`Cleaned up ${cleanedUpDerivatives} orphaned derivatives and ${cleanedUpFiles} orphaned files`,
);
}
}

43
backend/src/managers/image/image.service.ts
View File

@@ -7,7 +7,7 @@ import {
AnimFileType,
FileType,
ImageFileType,
Mime2FileType,
Mime2FileType
} from 'picsur-shared/dist/dto/mimes.dto';
import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.enum';
@@ -57,11 +57,13 @@ export class ImageManagerService {
userid: string | undefined,
options: Partial<Pick<EImageBackend, 'file_name' | 'expires_at'>>,
): AsyncFailable<EImageBackend> {
if (options.expires_at !== undefined && options.expires_at !== null) {
if (options.expires_at < new Date()) {
return Fail(FT.UsrValidation, 'Expiration date must be in the future');
}
}
if (
options.expires_at !== undefined &&
options.expires_at !== null &&
options.expires_at < new Date()
)
return Fail(FT.UsrValidation, 'Expiration date must be in the future');
return await this.imagesService.update(id, userid, options);
}
@@ -114,13 +116,24 @@ export class ImageManagerService {
);
if (HasFailed(imageEntity)) return imageEntity;
const onFail = async () => {
const result = await this.imagesService.delete(
[imageEntity.id],
undefined,
);
if (HasFailed(result)) result.print(this.logger);
};
const imageFileEntity = await this.imageFilesService.setFile(
imageEntity.id,
ImageEntryVariant.MASTER,
processResult.image,
processResult.filetype,
);
if (HasFailed(imageFileEntity)) return imageFileEntity;
if (HasFailed(imageFileEntity)) {
await onFail();
return imageFileEntity;
}
if (keepOriginal) {
const originalFileEntity = await this.imageFilesService.setFile(
@@ -129,7 +142,10 @@ export class ImageManagerService {
image,
fileType.identifier,
);
if (HasFailed(originalFileEntity)) return originalFileEntity;
if (HasFailed(originalFileEntity)) {
await onFail();
return originalFileEntity;
}
}
return imageEntity;
@@ -162,9 +178,12 @@ export class ImageManagerService {
const sourceFileType = ParseFileType(masterImage.filetype);
if (HasFailed(sourceFileType)) return sourceFileType;
const data = await this.imageFilesService.getFileData(masterImage);
if (HasFailed(data)) return data;
const startTime = Date.now();
const convertResult = await this.convertService.convert(
masterImage.data,
data,
sourceFileType,
targetFileType,
allow_editing ? options : {},
@@ -234,6 +253,12 @@ export class ImageManagerService {
};
}
public async getFileData(
file: EImageFileBackend | EImageDerivativeBackend,
): AsyncFailable<Buffer> {
return this.imageFilesService.getFileData(file);
}
// Util stuff ==================================================================
private async getFileTypeFromBuffer(image: Buffer): AsyncFailable<FileType> {

10
backend/src/routes/image/image-manage.controller.ts
View File

@@ -5,7 +5,7 @@ import {
Logger,
Param,
Post,
Res,
Res
} from '@nestjs/common';
import { Throttle } from '@nestjs/throttler';
import type { FastifyReply } from 'fastify';
@@ -18,7 +18,7 @@ import {
ImageListResponse,
ImageUpdateRequest,
ImageUpdateResponse,
ImageUploadResponse,
ImageUploadResponse
} from 'picsur-shared/dist/dto/api/image-manage.dto';
import { Permission } from 'picsur-shared/dist/dto/permissions.enum';
import { Fail, FT, HasFailed, ThrowIfFailed } from 'picsur-shared/dist/types';
@@ -26,7 +26,7 @@ import { PostFiles } from '../../decorators/multipart/multipart.decorator';
import type { FileIterator } from '../../decorators/multipart/postfiles.pipe';
import {
HasPermission,
RequiredPermissions,
RequiredPermissions
} from '../../decorators/permissions.decorator';
import { ReqUserID } from '../../decorators/request-user.decorator';
import { Returns } from '../../decorators/returns.decorator';
@@ -91,14 +91,14 @@ export class ImageManageController {
@RequiredPermissions(Permission.ImageManage)
@Returns(ImageUpdateResponse)
async updateImage(
@Body() body: ImageUpdateRequest,
@Body() options: ImageUpdateRequest,
@ReqUserID() userid: string,
@HasPermission(Permission.ImageAdmin) isImageAdmin: boolean,
): Promise<ImageUpdateResponse> {
const user_id = isImageAdmin ? undefined : userid;
const image = ThrowIfFailed(
await this.imagesService.update(body.id, user_id, body),
await this.imagesService.update(options.id, user_id, options),
);
return image;

28
backend/src/routes/image/image.controller.ts
View File

@@ -3,12 +3,14 @@ import { SkipThrottle } from '@nestjs/throttler';
import type { FastifyReply } from 'fastify';
import {
ImageMetaResponse,
ImageRequestParams,
ImageRequestParams
} from 'picsur-shared/dist/dto/api/image.dto';
import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
import { FileType2Mime } from 'picsur-shared/dist/dto/mimes.dto';
import { FT, IsFailure, ThrowIfFailed } from 'picsur-shared/dist/types';
import { UserDbService } from '../../collections/user-db/user-db.service';
import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
import { ImageFullIdParam } from '../../decorators/image-id/image-full-id.decorator';
import { ImageIdParam } from '../../decorators/image-id/image-id.decorator';
import { RequiredPermissions } from '../../decorators/permissions.decorator';
@@ -57,25 +59,23 @@ export class ImageController {
@Query() params: ImageRequestParams,
): Promise<Buffer> {
try {
let image: EImageFileBackend | EImageDerivativeBackend;
if (fullid.variant === ImageEntryVariant.ORIGINAL) {
const image = ThrowIfFailed(
await this.imagesService.getOriginal(fullid.id),
image = ThrowIfFailed(await this.imagesService.getOriginal(fullid.id));
} else {
image = ThrowIfFailed(
await this.imagesService.getConverted(
fullid.id,
fullid.filetype,
params,
),
);
res.type(ThrowIfFailed(FileType2Mime(image.filetype)));
return image.data;
}
const image = ThrowIfFailed(
await this.imagesService.getConverted(
fullid.id,
fullid.filetype,
params,
),
);
const data = ThrowIfFailed(await this.imagesService.getFileData(image));
res.type(ThrowIfFailed(FileType2Mime(image.filetype)));
return image.data;
return data;
} catch (e) {
if (!IsFailure(e) || e.getType() !== FT.NotFound) throw e;

8
frontend/src/app/services/api/api.service.ts
View File

@@ -3,7 +3,7 @@ import { WINDOW } from '@ng-web-apis/common';
import axios, {
AxiosRequestConfig,
AxiosResponse,
AxiosResponseHeaders,
AxiosResponseHeaders
} from 'axios';
import { ApiResponseSchema } from 'picsur-shared/dist/dto/api/api.dto';
import { FileType2Ext } from 'picsur-shared/dist/dto/mimes.dto';
@@ -13,7 +13,7 @@ import {
Failure,
FT,
HasFailed,
HasSuccess,
HasSuccess
} from 'picsur-shared/dist/types';
import { ZodDtoStatic } from 'picsur-shared/dist/util/create-zod-dto';
import { ParseMime2FileType } from 'picsur-shared/dist/util/parse-mime';
@@ -243,15 +243,13 @@ export class ApiService {
uploadProgress.next((e.loaded / (e.total ?? 1000000)) * 100);
},
signal: abortController.signal,
validateStatus: () => true,
...options,
});
uploadProgress.complete();
downloadProgress.complete();
if (result.status < 200 || result.status >= 300) {
return Fail(FT.Network, 'Recieved a non-ok response');
}
return result;
} catch (e) {
return Fail(FT.Network, e);

6
shared/src/types/failable.ts
View File

@@ -7,6 +7,7 @@
export enum FT {
Unknown = 'unknown',
Database = 'database',
S3 = 's3',
SysValidation = 'sysvalidation',
UsrValidation = 'usrvalidation',
BadRequest = 'badrequest',
@@ -51,6 +52,11 @@ const FTProps: {
code: 500,
message: 'A database error occurred',
},
[FT.S3]: {
important: true,
code: 500,
message: 'An S3 error occurred',
},
[FT.Network]: {
important: true,
code: 500,

20
yarn.lock
View File

@@ -5555,10 +5555,17 @@ __metadata:
languageName: node
linkType: hard
"@types/validator@npm:^13.7.14":
version: 13.7.14
resolution: "@types/validator@npm:13.7.14"
checksum: 51bd82cd08aa7d8006f97357b5768a77bfca30e4823b5962e63bbf6446f46b5afe236bec1089148a15fd04cc0a748a10e2dd1a559f07163ec5e4e9fb5581896e
"@types/uuid@npm:^9.0.0":
version: 9.0.0
resolution: "@types/uuid@npm:9.0.0"
checksum: 59ae56d9547c8758588659da2a2b4c97cce79c2aae1798c892bb29452ef08e87859dea2ec3a66bfa88d0d2153147520be2b1893be920f9f0bc9c53a3207ea6aa
languageName: node
linkType: hard
"@types/validator@npm:^13.7.10":
version: 13.7.10
resolution: "@types/validator@npm:13.7.10"
checksum: 7b142c08019f484d62c9f3074231f640c24311558f157dd253a60810dd0cb29e41ec64ca210a192b54f6de51f4fe016bfeb2c30f90fa49c9337ed54a9d8e02aa
languageName: node
linkType: hard
@@ -11567,8 +11574,9 @@ __metadata:
"@types/semver": ^7.3.12
"@types/sharp": ^0.31.0
"@types/supertest": ^2.0.12
"@typescript-eslint/eslint-plugin": ^5.55.0
"@typescript-eslint/parser": ^5.55.0
"@types/uuid": ^9.0.0
"@typescript-eslint/eslint-plugin": ^5.47.0
"@typescript-eslint/parser": ^5.47.0
bcrypt: ^5.1.0
bmp-img: ^1.2.1
cors: ^2.8.5