Add config for s3

backend/package.json

@@ -22,6 +22,7 @@
     "purge": "rm -rf dist && rm -rf node_modules"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.238.0",
     "@fastify/helmet": "^10.1.0",
     "@fastify/multipart": "^7.5.0",
     "@fastify/reply-from": "^9.0.1",

backend/src/collections/file-s3/file-s3.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { EarlyConfigModule } from '../../config/early/early-config.module';
+import { FileS3Service } from './file-s3.service';
+
+@Module({
+  imports: [EarlyConfigModule],
+  providers: [FileS3Service],
+  exports: [FileS3Service],
+})
+export class FileS3Module {}

backend/src/collections/file-s3/file-s3.service.ts

@@ -0,0 +1,122 @@
+import {
+  CreateBucketCommand,
+  DeleteObjectCommand,
+  DeleteObjectsCommand,
+  GetObjectCommand,
+  ListBucketsCommand,
+  PutObjectCommand,
+  S3Client,
+} from '@aws-sdk/client-s3';
+import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
+import { buffer as streamToBuffer } from 'get-stream';
+import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types';
+import { Readable } from 'stream';
+import { S3ConfigService } from '../../config/early/s3.config.service';
+
+@Injectable()
+export class FileS3Service implements OnModuleInit {
+  private readonly logger = new Logger(FileS3Service.name);
+
+  private S3: Promise<S3Client> = this.loadS3();
+
+  constructor(private readonly s3config: S3ConfigService) {}
+
+  onModuleInit() {
+    this.loadS3();
+  }
+
+  public async putFile(key: string, data: Buffer): AsyncFailable<string> {
+    const S3 = await this.S3;
+
+    const request = new PutObjectCommand({
+      Bucket: this.s3config.getS3Bucket(),
+      Key: key,
+      Body: data,
+    });
+
+    try {
+      await S3.send(request);
+      return key;
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
+  public async getFile(key: string): AsyncFailable<Buffer> {
+    const S3 = await this.S3;
+
+    const request = new GetObjectCommand({
+      Bucket: this.s3config.getS3Bucket(),
+      Key: key,
+    });
+
+    try {
+      const result = await S3.send(request);
+      if (!result.Body) return Fail(FT.NotFound, 'File not found');
+
+      if (result.Body instanceof Blob) {
+        return Buffer.from(await result.Body.arrayBuffer());
+      }
+      return streamToBuffer(result.Body as Readable);
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
+  public async deleteFile(key: string): AsyncFailable<true> {
+    const S3 = await this.S3;
+
+    const request = new DeleteObjectCommand({
+      Bucket: this.s3config.getS3Bucket(),
+      Key: key,
+    });
+
+    try {
+      await S3.send(request);
+      return true;
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
+  public async deleteFiles(keys: string[]): AsyncFailable<true> {
+    const S3 = await this.S3;
+
+    const request = new DeleteObjectsCommand({
+      Bucket: this.s3config.getS3Bucket(),
+      Delete: {
+        Objects: keys.map((key) => ({ Key: key })),
+      },
+    });
+
+    try {
+      await S3.send(request);
+      return true;
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
+  private async loadS3(): Promise<S3Client> {
+    const S3 = new S3Client(this.s3config.getS3Config());
+
+    try {
+      // Create bucket if it doesn't exist
+      const bucket = this.s3config.getS3Bucket();
+
+      // List buckets
+      const listBuckets = await S3.send(new ListBucketsCommand({}));
+
+      const bucketExists = listBuckets.Buckets?.some((b) => b.Name === bucket);
+      if (!bucketExists) {
+        this.logger.verbose(`Creating S3 Bucket ${bucket}`);
+        await S3.send(new CreateBucketCommand({ Bucket: bucket }));
+      } else {
+        this.logger.verbose(`Using existing S3 Bucket ${bucket}`);
+      }
+    } catch (e) {
+      this.logger.error(e);
+    }
+    return S3;
+  }
+}

backend/src/config/early/early-config.module.ts

@@ -5,6 +5,7 @@ import { EarlyJwtConfigService } from './early-jwt.config.service';
 import { HostConfigService } from './host.config.service';
 import { MultipartConfigService } from './multipart.config.service';
 import { RedisConfigService } from './redis.config.service';
+import { S3ConfigService } from './s3.config.service';
 import { ServeStaticConfigService } from './serve-static.config.service';
 import { TypeOrmConfigService } from './type-orm.config.service';
 
@@ -23,6 +24,7 @@ import { TypeOrmConfigService } from './type-orm.config.service';
     AuthConfigService,
     MultipartConfigService,
     RedisConfigService,
+    S3ConfigService,
   ],
   exports: [
     ConfigModule,
@@ -33,6 +35,7 @@ import { TypeOrmConfigService } from './type-orm.config.service';
     AuthConfigService,
     MultipartConfigService,
     RedisConfigService,
+    S3ConfigService,
   ],
 })
 export class EarlyConfigModule {}

backend/src/config/early/s3.config.service.ts

@@ -0,0 +1,73 @@
+import { S3ClientConfig } from '@aws-sdk/client-s3';
+import { Injectable, Logger } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { ParseString } from 'picsur-shared/dist/util/parse-simple';
+import { EnvPrefix } from '../config.static';
+
+@Injectable()
+export class S3ConfigService {
+  private readonly logger = new Logger(S3ConfigService.name);
+
+  constructor(private readonly configService: ConfigService) {
+    if (this.getS3Endpoint())
+      this.logger.log('Custom S3 Endpoint: ' + this.getS3Endpoint());
+
+    this.logger.log('S3 Region: ' + this.getS3Region());
+    this.logger.log('S3 Bucket: ' + this.getS3Bucket());
+
+    this.logger.verbose('S3 Access Key: ' + this.getS3AccessKey());
+    this.logger.verbose('S3 Secret Key: ' + this.getS3SecretKey());
+  }
+
+  public getS3Config(): S3ClientConfig {
+    return {
+      credentials: {
+        accessKeyId: this.getS3AccessKey(),
+        secretAccessKey: this.getS3SecretKey(),
+      },
+      endpoint: this.getS3Endpoint() ?? undefined,
+      region: this.getS3Region(),
+      tls: this.getS3TLS(),
+    };
+  }
+
+  public getS3Endpoint(): string | null {
+    return ParseString(this.configService.get(`${EnvPrefix}S3_ENDPOINT`), null);
+  }
+
+  public getS3TLS(): boolean | undefined {
+    const endpoint = this.getS3Endpoint();
+    if (endpoint) {
+      return endpoint.startsWith('https');
+    }
+    return undefined;
+  }
+
+  public getS3Bucket(): string {
+    return ParseString(
+      this.configService.get(`${EnvPrefix}S3_BUCKET`),
+      'picsur',
+    );
+  }
+
+  public getS3Region(): string {
+    return ParseString(
+      this.configService.get(`${EnvPrefix}S3_REGION`),
+      'us-east-1',
+    );
+  }
+
+  public getS3AccessKey(): string {
+    return ParseString(
+      this.configService.get(`${EnvPrefix}S3_ACCESS_KEY`),
+      'picsur',
+    );
+  }
+
+  public getS3SecretKey(): string {
+    return ParseString(
+      this.configService.get(`${EnvPrefix}S3_SECRET_KEY`),
+      'picsur',
+    );
+  }
+}

backend/src/config/early/type-orm.config.service.ts

@@ -6,6 +6,7 @@ import { EntityList } from '../../database/entities';
 import { MigrationList } from '../../database/migrations';
 import { DefaultName, EnvPrefix } from '../config.static';
 import { HostConfigService } from './host.config.service';
+import { RedisConfigService } from './redis.config.service';
 
 @Injectable()
 export class TypeOrmConfigService implements TypeOrmOptionsFactory {
@@ -13,6 +14,7 @@ export class TypeOrmConfigService implements TypeOrmOptionsFactory {
 
   constructor(
     private readonly configService: ConfigService,
+    private readonly redisConfig: RedisConfigService,
     private readonly hostService: HostConfigService,
   ) {
     const varOptions = this.getTypeOrmServerOptions();
@@ -66,6 +68,13 @@ export class TypeOrmConfigService implements TypeOrmOptionsFactory {
         entitiesDir: 'src/database/entities',
       },
 
+      // cache: {
+      //   duration: 60000,
+      //   type: 'ioredis',
+      //   alwaysEnabled: false,
+      //   options: this.redisConfig.getRedisUrl(),
+      // },
+
       ...varOptions,
     } as TypeOrmModuleOptions;
   }

backend/src/managers/image/image-manager.module.ts

@@ -38,6 +38,7 @@ export class ImageManagerModule implements OnModuleInit {
   private async imageManagerCron() {
     await this.cleanupDerivatives();
     await this.cleanupExpired();
+    await this.cleanupOrphanedFiles();
   }
 
   private async cleanupDerivatives() {
@@ -75,4 +76,25 @@ export class ImageManagerModule implements OnModuleInit {
     if (cleanedUp > 0)
       this.logger.log(`Cleaned up ${cleanedUp} expired images`);
   }
+
+  private async cleanupOrphanedFiles() {
+    // const cleanedUpDerivatives =
+    //   await this.imageFileDB.cleanupOrphanedDerivatives();
+
+    // if (HasFailed(cleanedUpDerivatives)) {
+    //   cleanedUpDerivatives.print(this.logger);
+    //   return;
+    // }
+
+    // const cleanedUpFiles = await this.imageFileDB.cleanupOrphanedFiles();
+    // if (HasFailed(cleanedUpFiles)) {
+    //   cleanedUpFiles.print(this.logger);
+    //   return;
+    // }
+
+    // if (cleanedUpDerivatives > 0 || cleanedUpFiles > 0)
+    //   this.logger.log(
+    //     `Cleaned up ${cleanedUpDerivatives} orphaned derivatives and ${cleanedUpFiles} orphaned files`,
+    //   );
+  }
 }

backend/src/workers/sharp/universal-sharp.ts

@@ -2,7 +2,7 @@ import { BMPdecode, BMPencode } from 'bmp-img';
 import {
   AnimFileType,
   FileType,
-  ImageFileType
+  ImageFileType,
 } from 'picsur-shared/dist/dto/mimes.dto';
 import { QOIdecode, QOIencode } from 'qoi-img';
 import sharp, { Sharp, SharpOptions } from 'sharp';

support/dev.docker-compose.yml

@@ -1,6 +1,7 @@
 version: '3'
 services:
   devdb:
+    container_name: devdb
     image: postgres:14-alpine
     environment:
       POSTGRES_DB: picsur
@@ -11,6 +12,23 @@ services:
     restart: unless-stopped
     volumes:
       - db-data:/var/lib/postgresql/data
-
+  devs3:
+    image: zenko/cloudserver:latest
+    container_name: devs3
+    environment:
+      S3BACKEND: file
+      S3DATAPATH: /storage/data/
+      S3METADATAPATH: /storage/metadata/
+      SCALITY_ACCESS_KEY_ID: username
+      SCALITY_SECRET_ACCESS_KEY: password
+      REMOTE_MANAGEMENT_DISABLE: 'true'
+    ports:
+      - '8000:8000'
+    restart: unless-stopped
+    volumes:
+      - s3-data:/storage/data
+      - s3-metadata:/storage/metadata
 volumes:
   db-data:
+  s3-data:
+  s3-metadata: