From 6a2e48dacb45a3b3e311ea5312f52066c3ff8420 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sun, 19 Apr 2026 22:28:27 +0300 Subject: [PATCH] feat(standalone): migrate to SAHPool for faster DB --- .../src/lightweight/sql_provider.ts | 130 ++++++++++++-- .../src/local-server-worker.ts | 167 +++++++++++++++--- 2 files changed, 258 insertions(+), 39 deletions(-) diff --git a/apps/client-standalone/src/lightweight/sql_provider.ts b/apps/client-standalone/src/lightweight/sql_provider.ts index 972b58db0d..7a3c8ff1ec 100644 --- a/apps/client-standalone/src/lightweight/sql_provider.ts +++ b/apps/client-standalone/src/lightweight/sql_provider.ts @@ -1,4 +1,4 @@ -import { type BindableValue, default as sqlite3InitModule } from "@sqlite.org/sqlite-wasm"; +import { type BindableValue, type SAHPoolUtil, default as sqlite3InitModule } from "@sqlite.org/sqlite-wasm"; import type { DatabaseProvider, RunResult, Statement, Transaction } from "@triliumnext/core"; // Type definitions for SQLite WASM (the library doesn't export these directly) @@ -227,6 +227,10 @@ export default class BrowserSqlProvider implements DatabaseProvider { // OPFS state tracking private opfsDbPath?: string; + // SAHPool state tracking + private sahPoolUtil?: SAHPoolUtil; + private sahPoolDbName?: string; + /** * Get the SQLite WASM module version info. * Returns undefined if the module hasn't been initialized yet. @@ -287,16 +291,111 @@ export default class BrowserSqlProvider implements DatabaseProvider { return this.sqlite3 !== undefined; } - // ==================== OPFS Support ==================== + // ==================== SAHPool VFS (preferred OPFS backend) ==================== /** - * Check if the OPFS VFS is available. + * Install the OPFS SAHPool VFS. This pre-allocates a pool of OPFS + * SyncAccessHandle objects, enabling WAL mode and significantly faster + * writes compared to the legacy OPFS VFS. + * + * Must be called after `initWasm()` and before `loadFromSahPool()`. + * This is async because it acquires OPFS file handles. + * + * @param options.directory - OPFS directory for the pool (default: auto-derived from VFS name) + * @param options.initialCapacity - Minimum number of file slots (default: 6) + * @throws Error if the environment doesn't support SAHPool (no OPFS, no Worker, no COOP/COEP) + */ + async installSahPool(options: { directory?: string; initialCapacity?: number } = {}): Promise { + this.ensureSqlite3(); + + console.log("[BrowserSqlProvider] Installing OPFS SAHPool VFS..."); + const startTime = performance.now(); + + this.sahPoolUtil = await this.sqlite3!.installOpfsSAHPoolVfs({ + clearOnInit: false, + initialCapacity: options.initialCapacity ?? 6, + directory: options.directory, + }); + + // Ensure enough slots for DB + WAL + journal + temp files + await this.sahPoolUtil.reserveMinimumCapacity(options.initialCapacity ?? 6); + + const initTime = performance.now() - startTime; + console.log( + `[BrowserSqlProvider] SAHPool VFS installed in ${initTime.toFixed(2)}ms ` + + `(capacity: ${this.sahPoolUtil.getCapacity()}, files: ${this.sahPoolUtil.getFileCount()})` + ); + } + + /** + * Whether the SAHPool VFS has been successfully installed. + */ + get isSahPoolInstalled(): boolean { + return this.sahPoolUtil !== undefined; + } + + /** + * Access the SAHPool utility for advanced operations (import/export/migration). + */ + get sahPool(): SAHPoolUtil | undefined { + return this.sahPoolUtil; + } + + /** + * Load or create a database using the SAHPool VFS. + * This is the preferred method for persistent storage — it supports WAL mode + * and is significantly faster than the legacy OPFS VFS. + * + * @param dbName - Virtual filename within the pool (e.g., "/trilium.db"). + * Must start with a slash. + * @throws Error if SAHPool VFS is not installed + */ + loadFromSahPool(dbName: string): void { + this.ensureSqlite3(); + if (!this.sahPoolUtil) { + throw new Error( + "SAHPool VFS not installed. Call installSahPool() first." + ); + } + + console.log(`[BrowserSqlProvider] Loading database from SAHPool: ${dbName}`); + const startTime = performance.now(); + + try { + this.db = new this.sahPoolUtil.OpfsSAHPoolDb(dbName); + this.sahPoolDbName = dbName; + this.opfsDbPath = undefined; + + // SAHPool supports WAL mode — the key advantage over legacy OPFS VFS + this.db.exec("PRAGMA journal_mode = WAL"); + this.db.exec("PRAGMA synchronous = NORMAL"); + + const loadTime = performance.now() - startTime; + console.log(`[BrowserSqlProvider] SAHPool database loaded in ${loadTime.toFixed(2)}ms (WAL mode)`); + } catch (e) { + const error = e instanceof Error ? e : new Error(String(e)); + console.error(`[BrowserSqlProvider] Failed to load SAHPool database: ${error.message}`); + throw error; + } + } + + /** + * Whether the currently open database is using the SAHPool VFS. + */ + get isUsingSahPool(): boolean { + return this.sahPoolDbName !== undefined; + } + + // ==================== Legacy OPFS Support ==================== + + /** + * Check if the legacy OPFS VFS is available. * This requires: * - Running in a Worker context * - Browser support for OPFS APIs * - COOP/COEP headers sent by the server (for SharedArrayBuffer) * - * @returns true if OPFS VFS is available for use + * @returns true if legacy OPFS VFS is available for use */ isOpfsAvailable(): boolean { this.ensureSqlite3(); @@ -307,6 +406,9 @@ export default class BrowserSqlProvider implements DatabaseProvider { /** * Load or create a database stored in OPFS for persistent storage. + * + * **Prefer `loadFromSahPool()` over this method** — it supports WAL mode + * and is significantly faster. This method is kept for migration purposes. * The database will persist across browser sessions. * * Requires COOP/COEP headers to be set by the server: @@ -354,9 +456,10 @@ export default class BrowserSqlProvider implements DatabaseProvider { const mode = options.createIfNotExists !== false ? 'c' : ''; this.db = new this.sqlite3!.oo1.OpfsDb(path, mode); this.opfsDbPath = path; + this.sahPoolDbName = undefined; - // Configure the database for OPFS - // Note: WAL mode requires exclusive locking in OPFS environment + // Configure the database for legacy OPFS + // Note: WAL mode is not supported by the legacy OPFS VFS this.db.exec("PRAGMA journal_mode = DELETE"); this.db.exec("PRAGMA synchronous = NORMAL"); @@ -370,10 +473,10 @@ export default class BrowserSqlProvider implements DatabaseProvider { } /** - * Check if the currently open database is stored in OPFS. + * Check if the currently open database is stored in OPFS (legacy or SAHPool). */ get isUsingOpfs(): boolean { - return this.opfsDbPath !== undefined; + return this.opfsDbPath !== undefined || this.sahPoolDbName !== undefined; } /** @@ -381,7 +484,7 @@ export default class BrowserSqlProvider implements DatabaseProvider { * Returns undefined if not using OPFS. */ get currentOpfsPath(): string | undefined { - return this.opfsDbPath; + return this.opfsDbPath ?? this.sahPoolDbName; } /** @@ -426,7 +529,8 @@ export default class BrowserSqlProvider implements DatabaseProvider { const startTime = performance.now(); this.db = new this.sqlite3!.oo1.DB(":memory:", "c"); - this.opfsDbPath = undefined; // Not using OPFS + this.opfsDbPath = undefined; + this.sahPoolDbName = undefined; this.db.exec("PRAGMA journal_mode = WAL"); const loadTime = performance.now() - startTime; @@ -448,7 +552,8 @@ export default class BrowserSqlProvider implements DatabaseProvider { // once we swap connections. Drop them so callers re-prepare. this.clearStatementCache(); this.db = new this.sqlite3!.oo1.DB({ filename: ":memory:", flags: "c" }); - this.opfsDbPath = undefined; // Not using OPFS + this.opfsDbPath = undefined; + this.sahPoolDbName = undefined; const rc = this.sqlite3!.capi.sqlite3_deserialize( this.db.pointer!, @@ -592,8 +697,9 @@ export default class BrowserSqlProvider implements DatabaseProvider { this.db = undefined; } - // Reset OPFS state + // Reset OPFS / SAHPool state this.opfsDbPath = undefined; + this.sahPoolDbName = undefined; } /** diff --git a/apps/client-standalone/src/local-server-worker.ts b/apps/client-standalone/src/local-server-worker.ts index 6431b0354f..9245d6276d 100644 --- a/apps/client-standalone/src/local-server-worker.ts +++ b/apps/client-standalone/src/local-server-worker.ts @@ -122,6 +122,127 @@ async function writeOpfsFile(fileName: string, buffer: Uint8Array): Promise { + const root = await navigator.storage.getDirectory(); + const fileHandle = await root.getFileHandle(fileName); + const file = await fileHandle.getFile(); + return new Uint8Array(await file.arrayBuffer()); +} + +/** + * Delete a file from the OPFS root. + * Used to clean up the legacy OPFS database after migration to SAHPool. + */ +async function deleteOpfsFile(fileName: string): Promise { + const root = await navigator.storage.getDirectory(); + await root.removeEntry(fileName); +} + +/** + * Verify that a buffer contains a valid SQLite database by checking the + * 16-byte magic string "SQLite format 3\0". + */ +function assertSqliteMagic(buffer: Uint8Array, source: string): void { + const magic = new TextDecoder().decode(buffer.subarray(0, 15)); + if (magic !== "SQLite format 3") { + throw new Error( + `${source} is not a SQLite database ` + + `(got ${buffer.byteLength} bytes starting with "${magic}"). ` + + `The file is likely missing and the SPA fallback is returning index.html.` + ); + } +} + +/** + * Migrate database from legacy OPFS VFS to SAHPool VFS. + * Checks if a legacy `/trilium.db` file exists in the OPFS root, and if the + * SAHPool doesn't already have it. If migration is needed, the legacy file is + * read, imported into the pool, and then deleted. + */ +async function migrateFromLegacyOpfs(dbName: string): Promise { + const legacyFileName = dbName.replace(/^\//, ""); // strip leading slash + const legacyExists = await opfsFileExists(legacyFileName); + + if (!legacyExists) { + return; // Nothing to migrate + } + + // Check if SAHPool already has this DB (e.g. migration already happened) + const poolFiles = sqlProvider!.sahPool!.getFileNames(); + if (poolFiles.includes(dbName)) { + console.log("[Worker] SAHPool already contains the database, deleting legacy OPFS file..."); + await deleteOpfsFile(legacyFileName); + return; + } + + console.log("[Worker] Migrating database from legacy OPFS to SAHPool VFS..."); + const startTime = performance.now(); + + const buffer = await readOpfsFile(legacyFileName); + assertSqliteMagic(buffer, "Legacy OPFS database"); + + await sqlProvider!.sahPool!.importDb(dbName, buffer); + await deleteOpfsFile(legacyFileName); + + // Also clean up legacy journal/WAL files if they exist + for (const suffix of ["-journal", "-wal", "-shm"]) { + try { + await deleteOpfsFile(legacyFileName + suffix); + } catch { + // Ignore — file may not exist + } + } + + const elapsed = performance.now() - startTime; + console.log(`[Worker] Migration complete in ${elapsed.toFixed(2)}ms (${buffer.byteLength} bytes)`); +} + +/** + * Load the test fixture database for integration tests. + * Seeds from the fixture if not already present, using SAHPool when available. + */ +async function loadTestDatabase(sahPoolAvailable: boolean, dbName: string): Promise { + if (sahPoolAvailable) { + const poolFiles = sqlProvider!.sahPool!.getFileNames(); + if (!poolFiles.includes(dbName)) { + console.log("[Worker] Integration test mode: seeding fixture database into SAHPool..."); + const buffer = await fetchTestFixture(); + await sqlProvider!.sahPool!.importDb(dbName, buffer); + } else { + console.log("[Worker] Integration test mode: reusing existing SAHPool DB from earlier in this test"); + } + sqlProvider!.loadFromSahPool(dbName); + } else { + // Fallback to legacy OPFS for tests when SAHPool isn't available + const legacyFileName = dbName.replace(/^\//, ""); + if (!(await opfsFileExists(legacyFileName))) { + console.log("[Worker] Integration test mode: seeding fixture database into OPFS..."); + const buffer = await fetchTestFixture(); + await writeOpfsFile(legacyFileName, buffer); + } else { + console.log("[Worker] Integration test mode: reusing existing OPFS DB from earlier in this test"); + } + sqlProvider!.loadFromOpfs(dbName); + } +} + +/** + * Fetch the test fixture database and validate it. + */ +async function fetchTestFixture(): Promise { + const response = await fetch("/test-fixtures/document.db"); + if (!response.ok) { + throw new Error(`Failed to fetch test fixture: ${response.status} ${response.statusText}`); + } + const buffer = new Uint8Array(await response.arrayBuffer()); + assertSqliteMagic(buffer, "Test fixture at /test-fixtures/document.db"); + return buffer; +} + /** * Load all required modules using dynamic imports. * This allows errors to be caught by our error handlers. @@ -193,10 +314,20 @@ async function initialize(): Promise { console.log("[Worker] Initializing SQLite WASM..."); await sqlProvider!.initWasm(); + // Try to install the SAHPool VFS (preferred: supports WAL, much faster) + let sahPoolAvailable = false; + try { + await sqlProvider!.installSahPool(); + sahPoolAvailable = true; + } catch (e) { + console.warn("[Worker] SAHPool VFS not available, will fall back to legacy OPFS or in-memory:", e); + } + // Integration test mode is baked in at build time via the // __TRILIUM_INTEGRATION_TEST__ Vite define (derived from the // TRILIUM_INTEGRATION_TEST env var when the bundle was built). const integrationTestMode = __TRILIUM_INTEGRATION_TEST__; + const dbName = "/trilium.db"; if (integrationTestMode === "memory") { // Use OPFS for the DB in integration test mode so option changes @@ -204,34 +335,16 @@ async function initialize(): Promise { // Playwright gives each test a fresh BrowserContext, which means a // fresh OPFS — so on the first worker init of a test we seed from // the fixture, and subsequent inits in the same test reuse it. - const opfsDbName = "trilium.db"; - if (!(await opfsFileExists(opfsDbName))) { - console.log("[Worker] Integration test mode: seeding fixture database into OPFS..."); - const response = await fetch("/test-fixtures/document.db"); - if (!response.ok) { - throw new Error(`Failed to fetch test fixture: ${response.status} ${response.statusText}`); - } - const buffer = new Uint8Array(await response.arrayBuffer()); - // Verify we actually got a SQLite database, not an SPA-fallback - // index.html served for a missing file. SQLite databases start - // with the 16-byte magic string "SQLite format 3\0". - const magic = new TextDecoder().decode(buffer.subarray(0, 15)); - if (magic !== "SQLite format 3") { - throw new Error( - `Test fixture at /test-fixtures/document.db is not a SQLite database ` + - `(got ${buffer.byteLength} bytes starting with "${magic}"). ` + - `The file is likely missing and the SPA fallback is returning index.html.` - ); - } - await writeOpfsFile(opfsDbName, buffer); - } else { - console.log("[Worker] Integration test mode: reusing existing OPFS DB from earlier in this test"); - } - sqlProvider!.loadFromOpfs(`/${opfsDbName}`); + await loadTestDatabase(sahPoolAvailable, dbName); + } else if (sahPoolAvailable) { + // SAHPool available — migrate from legacy OPFS if needed, then open + await migrateFromLegacyOpfs(dbName); + console.log("[Worker] SAHPool available, loading persistent database (WAL mode)..."); + sqlProvider!.loadFromSahPool(dbName); } else if (sqlProvider!.isOpfsAvailable()) { - // Try to use OPFS for persistent storage - console.log("[Worker] OPFS available, loading persistent database..."); - sqlProvider!.loadFromOpfs("/trilium.db"); + // Fall back to legacy OPFS VFS (no WAL, slower writes) + console.warn("[Worker] Using legacy OPFS VFS (no WAL mode). Consider enabling COOP/COEP headers for SAHPool."); + sqlProvider!.loadFromOpfs(dbName); } else { // Fall back to in-memory database (non-persistent) console.warn("[Worker] OPFS not available, using in-memory database (data will not persist)");