Dockery/Trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2026-05-06 13:37:17 +02:00
Code Issues Packages Projects Releases Activity

feat(standalone): migrate to SAHPool for faster DB

Browse Source
This commit is contained in:
Elian Doran
2026-04-19 22:28:27 +03:00
parent 9ac3fcb21c
commit 6a2e48dacb
2 changed files with 258 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
apps/client-standalone/src/lightweight/sql_provider.ts
View File

@@ -1,4 +1,4 @@
import { type BindableValue, default as sqlite3InitModule } from "@sqlite.org/sqlite-wasm";
import { type BindableValue, type SAHPoolUtil, default as sqlite3InitModule } from "@sqlite.org/sqlite-wasm";
import type { DatabaseProvider, RunResult, Statement, Transaction } from "@triliumnext/core";
// Type definitions for SQLite WASM (the library doesn't export these directly)
@@ -227,6 +227,10 @@ export default class BrowserSqlProvider implements DatabaseProvider {
// OPFS state tracking
private opfsDbPath?: string;
// SAHPool state tracking
private sahPoolUtil?: SAHPoolUtil;
private sahPoolDbName?: string;
/**
* Get the SQLite WASM module version info.
* Returns undefined if the module hasn't been initialized yet.
@@ -287,16 +291,111 @@ export default class BrowserSqlProvider implements DatabaseProvider {
return this.sqlite3 !== undefined;
}
// ==================== OPFS Support ====================
// ==================== SAHPool VFS (preferred OPFS backend) ====================
/**
* Check if the OPFS VFS is available.
* Install the OPFS SAHPool VFS. This pre-allocates a pool of OPFS
* SyncAccessHandle objects, enabling WAL mode and significantly faster
* writes compared to the legacy OPFS VFS.
*
* Must be called after `initWasm()` and before `loadFromSahPool()`.
* This is async because it acquires OPFS file handles.
*
* @param options.directory - OPFS directory for the pool (default: auto-derived from VFS name)
* @param options.initialCapacity - Minimum number of file slots (default: 6)
* @throws Error if the environment doesn't support SAHPool (no OPFS, no Worker, no COOP/COEP)
*/
async installSahPool(options: { directory?: string; initialCapacity?: number } = {}): Promise<void> {
this.ensureSqlite3();
console.log("[BrowserSqlProvider] Installing OPFS SAHPool VFS...");
const startTime = performance.now();
this.sahPoolUtil = await this.sqlite3!.installOpfsSAHPoolVfs({
clearOnInit: false,
initialCapacity: options.initialCapacity ?? 6,
directory: options.directory,
});
// Ensure enough slots for DB + WAL + journal + temp files
await this.sahPoolUtil.reserveMinimumCapacity(options.initialCapacity ?? 6);
const initTime = performance.now() - startTime;
console.log(
`[BrowserSqlProvider] SAHPool VFS installed in ${initTime.toFixed(2)}ms ` +
`(capacity: ${this.sahPoolUtil.getCapacity()}, files: ${this.sahPoolUtil.getFileCount()})`
);
}
/**
* Whether the SAHPool VFS has been successfully installed.
*/
get isSahPoolInstalled(): boolean {
return this.sahPoolUtil !== undefined;
}
/**
* Access the SAHPool utility for advanced operations (import/export/migration).
*/
get sahPool(): SAHPoolUtil | undefined {
return this.sahPoolUtil;
}
/**
* Load or create a database using the SAHPool VFS.
* This is the preferred method for persistent storage — it supports WAL mode
* and is significantly faster than the legacy OPFS VFS.
*
* @param dbName - Virtual filename within the pool (e.g., "/trilium.db").
* Must start with a slash.
* @throws Error if SAHPool VFS is not installed
*/
loadFromSahPool(dbName: string): void {
this.ensureSqlite3();
if (!this.sahPoolUtil) {
throw new Error(
"SAHPool VFS not installed. Call installSahPool() first."
);
}
console.log(`[BrowserSqlProvider] Loading database from SAHPool: ${dbName}`);
const startTime = performance.now();
try {
this.db = new this.sahPoolUtil.OpfsSAHPoolDb(dbName);
this.sahPoolDbName = dbName;
this.opfsDbPath = undefined;
// SAHPool supports WAL mode — the key advantage over legacy OPFS VFS
this.db.exec("PRAGMA journal_mode = WAL");
this.db.exec("PRAGMA synchronous = NORMAL");
const loadTime = performance.now() - startTime;
console.log(`[BrowserSqlProvider] SAHPool database loaded in ${loadTime.toFixed(2)}ms (WAL mode)`);
} catch (e) {
const error = e instanceof Error ? e : new Error(String(e));
console.error(`[BrowserSqlProvider] Failed to load SAHPool database: ${error.message}`);
throw error;
}
}
/**
* Whether the currently open database is using the SAHPool VFS.
*/
get isUsingSahPool(): boolean {
return this.sahPoolDbName !== undefined;
}
// ==================== Legacy OPFS Support ====================
/**
* Check if the legacy OPFS VFS is available.
* This requires:
* - Running in a Worker context
* - Browser support for OPFS APIs
* - COOP/COEP headers sent by the server (for SharedArrayBuffer)
*
* @returns true if OPFS VFS is available for use
* @returns true if legacy OPFS VFS is available for use
*/
isOpfsAvailable(): boolean {
this.ensureSqlite3();
@@ -307,6 +406,9 @@ export default class BrowserSqlProvider implements DatabaseProvider {
/**
* Load or create a database stored in OPFS for persistent storage.
*
* **Prefer `loadFromSahPool()` over this method** — it supports WAL mode
* and is significantly faster. This method is kept for migration purposes.
* The database will persist across browser sessions.
*
* Requires COOP/COEP headers to be set by the server:
@@ -354,9 +456,10 @@ export default class BrowserSqlProvider implements DatabaseProvider {
const mode = options.createIfNotExists !== false ? 'c' : '';
this.db = new this.sqlite3!.oo1.OpfsDb(path, mode);
this.opfsDbPath = path;
this.sahPoolDbName = undefined;
// Configure the database for OPFS
// Note: WAL mode requires exclusive locking in OPFS environment
// Configure the database for legacy OPFS
// Note: WAL mode is not supported by the legacy OPFS VFS
this.db.exec("PRAGMA journal_mode = DELETE");
this.db.exec("PRAGMA synchronous = NORMAL");
@@ -370,10 +473,10 @@ export default class BrowserSqlProvider implements DatabaseProvider {
}
/**
* Check if the currently open database is stored in OPFS.
* Check if the currently open database is stored in OPFS (legacy or SAHPool).
*/
get isUsingOpfs(): boolean {
return this.opfsDbPath !== undefined;
return this.opfsDbPath !== undefined || this.sahPoolDbName !== undefined;
}
/**
@@ -381,7 +484,7 @@ export default class BrowserSqlProvider implements DatabaseProvider {
* Returns undefined if not using OPFS.
*/
get currentOpfsPath(): string | undefined {
return this.opfsDbPath;
return this.opfsDbPath ?? this.sahPoolDbName;
}
/**
@@ -426,7 +529,8 @@ export default class BrowserSqlProvider implements DatabaseProvider {
const startTime = performance.now();
this.db = new this.sqlite3!.oo1.DB(":memory:", "c");
this.opfsDbPath = undefined; // Not using OPFS
this.opfsDbPath = undefined;
this.sahPoolDbName = undefined;
this.db.exec("PRAGMA journal_mode = WAL");
const loadTime = performance.now() - startTime;
@@ -448,7 +552,8 @@ export default class BrowserSqlProvider implements DatabaseProvider {
// once we swap connections. Drop them so callers re-prepare.
this.clearStatementCache();
this.db = new this.sqlite3!.oo1.DB({ filename: ":memory:", flags: "c" });
this.opfsDbPath = undefined; // Not using OPFS
this.opfsDbPath = undefined;
this.sahPoolDbName = undefined;
const rc = this.sqlite3!.capi.sqlite3_deserialize(
this.db.pointer!,
@@ -592,8 +697,9 @@ export default class BrowserSqlProvider implements DatabaseProvider {
this.db = undefined;
}
// Reset OPFS state
// Reset OPFS / SAHPool state
this.opfsDbPath = undefined;
this.sahPoolDbName = undefined;
}
/**

167
apps/client-standalone/src/local-server-worker.ts
View File

@@ -122,6 +122,127 @@ async function writeOpfsFile(fileName: string, buffer: Uint8Array): Promise<void
}
}
/**
* Read a file from the OPFS root into a Uint8Array.
* Used during migration from legacy OPFS VFS to SAHPool.
*/
async function readOpfsFile(fileName: string): Promise<Uint8Array> {
const root = await navigator.storage.getDirectory();
const fileHandle = await root.getFileHandle(fileName);
const file = await fileHandle.getFile();
return new Uint8Array(await file.arrayBuffer());
}
/**
* Delete a file from the OPFS root.
* Used to clean up the legacy OPFS database after migration to SAHPool.
*/
async function deleteOpfsFile(fileName: string): Promise<void> {
const root = await navigator.storage.getDirectory();
await root.removeEntry(fileName);
}
/**
* Verify that a buffer contains a valid SQLite database by checking the
* 16-byte magic string "SQLite format 3\0".
*/
function assertSqliteMagic(buffer: Uint8Array, source: string): void {
const magic = new TextDecoder().decode(buffer.subarray(0, 15));
if (magic !== "SQLite format 3") {
throw new Error(
`${source} is not a SQLite database ` +
`(got ${buffer.byteLength} bytes starting with "${magic}"). ` +
`The file is likely missing and the SPA fallback is returning index.html.`
);
}
}
/**
* Migrate database from legacy OPFS VFS to SAHPool VFS.
* Checks if a legacy `/trilium.db` file exists in the OPFS root, and if the
* SAHPool doesn't already have it. If migration is needed, the legacy file is
* read, imported into the pool, and then deleted.
*/
async function migrateFromLegacyOpfs(dbName: string): Promise<void> {
const legacyFileName = dbName.replace(/^\//, ""); // strip leading slash
const legacyExists = await opfsFileExists(legacyFileName);
if (!legacyExists) {
return; // Nothing to migrate
}
// Check if SAHPool already has this DB (e.g. migration already happened)
const poolFiles = sqlProvider!.sahPool!.getFileNames();
if (poolFiles.includes(dbName)) {
console.log("[Worker] SAHPool already contains the database, deleting legacy OPFS file...");
await deleteOpfsFile(legacyFileName);
return;
}
console.log("[Worker] Migrating database from legacy OPFS to SAHPool VFS...");
const startTime = performance.now();
const buffer = await readOpfsFile(legacyFileName);
assertSqliteMagic(buffer, "Legacy OPFS database");
await sqlProvider!.sahPool!.importDb(dbName, buffer);
await deleteOpfsFile(legacyFileName);
// Also clean up legacy journal/WAL files if they exist
for (const suffix of ["-journal", "-wal", "-shm"]) {
try {
await deleteOpfsFile(legacyFileName + suffix);
} catch {
// Ignore — file may not exist
}
}
const elapsed = performance.now() - startTime;
console.log(`[Worker] Migration complete in ${elapsed.toFixed(2)}ms (${buffer.byteLength} bytes)`);
}
/**
* Load the test fixture database for integration tests.
* Seeds from the fixture if not already present, using SAHPool when available.
*/
async function loadTestDatabase(sahPoolAvailable: boolean, dbName: string): Promise<void> {
if (sahPoolAvailable) {
const poolFiles = sqlProvider!.sahPool!.getFileNames();
if (!poolFiles.includes(dbName)) {
console.log("[Worker] Integration test mode: seeding fixture database into SAHPool...");
const buffer = await fetchTestFixture();
await sqlProvider!.sahPool!.importDb(dbName, buffer);
} else {
console.log("[Worker] Integration test mode: reusing existing SAHPool DB from earlier in this test");
}
sqlProvider!.loadFromSahPool(dbName);
} else {
// Fallback to legacy OPFS for tests when SAHPool isn't available
const legacyFileName = dbName.replace(/^\//, "");
if (!(await opfsFileExists(legacyFileName))) {
console.log("[Worker] Integration test mode: seeding fixture database into OPFS...");
const buffer = await fetchTestFixture();
await writeOpfsFile(legacyFileName, buffer);
} else {
console.log("[Worker] Integration test mode: reusing existing OPFS DB from earlier in this test");
}
sqlProvider!.loadFromOpfs(dbName);
}
}
/**
* Fetch the test fixture database and validate it.
*/
async function fetchTestFixture(): Promise<Uint8Array> {
const response = await fetch("/test-fixtures/document.db");
if (!response.ok) {
throw new Error(`Failed to fetch test fixture: ${response.status} ${response.statusText}`);
}
const buffer = new Uint8Array(await response.arrayBuffer());
assertSqliteMagic(buffer, "Test fixture at /test-fixtures/document.db");
return buffer;
}
/**
* Load all required modules using dynamic imports.
* This allows errors to be caught by our error handlers.
@@ -193,10 +314,20 @@ async function initialize(): Promise<void> {
console.log("[Worker] Initializing SQLite WASM...");
await sqlProvider!.initWasm();
// Try to install the SAHPool VFS (preferred: supports WAL, much faster)
let sahPoolAvailable = false;
try {
await sqlProvider!.installSahPool();
sahPoolAvailable = true;
} catch (e) {
console.warn("[Worker] SAHPool VFS not available, will fall back to legacy OPFS or in-memory:", e);
}
// Integration test mode is baked in at build time via the
// __TRILIUM_INTEGRATION_TEST__ Vite define (derived from the
// TRILIUM_INTEGRATION_TEST env var when the bundle was built).
const integrationTestMode = __TRILIUM_INTEGRATION_TEST__;
const dbName = "/trilium.db";
if (integrationTestMode === "memory") {
// Use OPFS for the DB in integration test mode so option changes
@@ -204,34 +335,16 @@ async function initialize(): Promise<void> {
// Playwright gives each test a fresh BrowserContext, which means a
// fresh OPFS — so on the first worker init of a test we seed from
// the fixture, and subsequent inits in the same test reuse it.
const opfsDbName = "trilium.db";
if (!(await opfsFileExists(opfsDbName))) {
console.log("[Worker] Integration test mode: seeding fixture database into OPFS...");
const response = await fetch("/test-fixtures/document.db");
if (!response.ok) {
throw new Error(`Failed to fetch test fixture: ${response.status} ${response.statusText}`);
}
const buffer = new Uint8Array(await response.arrayBuffer());
// Verify we actually got a SQLite database, not an SPA-fallback
// index.html served for a missing file. SQLite databases start
// with the 16-byte magic string "SQLite format 3\0".
const magic = new TextDecoder().decode(buffer.subarray(0, 15));
if (magic !== "SQLite format 3") {
throw new Error(
`Test fixture at /test-fixtures/document.db is not a SQLite database ` +
`(got ${buffer.byteLength} bytes starting with "${magic}"). ` +
`The file is likely missing and the SPA fallback is returning index.html.`
);
}
await writeOpfsFile(opfsDbName, buffer);
} else {
console.log("[Worker] Integration test mode: reusing existing OPFS DB from earlier in this test");
}
sqlProvider!.loadFromOpfs(`/${opfsDbName}`);
await loadTestDatabase(sahPoolAvailable, dbName);
} else if (sahPoolAvailable) {
// SAHPool available — migrate from legacy OPFS if needed, then open
await migrateFromLegacyOpfs(dbName);
console.log("[Worker] SAHPool available, loading persistent database (WAL mode)...");
sqlProvider!.loadFromSahPool(dbName);
} else if (sqlProvider!.isOpfsAvailable()) {
// Try to use OPFS for persistent storage
console.log("[Worker] OPFS available, loading persistent database...");
sqlProvider!.loadFromOpfs("/trilium.db");
// Fall back to legacy OPFS VFS (no WAL, slower writes)
console.warn("[Worker] Using legacy OPFS VFS (no WAL mode). Consider enabling COOP/COEP headers for SAHPool.");
sqlProvider!.loadFromOpfs(dbName);
} else {
// Fall back to in-memory database (non-persistent)
console.warn("[Worker] OPFS not available, using in-memory database (data will not persist)");