From 8d38b818c0d83a6420b7bb7c5567856f2db07b80 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Wed, 25 Mar 2026 22:16:07 +0200 Subject: [PATCH] feat(core): reintroduce DB migration --- apps/server/src/services/migration.ts | 138 ----------------- apps/server/src/services/sql_init.ts | 6 - .../0216__move_content_into_blobs.ts | 9 +- .../0220__migrate_images_to_attachments.ts | 14 +- ...233__migrate_geo_map_to_collection.spec.ts | 14 +- .../0233__migrate_geo_map_to_collection.ts | 7 +- .../0234__migrate_ai_chat_to_code.ts | 4 +- .../src/migrations/migrations.ts | 0 packages/trilium-core/src/services/backup.ts | 5 + .../src/services/hidden_subtree.ts | 2 +- .../src/services/migration.spec.ts | 0 .../trilium-core/src/services/migration.ts | 142 +++++++++++++++++- .../trilium-core/src/services/sql_init.ts | 4 +- .../trilium-core/src/services/utils/index.ts | 4 + 14 files changed, 177 insertions(+), 172 deletions(-) delete mode 100644 apps/server/src/services/migration.ts rename {apps/server => packages/trilium-core}/src/migrations/0216__move_content_into_blobs.ts (92%) rename {apps/server => packages/trilium-core}/src/migrations/0220__migrate_images_to_attachments.ts (73%) rename {apps/server => packages/trilium-core}/src/migrations/0233__migrate_geo_map_to_collection.spec.ts (97%) rename {apps/server => packages/trilium-core}/src/migrations/0233__migrate_geo_map_to_collection.ts (91%) rename {apps/server => packages/trilium-core}/src/migrations/0234__migrate_ai_chat_to_code.ts (86%) rename {apps/server => packages/trilium-core}/src/migrations/migrations.ts (100%) create mode 100644 packages/trilium-core/src/services/backup.ts rename {apps/server => packages/trilium-core}/src/services/migration.spec.ts (100%) diff --git a/apps/server/src/services/migration.ts b/apps/server/src/services/migration.ts deleted file mode 100644 index 6aaaa97bb5..0000000000 --- a/apps/server/src/services/migration.ts +++ /dev/null @@ -1,138 +0,0 @@ -import backupService from "./backup.js"; -import sql from "./sql.js"; -import log from "./log.js"; -import { crash } from "./utils.js"; -import appInfo from "./app_info.js"; -import cls from "./cls.js"; -import { t } from "i18next"; -import MIGRATIONS from "../migrations/migrations.js"; - -interface MigrationInfo { - dbVersion: number; - /** - * If string, then the migration is an SQL script that will be executed. - * If a function, then the migration is a JavaScript/TypeScript module that will be executed. - */ - migration: string | (() => void); -} - -async function migrate() { - const currentDbVersion = getDbVersion(); - - if (currentDbVersion < 214) { - await crash(t("migration.old_version")); - return; - } - - // backup before attempting migration - if (!process.env.TRILIUM_INTEGRATION_TEST) { - await backupService.backupNow( - // creating a special backup for version 0.60.4, the changes in 0.61 are major. - currentDbVersion === 214 ? `before-migration-v060` : "before-migration" - ); - } - - const migrations = await prepareMigrations(currentDbVersion); - - // all migrations are executed in one transaction - upgrade either succeeds, or the user can stay at the old version - // otherwise if half of the migrations succeed, user can't use any version - DB is too "new" for the old app, - // and too old for the new app version. - - cls.setMigrationRunning(true); - - sql.transactional(() => { - for (const mig of migrations) { - try { - log.info(`Attempting migration to version ${mig.dbVersion}`); - - executeMigration(mig); - - sql.execute( - /*sql*/`UPDATE options - SET value = ? - WHERE name = ?`, - [mig.dbVersion.toString(), "dbVersion"] - ); - - log.info(`Migration to version ${mig.dbVersion} has been successful.`); - } catch (e: any) { - console.error(e); - crash(t("migration.error_message", { version: mig.dbVersion, stack: e.stack })); - break; // crash() is sometimes async - } - } - }); - - if (currentDbVersion === 214) { - // special VACUUM after the big migration - log.info("VACUUMing database, this might take a while ..."); - sql.execute("VACUUM"); - } -} - -async function prepareMigrations(currentDbVersion: number): Promise { - MIGRATIONS.sort((a, b) => a.version - b.version); - const migrations: MigrationInfo[] = []; - for (const migration of MIGRATIONS) { - const dbVersion = migration.version; - if (dbVersion > currentDbVersion) { - if ("sql" in migration) { - migrations.push({ - dbVersion, - migration: migration.sql - }); - } else { - // Due to ESM imports, the migration file needs to be imported asynchronously and thus cannot be loaded at migration time (since migration is not asynchronous). - // As such we have to preload the ESM. - migrations.push({ - dbVersion, - migration: (await migration.module()).default - }); - } - } - } - return migrations; -} - -function executeMigration({ migration }: MigrationInfo) { - if (typeof migration === "string") { - console.log(`Migration with SQL script: ${migration}`); - sql.executeScript(migration); - } else { - console.log("Migration with JS module"); - migration(); - }; -} - -function getDbVersion() { - return parseInt(sql.getValue("SELECT value FROM options WHERE name = 'dbVersion'")); -} - -function isDbUpToDate() { - const dbVersion = getDbVersion(); - - const upToDate = dbVersion >= appInfo.dbVersion; - - if (!upToDate) { - log.info(`App db version is ${appInfo.dbVersion}, while db version is ${dbVersion}. Migration needed.`); - } - - return upToDate; -} - -async function migrateIfNecessary() { - const currentDbVersion = getDbVersion(); - - if (currentDbVersion > appInfo.dbVersion && process.env.TRILIUM_IGNORE_DB_VERSION !== "true") { - await crash(t("migration.wrong_db_version", { version: currentDbVersion, targetVersion: appInfo.dbVersion })); - } - - if (!isDbUpToDate()) { - await migrate(); - } -} - -export default { - migrateIfNecessary, - isDbUpToDate -}; diff --git a/apps/server/src/services/sql_init.ts b/apps/server/src/services/sql_init.ts index 263f4c7488..0b09bdc9a5 100644 --- a/apps/server/src/services/sql_init.ts +++ b/apps/server/src/services/sql_init.ts @@ -1,11 +1,5 @@ import { type OptionRow } from "@triliumnext/commons"; import { sql_init as coreSqlInit } from "@triliumnext/core"; -import fs from "fs"; - -import BOption from "../becca/entities/boption.js"; -import log from "./log.js"; -import resourceDir from "./resource_dir.js"; -import sql from "./sql.js"; const schemaExists = coreSqlInit.schemaExists; const isDbInitialized = coreSqlInit.isDbInitialized; diff --git a/apps/server/src/migrations/0216__move_content_into_blobs.ts b/packages/trilium-core/src/migrations/0216__move_content_into_blobs.ts similarity index 92% rename from apps/server/src/migrations/0216__move_content_into_blobs.ts rename to packages/trilium-core/src/migrations/0216__move_content_into_blobs.ts index 47d4c57612..99f7cdf821 100644 --- a/apps/server/src/migrations/0216__move_content_into_blobs.ts +++ b/packages/trilium-core/src/migrations/0216__move_content_into_blobs.ts @@ -1,5 +1,5 @@ -import sql from "../services/sql.js"; -import utils from "../services/utils.js"; +import { getSql } from "../services/sql/index"; +import { hashedBlobId } from "../services/utils/index"; interface NoteContentsRow { noteId: string; @@ -17,9 +17,10 @@ interface NoteRevisionContents { export default () => { const existingBlobIds = new Set(); + const sql = getSql(); for (const noteId of sql.getColumn(/*sql*/`SELECT noteId FROM note_contents`)) { const row = sql.getRow(/*sql*/`SELECT noteId, content, dateModified, utcDateModified FROM note_contents WHERE noteId = ?`, [noteId]); - const blobId = utils.hashedBlobId(row.content); + const blobId = hashedBlobId(row.content); if (!existingBlobIds.has(blobId)) { existingBlobIds.add(blobId); @@ -42,7 +43,7 @@ export default () => { for (const noteRevisionId of sql.getColumn(/*sql*/`SELECT noteRevisionId FROM note_revision_contents`)) { const row = sql.getRow(/*sql*/`SELECT noteRevisionId, content, utcDateModified FROM note_revision_contents WHERE noteRevisionId = ?`, [noteRevisionId]); - const blobId = utils.hashedBlobId(row.content); + const blobId = hashedBlobId(row.content); if (!existingBlobIds.has(blobId)) { existingBlobIds.add(blobId); diff --git a/apps/server/src/migrations/0220__migrate_images_to_attachments.ts b/packages/trilium-core/src/migrations/0220__migrate_images_to_attachments.ts similarity index 73% rename from apps/server/src/migrations/0220__migrate_images_to_attachments.ts rename to packages/trilium-core/src/migrations/0220__migrate_images_to_attachments.ts index f20e21cb8a..df9f5096a0 100644 --- a/apps/server/src/migrations/0220__migrate_images_to_attachments.ts +++ b/packages/trilium-core/src/migrations/0220__migrate_images_to_attachments.ts @@ -1,12 +1,14 @@ -import { becca_loader } from "@triliumnext/core"; - import becca from "../becca/becca.js"; -import cls from "../services/cls.js"; -import log from "../services/log.js"; -import sql from "../services/sql.js"; +import { getLog } from "../services/log.js"; +import { getSql } from "../services/sql/index.js"; +import { getContext } from "../services/context.js"; +import becca_loader from "../becca/becca_loader.js"; export default () => { - cls.init(() => { + getContext().init(() => { + const sql = getSql(); + const log = getLog(); + // emergency disabling of image compression since it appears to make problems in migration to 0.61 sql.execute(/*sql*/`UPDATE options SET value = 'false' WHERE name = 'compressImages'`); diff --git a/apps/server/src/migrations/0233__migrate_geo_map_to_collection.spec.ts b/packages/trilium-core/src/migrations/0233__migrate_geo_map_to_collection.spec.ts similarity index 97% rename from apps/server/src/migrations/0233__migrate_geo_map_to_collection.spec.ts rename to packages/trilium-core/src/migrations/0233__migrate_geo_map_to_collection.spec.ts index 1ce4fd2bdc..61a89584dc 100644 --- a/apps/server/src/migrations/0233__migrate_geo_map_to_collection.spec.ts +++ b/packages/trilium-core/src/migrations/0233__migrate_geo_map_to_collection.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, it, beforeEach } from "vitest"; -import cls from "../services/cls.js"; -import sql from "../services/sql.js"; +import * as cls from "../services/context.js"; +import { getSql } from "../services/sql/index.js"; import becca from "../becca/becca.js"; import becca_loader from "../becca/becca_loader.js"; import migration from "./0233__migrate_geo_map_to_collection.js"; @@ -19,12 +19,14 @@ import migration from "./0233__migrate_geo_map_to_collection.js"; * test data into the database, then verifies the migration transforms the data correctly. */ describe("Migration 0233: Migrate geoMap to collection", () => { + const sql = getSql(); + beforeEach(async () => { // Set up a clean in-memory database for each test sql.rebuildIntegrationTestDatabase(); await new Promise((resolve) => { - cls.init(() => { + cls.getContext().init(() => { becca_loader.load(); resolve(); }); @@ -33,7 +35,7 @@ describe("Migration 0233: Migrate geoMap to collection", () => { it("should migrate geoMap notes to book type with viewConfig attachment", async () => { await new Promise((resolve) => { - cls.init(() => { + cls.getContext().init(() => { // Create a test geoMap note with content const geoMapContent = JSON.stringify({ markers: [ @@ -164,7 +166,7 @@ describe("Migration 0233: Migrate geoMap to collection", () => { it("should handle existing viewConfig attachments with same title", async () => { await new Promise((resolve) => { - cls.init(() => { + cls.getContext().init(() => { const geoMapContent = JSON.stringify({ test: "data" }); const testNoteId = "test_geo_note_existing"; const testBlobId = "test_blob_geo_existing"; @@ -226,7 +228,7 @@ describe("Migration 0233: Migrate geoMap to collection", () => { it("should handle protected geoMap notes appropriately", async () => { await new Promise((resolve, reject) => { - cls.init(() => { + cls.getContext().init(() => { const geoMapContent = JSON.stringify({ markers: [{ lat: 51.5074, lng: -0.1278, title: "London" }], center: { lat: 51.5074, lng: -0.1278 }, diff --git a/apps/server/src/migrations/0233__migrate_geo_map_to_collection.ts b/packages/trilium-core/src/migrations/0233__migrate_geo_map_to_collection.ts similarity index 91% rename from apps/server/src/migrations/0233__migrate_geo_map_to_collection.ts rename to packages/trilium-core/src/migrations/0233__migrate_geo_map_to_collection.ts index ccd32b1b0d..ccdb0163bd 100644 --- a/apps/server/src/migrations/0233__migrate_geo_map_to_collection.ts +++ b/packages/trilium-core/src/migrations/0233__migrate_geo_map_to_collection.ts @@ -1,11 +1,10 @@ -import { becca_loader } from "@triliumnext/core"; - import becca from "../becca/becca"; -import cls from "../services/cls.js"; +import becca_loader from "../becca/becca_loader"; +import { getContext } from "../services/context"; import hidden_subtree from "../services/hidden_subtree"; export default () => { - cls.init(() => { + getContext().init(() => { becca_loader.load(); // Ensure the geomap template is generated. diff --git a/apps/server/src/migrations/0234__migrate_ai_chat_to_code.ts b/packages/trilium-core/src/migrations/0234__migrate_ai_chat_to_code.ts similarity index 86% rename from apps/server/src/migrations/0234__migrate_ai_chat_to_code.ts rename to packages/trilium-core/src/migrations/0234__migrate_ai_chat_to_code.ts index 72eb00ae27..613122c670 100644 --- a/apps/server/src/migrations/0234__migrate_ai_chat_to_code.ts +++ b/packages/trilium-core/src/migrations/0234__migrate_ai_chat_to_code.ts @@ -1,9 +1,9 @@ import becca from "../becca/becca"; import becca_loader from "../becca/becca_loader"; -import cls from "../services/cls.js"; +import { getContext } from "../services/context"; export default () => { - cls.init(() => { + getContext().init(() => { becca_loader.load(); for (const note of Object.values(becca.notes)) { diff --git a/apps/server/src/migrations/migrations.ts b/packages/trilium-core/src/migrations/migrations.ts similarity index 100% rename from apps/server/src/migrations/migrations.ts rename to packages/trilium-core/src/migrations/migrations.ts diff --git a/packages/trilium-core/src/services/backup.ts b/packages/trilium-core/src/services/backup.ts new file mode 100644 index 0000000000..b596ccabe5 --- /dev/null +++ b/packages/trilium-core/src/services/backup.ts @@ -0,0 +1,5 @@ +export default { + backupNow() { + console.warn("Backup not yet available."); + } +} diff --git a/packages/trilium-core/src/services/hidden_subtree.ts b/packages/trilium-core/src/services/hidden_subtree.ts index 0931b5f1dd..6150ffe7ce 100644 --- a/packages/trilium-core/src/services/hidden_subtree.ts +++ b/packages/trilium-core/src/services/hidden_subtree.ts @@ -8,7 +8,7 @@ import BNote from "../becca/entities/bnote.js"; import buildLaunchBarConfig from "./hidden_subtree_launcherbar.js"; import buildHiddenSubtreeTemplates from "./hidden_subtree_templates.js"; import { cleanUpHelp, getHelpHiddenSubtreeData } from "./in_app_help.js"; -import * as migrationService from "./migration.js"; +import migrationService from "./migration.js"; import noteService from "./notes.js"; import { getLog } from "./log.js"; diff --git a/apps/server/src/services/migration.spec.ts b/packages/trilium-core/src/services/migration.spec.ts similarity index 100% rename from apps/server/src/services/migration.spec.ts rename to packages/trilium-core/src/services/migration.spec.ts diff --git a/packages/trilium-core/src/services/migration.ts b/packages/trilium-core/src/services/migration.ts index fe61ef7b92..cc2b21ed8e 100644 --- a/packages/trilium-core/src/services/migration.ts +++ b/packages/trilium-core/src/services/migration.ts @@ -1,4 +1,140 @@ -export function isDbUpToDate() { - // TODO: Implement. - return true; +import backupService from "./backup.js"; +import { getSql } from "./sql/index.js"; +import { getLog } from "./log.js"; +import { crash } from "./utils/index.js"; +import appInfo from "./app_info.js"; +import * as cls from "./context.js"; +import { t } from "i18next"; +import MIGRATIONS from "../migrations/migrations.js"; + +interface MigrationInfo { + dbVersion: number; + /** + * If string, then the migration is an SQL script that will be executed. + * If a function, then the migration is a JavaScript/TypeScript module that will be executed. + */ + migration: string | (() => void); } + +async function migrate() { + const currentDbVersion = getDbVersion(); + + if (currentDbVersion < 214) { + await crash(t("migration.old_version")); + return; + } + + // backup before attempting migration + if (!process.env.TRILIUM_INTEGRATION_TEST) { + await backupService.backupNow( + // creating a special backup for version 0.60.4, the changes in 0.61 are major. + currentDbVersion === 214 ? `before-migration-v060` : "before-migration" + ); + } + + const migrations = await prepareMigrations(currentDbVersion); + + // all migrations are executed in one transaction - upgrade either succeeds, or the user can stay at the old version + // otherwise if half of the migrations succeed, user can't use any version - DB is too "new" for the old app, + // and too old for the new app version. + + cls.setMigrationRunning(true); + + const sql = getSql(); + const log = getLog(); + sql.transactional(() => { + for (const mig of migrations) { + try { + log.info(`Attempting migration to version ${mig.dbVersion}`); + + executeMigration(mig); + + sql.execute( + /*sql*/`UPDATE options + SET value = ? + WHERE name = ?`, + [mig.dbVersion.toString(), "dbVersion"] + ); + + log.info(`Migration to version ${mig.dbVersion} has been successful.`); + } catch (e: any) { + console.error(e); + crash(t("migration.error_message", { version: mig.dbVersion, stack: e.stack })); + break; // crash() is sometimes async + } + } + }); + + if (currentDbVersion === 214) { + // special VACUUM after the big migration + log.info("VACUUMing database, this might take a while ..."); + sql.execute("VACUUM"); + } +} + +async function prepareMigrations(currentDbVersion: number): Promise { + MIGRATIONS.sort((a, b) => a.version - b.version); + const migrations: MigrationInfo[] = []; + for (const migration of MIGRATIONS) { + const dbVersion = migration.version; + if (dbVersion > currentDbVersion) { + if ("sql" in migration) { + migrations.push({ + dbVersion, + migration: migration.sql + }); + } else { + // Due to ESM imports, the migration file needs to be imported asynchronously and thus cannot be loaded at migration time (since migration is not asynchronous). + // As such we have to preload the ESM. + migrations.push({ + dbVersion, + migration: (await migration.module()).default + }); + } + } + } + return migrations; +} + +function executeMigration({ migration }: MigrationInfo) { + if (typeof migration === "string") { + console.log(`Migration with SQL script: ${migration}`); + getSql().executeScript(migration); + } else { + console.log("Migration with JS module"); + migration(); + }; +} + +function getDbVersion() { + return parseInt(getSql().getValue("SELECT value FROM options WHERE name = 'dbVersion'")); +} + +function isDbUpToDate() { + const dbVersion = getDbVersion(); + + const upToDate = dbVersion >= appInfo.dbVersion; + + if (!upToDate) { + getLog().info(`App db version is ${appInfo.dbVersion}, while db version is ${dbVersion}. Migration needed.`); + } + + return upToDate; +} + +async function migrateIfNecessary() { + const currentDbVersion = getDbVersion(); + + if (currentDbVersion > appInfo.dbVersion && process.env.TRILIUM_IGNORE_DB_VERSION !== "true") { + await crash(t("migration.wrong_db_version", { version: currentDbVersion, targetVersion: appInfo.dbVersion })); + } + + if (!isDbUpToDate()) { + await migrate(); + } +} + +export default { + migrateIfNecessary, + isDbUpToDate +}; diff --git a/packages/trilium-core/src/services/sql_init.ts b/packages/trilium-core/src/services/sql_init.ts index 3c1b97a7fe..af82213724 100644 --- a/packages/trilium-core/src/services/sql_init.ts +++ b/packages/trilium-core/src/services/sql_init.ts @@ -12,6 +12,7 @@ import BBranch from "../becca/entities/bbranch"; import hidden_subtree from "./hidden_subtree"; import TaskContext from "./task_context"; import BOption from "../becca/entities/boption"; +import migrationService from "./migration"; export const dbReady = deferred(); @@ -52,8 +53,7 @@ async function initDbConnection() { return; } - //TODO: Renable migration - //await migrationService.migrateIfNecessary(); + await migrationService.migrateIfNecessary(); const sql = getSql(); sql.execute('CREATE TEMP TABLE IF NOT EXISTS "param_list" (`paramId` TEXT NOT NULL PRIMARY KEY)'); diff --git a/packages/trilium-core/src/services/utils/index.ts b/packages/trilium-core/src/services/utils/index.ts index 74f47927a4..be338a5712 100644 --- a/packages/trilium-core/src/services/utils/index.ts +++ b/packages/trilium-core/src/services/utils/index.ts @@ -202,3 +202,7 @@ export function isEmptyOrWhitespace(str: string | null | undefined) { export function escapeRegExp(str: string) { return str.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1"); } + +export async function crash(message: string): never { + throw new Error(message); +}