Dockery/Trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2026-06-18 14:59:44 +02:00
Code Issues Packages Projects Releases Activity

feat(core): reintroduce DB migration

Browse Source
This commit is contained in:
Elian Doran
2026-03-25 22:16:07 +02:00
parent af462ab0f9
commit 8d38b818c0
14 changed files with 177 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
apps/server/src/services/migration.ts
View File

@@ -1,138 +0,0 @@
import backupService from "./backup.js";
import sql from "./sql.js";
import log from "./log.js";
import { crash } from "./utils.js";
import appInfo from "./app_info.js";
import cls from "./cls.js";
import { t } from "i18next";
import MIGRATIONS from "../migrations/migrations.js";
interface MigrationInfo {
dbVersion: number;
/**
* If string, then the migration is an SQL script that will be executed.
* If a function, then the migration is a JavaScript/TypeScript module that will be executed.
*/
migration: string | (() => void);
}
async function migrate() {
const currentDbVersion = getDbVersion();
if (currentDbVersion < 214) {
await crash(t("migration.old_version"));
return;
}
// backup before attempting migration
if (!process.env.TRILIUM_INTEGRATION_TEST) {
await backupService.backupNow(
// creating a special backup for version 0.60.4, the changes in 0.61 are major.
currentDbVersion === 214 ? `before-migration-v060` : "before-migration"
);
}
const migrations = await prepareMigrations(currentDbVersion);
// all migrations are executed in one transaction - upgrade either succeeds, or the user can stay at the old version
// otherwise if half of the migrations succeed, user can't use any version - DB is too "new" for the old app,
// and too old for the new app version.
cls.setMigrationRunning(true);
sql.transactional(() => {
for (const mig of migrations) {
try {
log.info(`Attempting migration to version ${mig.dbVersion}`);
executeMigration(mig);
sql.execute(
/*sql*/`UPDATE options
SET value = ?
WHERE name = ?`,
[mig.dbVersion.toString(), "dbVersion"]
);
log.info(`Migration to version ${mig.dbVersion} has been successful.`);
} catch (e: any) {
console.error(e);
crash(t("migration.error_message", { version: mig.dbVersion, stack: e.stack }));
break; // crash() is sometimes async
}
}
});
if (currentDbVersion === 214) {
// special VACUUM after the big migration
log.info("VACUUMing database, this might take a while ...");
sql.execute("VACUUM");
}
}
async function prepareMigrations(currentDbVersion: number): Promise<MigrationInfo[]> {
MIGRATIONS.sort((a, b) => a.version - b.version);
const migrations: MigrationInfo[] = [];
for (const migration of MIGRATIONS) {
const dbVersion = migration.version;
if (dbVersion > currentDbVersion) {
if ("sql" in migration) {
migrations.push({
dbVersion,
migration: migration.sql
});
} else {
// Due to ESM imports, the migration file needs to be imported asynchronously and thus cannot be loaded at migration time (since migration is not asynchronous).
// As such we have to preload the ESM.
migrations.push({
dbVersion,
migration: (await migration.module()).default
});
}
}
}
return migrations;
}
function executeMigration({ migration }: MigrationInfo) {
if (typeof migration === "string") {
console.log(`Migration with SQL script: ${migration}`);
sql.executeScript(migration);
} else {
console.log("Migration with JS module");
migration();
};
}
function getDbVersion() {
return parseInt(sql.getValue("SELECT value FROM options WHERE name = 'dbVersion'"));
}
function isDbUpToDate() {
const dbVersion = getDbVersion();
const upToDate = dbVersion >= appInfo.dbVersion;
if (!upToDate) {
log.info(`App db version is ${appInfo.dbVersion}, while db version is ${dbVersion}. Migration needed.`);
}
return upToDate;
}
async function migrateIfNecessary() {
const currentDbVersion = getDbVersion();
if (currentDbVersion > appInfo.dbVersion && process.env.TRILIUM_IGNORE_DB_VERSION !== "true") {
await crash(t("migration.wrong_db_version", { version: currentDbVersion, targetVersion: appInfo.dbVersion }));
}
if (!isDbUpToDate()) {
await migrate();
}
}
export default {
migrateIfNecessary,
isDbUpToDate
};

6
apps/server/src/services/sql_init.ts
View File

@@ -1,11 +1,5 @@
import { type OptionRow } from "@triliumnext/commons";
import { sql_init as coreSqlInit } from "@triliumnext/core";
import fs from "fs";
import BOption from "../becca/entities/boption.js";
import log from "./log.js";
import resourceDir from "./resource_dir.js";
import sql from "./sql.js";
const schemaExists = coreSqlInit.schemaExists;
const isDbInitialized = coreSqlInit.isDbInitialized;

9
apps/server/src/migrations/0216__move_content_into_blobs.ts → packages/trilium-core/src/migrations/0216__move_content_into_blobs.ts
View File

@@ -1,5 +1,5 @@
import sql from "../services/sql.js";
import utils from "../services/utils.js";
import { getSql } from "../services/sql/index";
import { hashedBlobId } from "../services/utils/index";
interface NoteContentsRow {
noteId: string;
@@ -17,9 +17,10 @@ interface NoteRevisionContents {
export default () => {
const existingBlobIds = new Set();
const sql = getSql();
for (const noteId of sql.getColumn<string>(/*sql*/`SELECT noteId FROM note_contents`)) {
const row = sql.getRow<NoteContentsRow>(/*sql*/`SELECT noteId, content, dateModified, utcDateModified FROM note_contents WHERE noteId = ?`, [noteId]);
const blobId = utils.hashedBlobId(row.content);
const blobId = hashedBlobId(row.content);
if (!existingBlobIds.has(blobId)) {
existingBlobIds.add(blobId);
@@ -42,7 +43,7 @@ export default () => {
for (const noteRevisionId of sql.getColumn(/*sql*/`SELECT noteRevisionId FROM note_revision_contents`)) {
const row = sql.getRow<NoteRevisionContents>(/*sql*/`SELECT noteRevisionId, content, utcDateModified FROM note_revision_contents WHERE noteRevisionId = ?`, [noteRevisionId]);
const blobId = utils.hashedBlobId(row.content);
const blobId = hashedBlobId(row.content);
if (!existingBlobIds.has(blobId)) {
existingBlobIds.add(blobId);

14
apps/server/src/migrations/0220__migrate_images_to_attachments.ts → packages/trilium-core/src/migrations/0220__migrate_images_to_attachments.ts
View File

@@ -1,12 +1,14 @@
import { becca_loader } from "@triliumnext/core";
import becca from "../becca/becca.js";
import cls from "../services/cls.js";
import log from "../services/log.js";
import sql from "../services/sql.js";
import { getLog } from "../services/log.js";
import { getSql } from "../services/sql/index.js";
import { getContext } from "../services/context.js";
import becca_loader from "../becca/becca_loader.js";
export default () => {
cls.init(() => {
getContext().init(() => {
const sql = getSql();
const log = getLog();
// emergency disabling of image compression since it appears to make problems in migration to 0.61
sql.execute(/*sql*/`UPDATE options SET value = 'false' WHERE name = 'compressImages'`);

14
apps/server/src/migrations/0233__migrate_geo_map_to_collection.spec.ts → packages/trilium-core/src/migrations/0233__migrate_geo_map_to_collection.spec.ts
View File

@@ -1,6 +1,6 @@
import { describe, expect, it, beforeEach } from "vitest";
import cls from "../services/cls.js";
import sql from "../services/sql.js";
import * as cls from "../services/context.js";
import { getSql } from "../services/sql/index.js";
import becca from "../becca/becca.js";
import becca_loader from "../becca/becca_loader.js";
import migration from "./0233__migrate_geo_map_to_collection.js";
@@ -19,12 +19,14 @@ import migration from "./0233__migrate_geo_map_to_collection.js";
* test data into the database, then verifies the migration transforms the data correctly.
*/
describe("Migration 0233: Migrate geoMap to collection", () => {
const sql = getSql();
beforeEach(async () => {
// Set up a clean in-memory database for each test
sql.rebuildIntegrationTestDatabase();
await new Promise<void>((resolve) => {
cls.init(() => {
cls.getContext().init(() => {
becca_loader.load();
resolve();
});
@@ -33,7 +35,7 @@ describe("Migration 0233: Migrate geoMap to collection", () => {
it("should migrate geoMap notes to book type with viewConfig attachment", async () => {
await new Promise<void>((resolve) => {
cls.init(() => {
cls.getContext().init(() => {
// Create a test geoMap note with content
const geoMapContent = JSON.stringify({
markers: [
@@ -164,7 +166,7 @@ describe("Migration 0233: Migrate geoMap to collection", () => {
it("should handle existing viewConfig attachments with same title", async () => {
await new Promise<void>((resolve) => {
cls.init(() => {
cls.getContext().init(() => {
const geoMapContent = JSON.stringify({ test: "data" });
const testNoteId = "test_geo_note_existing";
const testBlobId = "test_blob_geo_existing";
@@ -226,7 +228,7 @@ describe("Migration 0233: Migrate geoMap to collection", () => {
it("should handle protected geoMap notes appropriately", async () => {
await new Promise<void>((resolve, reject) => {
cls.init(() => {
cls.getContext().init(() => {
const geoMapContent = JSON.stringify({
markers: [{ lat: 51.5074, lng: -0.1278, title: "London" }],
center: { lat: 51.5074, lng: -0.1278 },

7
apps/server/src/migrations/0233__migrate_geo_map_to_collection.ts → packages/trilium-core/src/migrations/0233__migrate_geo_map_to_collection.ts
View File

@@ -1,11 +1,10 @@
import { becca_loader } from "@triliumnext/core";
import becca from "../becca/becca";
import cls from "../services/cls.js";
import becca_loader from "../becca/becca_loader";
import { getContext } from "../services/context";
import hidden_subtree from "../services/hidden_subtree";
export default () => {
cls.init(() => {
getContext().init(() => {
becca_loader.load();
// Ensure the geomap template is generated.

4
apps/server/src/migrations/0234__migrate_ai_chat_to_code.ts → packages/trilium-core/src/migrations/0234__migrate_ai_chat_to_code.ts
View File

@@ -1,9 +1,9 @@
import becca from "../becca/becca";
import becca_loader from "../becca/becca_loader";
import cls from "../services/cls.js";
import { getContext } from "../services/context";
export default () => {
cls.init(() => {
getContext().init(() => {
becca_loader.load();
for (const note of Object.values(becca.notes)) {

0
apps/server/src/migrations/migrations.ts → packages/trilium-core/src/migrations/migrations.ts
View File

5
packages/trilium-core/src/services/backup.ts Normal file
View File

@@ -0,0 +1,5 @@
export default {
backupNow() {
console.warn("Backup not yet available.");
}
}

2
packages/trilium-core/src/services/hidden_subtree.ts
View File

@@ -8,7 +8,7 @@ import BNote from "../becca/entities/bnote.js";
import buildLaunchBarConfig from "./hidden_subtree_launcherbar.js";
import buildHiddenSubtreeTemplates from "./hidden_subtree_templates.js";
import { cleanUpHelp, getHelpHiddenSubtreeData } from "./in_app_help.js";
import * as migrationService from "./migration.js";
import migrationService from "./migration.js";
import noteService from "./notes.js";
import { getLog } from "./log.js";

0
apps/server/src/services/migration.spec.ts → packages/trilium-core/src/services/migration.spec.ts
View File

142
packages/trilium-core/src/services/migration.ts
View File

@@ -1,4 +1,140 @@
export function isDbUpToDate() {
// TODO: Implement.
return true;
import backupService from "./backup.js";
import { getSql } from "./sql/index.js";
import { getLog } from "./log.js";
import { crash } from "./utils/index.js";
import appInfo from "./app_info.js";
import * as cls from "./context.js";
import { t } from "i18next";
import MIGRATIONS from "../migrations/migrations.js";
interface MigrationInfo {
dbVersion: number;
/**
* If string, then the migration is an SQL script that will be executed.
* If a function, then the migration is a JavaScript/TypeScript module that will be executed.
*/
migration: string | (() => void);
}
async function migrate() {
const currentDbVersion = getDbVersion();
if (currentDbVersion < 214) {
await crash(t("migration.old_version"));
return;
}
// backup before attempting migration
if (!process.env.TRILIUM_INTEGRATION_TEST) {
await backupService.backupNow(
// creating a special backup for version 0.60.4, the changes in 0.61 are major.
currentDbVersion === 214 ? `before-migration-v060` : "before-migration"
);
}
const migrations = await prepareMigrations(currentDbVersion);
// all migrations are executed in one transaction - upgrade either succeeds, or the user can stay at the old version
// otherwise if half of the migrations succeed, user can't use any version - DB is too "new" for the old app,
// and too old for the new app version.
cls.setMigrationRunning(true);
const sql = getSql();
const log = getLog();
sql.transactional(() => {
for (const mig of migrations) {
try {
log.info(`Attempting migration to version ${mig.dbVersion}`);
executeMigration(mig);
sql.execute(
/*sql*/`UPDATE options
SET value = ?
WHERE name = ?`,
[mig.dbVersion.toString(), "dbVersion"]
);
log.info(`Migration to version ${mig.dbVersion} has been successful.`);
} catch (e: any) {
console.error(e);
crash(t("migration.error_message", { version: mig.dbVersion, stack: e.stack }));
break; // crash() is sometimes async
}
}
});
if (currentDbVersion === 214) {
// special VACUUM after the big migration
log.info("VACUUMing database, this might take a while ...");
sql.execute("VACUUM");
}
}
async function prepareMigrations(currentDbVersion: number): Promise<MigrationInfo[]> {
MIGRATIONS.sort((a, b) => a.version - b.version);
const migrations: MigrationInfo[] = [];
for (const migration of MIGRATIONS) {
const dbVersion = migration.version;
if (dbVersion > currentDbVersion) {
if ("sql" in migration) {
migrations.push({
dbVersion,
migration: migration.sql
});
} else {
// Due to ESM imports, the migration file needs to be imported asynchronously and thus cannot be loaded at migration time (since migration is not asynchronous).
// As such we have to preload the ESM.
migrations.push({
dbVersion,
migration: (await migration.module()).default
});
}
}
}
return migrations;
}
function executeMigration({ migration }: MigrationInfo) {
if (typeof migration === "string") {
console.log(`Migration with SQL script: ${migration}`);
getSql().executeScript(migration);
} else {
console.log("Migration with JS module");
migration();
};
}
function getDbVersion() {
return parseInt(getSql().getValue("SELECT value FROM options WHERE name = 'dbVersion'"));
}
function isDbUpToDate() {
const dbVersion = getDbVersion();
const upToDate = dbVersion >= appInfo.dbVersion;
if (!upToDate) {
getLog().info(`App db version is ${appInfo.dbVersion}, while db version is ${dbVersion}. Migration needed.`);
}
return upToDate;
}
async function migrateIfNecessary() {
const currentDbVersion = getDbVersion();
if (currentDbVersion > appInfo.dbVersion && process.env.TRILIUM_IGNORE_DB_VERSION !== "true") {
await crash(t("migration.wrong_db_version", { version: currentDbVersion, targetVersion: appInfo.dbVersion }));
}
if (!isDbUpToDate()) {
await migrate();
}
}
export default {
migrateIfNecessary,
isDbUpToDate
};

4
packages/trilium-core/src/services/sql_init.ts
View File

@@ -12,6 +12,7 @@ import BBranch from "../becca/entities/bbranch";
import hidden_subtree from "./hidden_subtree";
import TaskContext from "./task_context";
import BOption from "../becca/entities/boption";
import migrationService from "./migration";
export const dbReady = deferred<void>();
@@ -52,8 +53,7 @@ async function initDbConnection() {
return;
}
//TODO: Renable migration
//await migrationService.migrateIfNecessary();
await migrationService.migrateIfNecessary();
const sql = getSql();
sql.execute('CREATE TEMP TABLE IF NOT EXISTS "param_list" (`paramId` TEXT NOT NULL PRIMARY KEY)');

4
packages/trilium-core/src/services/utils/index.ts
View File

@@ -202,3 +202,7 @@ export function isEmptyOrWhitespace(str: string | null | undefined) {
export function escapeRegExp(str: string) {
return str.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1");
}
export async function crash(message: string): never {
throw new Error(message);
}