sql_init/migration fixes

db/migrations/0161__drop_source_ids.sql

@@ -0,0 +1 @@
+DROP TABLE source_ids;

db/schema.sql

@@ -1,8 +1,3 @@
-CREATE TABLE IF NOT EXISTS "source_ids" (
-                                          `sourceId`	TEXT NOT NULL,
-                                          `utcDateCreated`	TEXT NOT NULL,
-                                          PRIMARY KEY(`sourceId`)
-);
 CREATE TABLE IF NOT EXISTS "api_tokens"
 (
   apiTokenId TEXT PRIMARY KEY NOT NULL,
@@ -57,8 +52,6 @@ CREATE INDEX `IDX_note_revisions_utcDateCreated` ON `note_revisions` (`utcDateCr
 CREATE INDEX `IDX_note_revisions_utcDateLastEdited` ON `note_revisions` (`utcDateLastEdited`);
 CREATE INDEX `IDX_note_revisions_dateCreated` ON `note_revisions` (`dateCreated`);
 CREATE INDEX `IDX_note_revisions_dateLastEdited` ON `note_revisions` (`dateLastEdited`);
-CREATE INDEX IDX_source_ids_utcDateCreated
-    on source_ids (utcDateCreated);
 CREATE TABLE IF NOT EXISTS "notes" (
                                            `noteId`	TEXT NOT NULL,
                                            `title`	TEXT NOT NULL DEFAULT "note",

src/services/app_info.js

@@ -4,7 +4,7 @@ const build = require('./build');
 const packageJson = require('../../package');
 const {TRILIUM_DATA_DIR} = require('./data_dir');
 
-const APP_DB_VERSION = 160;
+const APP_DB_VERSION = 161;
 const SYNC_VERSION = 14;
 const CLIPPER_PROTOCOL_VERSION = "1.0";
 

src/services/backup.js

@@ -99,14 +99,8 @@ if (!fs.existsSync(dataDir.BACKUP_DIR)) {
     fs.mkdirSync(dataDir.BACKUP_DIR, 0o700);
 }
 
-sqlInit.dbReady.then(() => {
-    setInterval(cls.wrap(regularBackup), 4 * 60 * 60 * 1000);
-
-    // kickoff first backup soon after start up
-    setTimeout(cls.wrap(regularBackup), 5 * 60 * 1000);
-});
-
 module.exports = {
     backupNow,
-    anonymize
+    anonymize,
+    regularBackup
 };

src/services/migration.js

@@ -5,12 +5,13 @@ const fs = require('fs-extra');
 const log = require('./log');
 const utils = require('./utils');
 const resourceDir = require('./resource_dir');
+const appInfo = require('./app_info');
 
-function migrate() {
+async function migrate() {
     const migrations = [];
 
     // backup before attempting migration
-    backupService.backupNow("before-migration");
+    await backupService.backupNow("before-migration");
 
     const currentDbVersion = parseInt(optionService.getOption('dbVersion'));
 
@@ -73,14 +74,38 @@ function migrate() {
             utils.crash();
         }
     }
+}
 
-    const sqlInit = require('./sql_init');
+function getDbVersion() {
+    return parseInt(sql.getValue("SELECT value FROM options WHERE name = 'dbVersion'"));
+}
 
-    if (sqlInit.isDbUpToDate()) {
+function isDbUpToDate() {
-        sqlInit.initDbConnection();
+    const dbVersion = getDbVersion();
+
+    const upToDate = dbVersion >= appInfo.dbVersion;
+
+    if (!upToDate) {
+        log.info("App db version is " + appInfo.dbVersion + ", while db version is " + dbVersion + ". Migration needed.");
+    }
+
+    return upToDate;
+}
+
+async function migrateIfNecessary() {
+    const currentDbVersion = getDbVersion();
+
+    if (currentDbVersion > appInfo.dbVersion) {
+        log.error(`Current DB version ${currentDbVersion} is newer than app db version ${appInfo.dbVersion} which means that it was created by newer and incompatible version of Trilium. Upgrade to latest version of Trilium to resolve this issue.`);
+
+        utils.crash();
+    }
+
+    if (!isDbUpToDate()) {
+        await migrate();
     }
 }
 
 module.exports = {
-    migrate
+    migrateIfNecessary
 };

src/services/source_id.js

@@ -1,53 +1,20 @@
 const utils = require('./utils');
-const dateUtils = require('./date_utils');
-const log = require('./log');
-const sql = require('./sql');
-const sqlInit = require('./sql_init');
-const cls = require('./cls');
 
-function saveSourceId(sourceId) {
+const localSourceIds = {};
-    sql.insert("source_ids", {
-        sourceId: sourceId,
-        utcDateCreated: dateUtils.utcNowDateTime()
-    });
-
-    refreshSourceIds();
-}
-
-function createSourceId() {
-    const sourceId = utils.randomString(12);
-
-    log.info("Generated sourceId=" + sourceId);
-    return sourceId;
-}
 
 function generateSourceId() {
-    const sourceId = createSourceId();
+    const sourceId = utils.randomString(12);
 
-    saveSourceId(sourceId);
+    localSourceIds[sourceId] = true;
 
     return sourceId;
 }
 
-function refreshSourceIds() {
-    const sourceIdsArr = sql.getColumn("SELECT sourceId FROM source_ids ORDER BY utcDateCreated DESC");
-
-    allSourceIds = {};
-
-    for (const sourceId of sourceIdsArr) {
-        allSourceIds[sourceId] = true;
-    }
-}
-
-let allSourceIds = {};
-
 function isLocalSourceId(srcId) {
-    return !!allSourceIds[srcId];
+    return !!localSourceIds[srcId];
 }
 
-const currentSourceId = createSourceId();
+const currentSourceId = generateSourceId();
-
-sqlInit.dbReady.then(cls.wrap(() => saveSourceId(currentSourceId)));
 
 function getCurrentSourceId() {
     return currentSourceId;

src/services/sql_init.js

@@ -8,6 +8,7 @@ const optionService = require('./options');
 const port = require('./port');
 const Option = require('../entities/option');
 const TaskContext = require('./task_context.js');
+const migrationService = require('./migration');
 
 const dbReady = utils.deferred();
 
@@ -28,27 +29,14 @@ function isDbInitialized() {
     return initialized === 'true';
 }
 
-function initDbConnection() {
+async function initDbConnection() {
     if (!isDbInitialized()) {
         log.info(`DB not initialized, please visit setup page` + (utils.isElectron() ? '' : ` - http://[your-server-host]:${port} to see instructions on how to initialize Trilium.`));
 
         return;
     }
 
-    const currentDbVersion = getDbVersion();
+    await migrationService.migrateIfNecessary();
-
-    if (currentDbVersion > appInfo.dbVersion) {
-        log.error(`Current DB version ${currentDbVersion} is newer than app db version ${appInfo.dbVersion} which means that it was created by newer and incompatible version of Trilium. Upgrade to latest version of Trilium to resolve this issue.`);
-
-        utils.crash();
-    }
-
-    if (!isDbUpToDate()) {
-        // avoiding circular dependency
-        const migrationService = require('./migration');
-
-        migrationService.migrate();
-    }
 
     require('./options_init').initStartupOptions();
 
@@ -136,22 +124,6 @@ function createDatabaseForSync(options, syncServerHost = '', syncProxy = '') {
     log.info("Schema and not synced options generated.");
 }
 
-function getDbVersion() {
-    return parseInt(sql.getValue("SELECT value FROM options WHERE name = 'dbVersion'"));
-}
-
-function isDbUpToDate() {
-    const dbVersion = getDbVersion();
-
-    const upToDate = dbVersion >= appInfo.dbVersion;
-
-    if (!upToDate) {
-        log.info("App db version is " + appInfo.dbVersion + ", while db version is " + dbVersion + ". Migration needed.");
-    }
-
-    return upToDate;
-}
-
 function setDbAsInitialized() {
     if (!isDbInitialized()) {
         optionService.setOption('initialized', 'true');
@@ -160,6 +132,13 @@ function setDbAsInitialized() {
     }
 }
 
+dbReady.then(() => {
+    setInterval(() => require('./backup').regularBackup(), 4 * 60 * 60 * 1000);
+
+    // kickoff first backup soon after start up
+    setTimeout(() => require('./backup').regularBackup(), 5 * 60 * 1000);
+});
+
 log.info("DB size: " + sql.getValue("SELECT page_count * page_size / 1000 as size FROM pragma_page_count(), pragma_page_size()") + " KB");
 
 module.exports = {
@@ -167,7 +146,6 @@ module.exports = {
     schemaExists,
     isDbInitialized,
     initDbConnection,
-    isDbUpToDate,
     createInitialDatabase,
     createDatabaseForSync,
     setDbAsInitialized

src/services/sync.js

@@ -104,7 +104,7 @@ async function doLogin() {
     });
 
     if (sourceIdService.isLocalSourceId(resp.sourceId)) {
-        throw new Error(`Sync server has source ID ${resp.sourceId} which is also local. Try restarting sync server.`);
+        throw new Error(`Sync server has source ID ${resp.sourceId} which is also local. Your sync setup is probably trying to connect to itself.`);
     }
 
     syncContext.sourceId = resp.sourceId;