#98, sync to server now works as well + a lot of related changes

src/services/app_info.js

@@ -3,7 +3,7 @@
 const build = require('./build');
 const packageJson = require('../../package');
 
-const APP_DB_VERSION = 103;
+const APP_DB_VERSION = 104;
 const SYNC_VERSION = 1;
 
 module.exports = {

src/services/auth.js

@@ -38,6 +38,15 @@ async function checkApiAuth(req, res, next) {
     }
 }
 
+async function checkAppInitialized(req, res, next) {
+    if (!await sqlInit.isDbInitialized()) {
+        res.redirect("setup");
+    }
+    else {
+        next();
+    }
+}
+
 async function checkAppNotInitialized(req, res, next) {
     if (await sqlInit.isDbInitialized()) {
         res.status(400).send("App already initialized.");
@@ -77,6 +86,7 @@ async function checkBasicAuth(req, res, next) {
 module.exports = {
     checkAuth,
     checkApiAuth,
+    checkAppInitialized,
     checkAppNotInitialized,
     checkApiAuthOrElectron,
     checkSenderToken,

src/services/migration.js

@@ -56,28 +56,23 @@ async function migrate() {
                 else if (mig.type === 'js') {
                     console.log("Migration with JS module");
 
-                    const migrationModule = require("../" + resourceDir.MIGRATIONS_DIR + "/" + mig.file);
-                    await migrationModule(db);
+                    const migrationModule = require(resourceDir.MIGRATIONS_DIR + "/" + mig.file);
+                    await migrationModule();
                 }
                 else {
                     throw new Error("Unknown migration type " + mig.type);
                 }
 
                 await optionService.setOption("dbVersion", mig.dbVersion);
-
             });
 
             log.info("Migration to version " + mig.dbVersion + " has been successful.");
-
-            mig['success'] = true;
         }
         catch (e) {
-            mig['success'] = false;
-            mig['error'] = e.stack;
-
             log.error("error during migration to version " + mig.dbVersion + ": " + e.stack);
+            log.error("migration failed, crashing hard"); // this is not very user friendly :-/
 
-            break;
+            process.exit(1);
         }
         finally {
             // make sure foreign keys are enabled even if migration script disables them
@@ -88,8 +83,6 @@ async function migrate() {
     if (await sqlInit.isDbUpToDate()) {
         await sqlInit.initDbConnection();
     }
-
-    return migrations;
 }
 
 module.exports = {

src/services/setup.js

@@ -1,15 +1,16 @@
-const sqlInit = require('./sql_init');
-const sql = require('./sql');
 const rp = require('request-promise');
-const Option = require('../entities/option');
 const syncService = require('./sync');
 const log = require('./log');
-const optionService = require('./options');
+const sqlInit = require('./sql_init');
 
 function triggerSync() {
-// it's ok to not wait for it here
-    syncService.sync().then(async () => {
-        await optionService.setOption('initialized', 'true');
+    log.info("Triggering sync.");
+
+    // it's ok to not wait for it here
+    syncService.sync().then(async res => {
+        if (res.success) {
+            await sqlInit.dbInitialized();
+        }
     });
 }
 
@@ -34,17 +35,7 @@ async function setupSyncFromSyncServer(serverAddress, username, password) {
             json: true
         });
 
-        log.info("Creating database for sync");
-
-        await sql.transactional(async () => {
-            await sqlInit.createDatabaseForSync(serverAddress);
-
-            for (const opt of options) {
-                await new Option(opt).save();
-            }
-        });
-
-        log.info("Triggering sync.");
+        await sqlInit.createDatabaseForSync(options, serverAddress);
 
         triggerSync();
 

src/services/sql_init.js

@@ -6,6 +6,8 @@ const resourceDir = require('./resource_dir');
 const appInfo = require('./app_info');
 const sql = require('./sql');
 const cls = require('./cls');
+const optionService = require('./options');
+const Option = require('../entities/option');
 
 async function createConnection() {
     return await sqlite.open(dataDir.DOCUMENT_PATH, {Promise});
@@ -35,19 +37,20 @@ async function isDbInitialized() {
 
     const initialized = await sql.getValue("SELECT value FROM options WHERE name = 'initialized'");
 
-    return initialized === 'true';
+    // !initialized may be removed in the future, required only for migration
+    return !initialized || initialized === 'true';
 }
 
 async function initDbConnection() {
     await cls.init(async () => {
-        await sql.execute("PRAGMA foreign_keys = ON");
-
         if (!await isDbInitialized()) {
             log.info("DB not initialized, please visit setup page to initialize Trilium.");
 
             return;
         }
 
+        await sql.execute("PRAGMA foreign_keys = ON");
+
         if (!await isDbUpToDate()) {
             // avoiding circular dependency
             const migrationService = require('./migration');
@@ -96,8 +99,12 @@ async function createInitialDatabase(username, password) {
     await initDbConnection();
 }
 
-async function createDatabaseForSync(syncServerHost) {
-    log.info("Creating database for sync with server ...");
+async function createDatabaseForSync(options, syncServerHost = '') {
+    log.info("Creating database for sync");
+
+    if (await isDbInitialized()) {
+        throw new Error("DB is already initialized");
+    }
 
     const schema = fs.readFileSync(resourceDir.DB_INIT_DIR + '/schema.sql', 'UTF-8');
 
@@ -105,6 +112,11 @@ async function createDatabaseForSync(syncServerHost) {
         await sql.executeScript(schema);
 
         await require('./options_init').initNotSyncedOptions(false, '', syncServerHost);
+
+        // document options required for sync to kick off
+        for (const opt of options) {
+            await new Option(opt).save();
+        }
     });
 
     log.info("Schema and not synced options generated.");
@@ -122,6 +134,12 @@ async function isDbUpToDate() {
     return upToDate;
 }
 
+async function dbInitialized() {
+    await optionService.setOption('initialized', 'true');
+
+    await initDbConnection();
+}
+
 module.exports = {
     dbReady,
     schemaExists,
@@ -129,5 +147,6 @@ module.exports = {
     initDbConnection,
     isDbUpToDate,
     createInitialDatabase,
-    createDatabaseForSync
+    createDatabaseForSync,
+    dbInitialized
 };

src/services/sync.js

@@ -10,7 +10,6 @@ const sourceIdService = require('./source_id');
 const dateUtils = require('./date_utils');
 const syncUpdateService = require('./sync_update');
 const contentHashService = require('./content_hash');
-const fs = require('fs');
 const appInfo = require('./app_info');
 const syncSetup = require('./sync_setup');
 const syncMutexService = require('./sync_mutex');
@@ -25,7 +24,11 @@ const stats = {
 
 async function sync() {
     try {
-        await syncMutexService.doExclusively(async () => {
+        return await syncMutexService.doExclusively(async () => {
+            if (!await syncSetup.isSyncSetup()) {
+                return { success: false, message: 'Sync not configured' };
+            }
+
             const syncContext = await login();
 
             await pushSync(syncContext);
@@ -34,12 +37,14 @@ async function sync() {
 
             await pushSync(syncContext);
 
-            await checkContentHash(syncContext);
-        });
+            await syncFinished(syncContext);
 
-        return {
-            success: true
-        };
+            await checkContentHash(syncContext);
+
+            return {
+                success: true
+            };
+        });
     }
     catch (e) {
         proxyToggle = !proxyToggle;
@@ -53,7 +58,7 @@ async function sync() {
             };
         }
         else {
-            log.info("sync failed: " + e.stack);
+            log.info("sync failed: " + e.message);
 
             return {
                 success: false,
@@ -104,7 +109,8 @@ async function pullSync(syncContext) {
 
         for (const {sync, entity} of rows) {
             if (sourceIdService.isLocalSourceId(sync.sourceId)) {
-                log.info(`Skipping pull #${sync.id} ${sync.entityName} ${sync.entityId} because ${sync.sourceId} is a local source id.`);
+                // too noisy
+                //log.info(`Skipping pull #${sync.id} ${sync.entityName} ${sync.entityId} because ${sync.sourceId} is a local source id.`);
             }
             else {
                 await syncUpdateService.updateEntity(sync, entity, syncContext.sourceId);
@@ -127,7 +133,8 @@ async function pushSync(syncContext) {
 
         const filteredSyncs = syncs.filter(sync => {
             if (sync.sourceId === syncContext.sourceId) {
-                log.info(`Skipping push #${sync.id} ${sync.entityName} ${sync.entityId} because it originates from sync target`);
+                // too noisy
+                //log.info(`Skipping push #${sync.id} ${sync.entityName} ${sync.entityId} because it originates from sync target`);
 
                 // this may set lastSyncedPush beyond what's actually sent (because of size limit)
                 // so this is applied to the database only if there's no actual update
@@ -168,6 +175,10 @@ async function pushSync(syncContext) {
     }
 }
 
+async function syncFinished(syncContext) {
+    await syncRequest(syncContext, 'POST', '/api/sync/finished');
+}
+
 async function checkContentHash(syncContext) {
     const resp = await syncRequest(syncContext, 'GET', '/api/sync/check');
 
@@ -210,7 +221,7 @@ async function syncRequest(syncContext, method, uri, body) {
         return await rp(options);
     }
     catch (e) {
-        throw new Error(`Request to ${method} ${fullUri} failed, inner exception: ${e.stack}`);
+        throw new Error(`Request to ${method} ${fullUri} failed, error: ${e.message}`);
     }
 }
 

src/services/sync_mutex.js

@@ -10,12 +10,11 @@ async function doExclusively(func) {
     const releaseMutex = await instance.acquire();
 
     try {
-        await func();
+        return await func();
     }
     finally {
         releaseMutex();
     }
-
 }
 
 module.exports = {

src/services/sync_setup.js

@@ -1,6 +1,5 @@
 "use strict";
 
-const config = require('./config');
 const optionService = require('./options');
 
 module.exports = {

src/services/sync_table.js

@@ -74,10 +74,11 @@ async function cleanupSyncRowsForMissingEntities(entityName, entityKey) {
         AND sync.entityId NOT IN (SELECT ${entityKey} FROM ${entityName})`);
 }
 
-async function fillSyncRows(entityName, entityKey) {
+async function fillSyncRows(entityName, entityKey, condition = '') {
     await cleanupSyncRowsForMissingEntities(entityName, entityKey);
 
-    const entityIds = await sql.getColumn(`SELECT ${entityKey} FROM ${entityName}`);
+    const entityIds = await sql.getColumn(`SELECT ${entityKey} FROM ${entityName}`
+        + (condition ? ` WHERE ${condition}` : ''));
 
     for (const entityId of entityIds) {
         const existingRows = await sql.getValue("SELECT COUNT(id) FROM sync WHERE entityName = ? AND entityId = ?", [entityName, entityId]);
@@ -107,6 +108,7 @@ async function fillAllSyncRows() {
     await fillSyncRows("note_images", "noteImageId");
     await fillSyncRows("labels", "labelId");
     await fillSyncRows("api_tokens", "apiTokenId");
+    await fillSyncRows("options", "name", 'isSynced = 1');
 }
 
 module.exports = {

src/services/sync_update.js

@@ -109,7 +109,7 @@ async function updateNoteReordering(entityId, entity, sourceId) {
 async function updateOptions(entity, sourceId) {
     const orig = await sql.getRowOrNull("SELECT * FROM options WHERE name = ?", [entity.name]);
 
-    if (!orig.isSynced) {
+    if (orig && !orig.isSynced) {
         return;
     }