use better-sqlite3

electron.js

@@ -24,8 +24,6 @@ app.on('window-all-closed', () => {
 app.on('ready', async () => {
     app.setAppUserModelId('com.github.zadam.trilium');
 
-    await sqlInit.dbConnection;
-
     // if db is not initialized -> setup process
     // if db is initialized, then we need to wait until the migration process is finished
     if (await sqlInit.isDbInitialized()) {

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "trilium",
-  "version": "0.42.7",
+  "version": "0.43.0-beta",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -1487,6 +1487,32 @@
       "resolved": "https://registry.npmjs.org/beeper/-/beeper-1.1.1.tgz",
       "integrity": "sha1-5tXqjF2tABMEpwsiY4RH9pyy+Ak="
     },
+    "better-sqlite3": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-7.1.0.tgz",
+      "integrity": "sha512-FV/snQ8F/kyqhdxsevzbojVtMowDWOfe1A5N3lYu1KJwoho2t7JgITmdlSc7DkOh3Zq65I+ZyeNWXQrkLEDFTg==",
+      "requires": {
+        "bindings": "^1.5.0",
+        "prebuild-install": "^5.3.3",
+        "tar": "4.4.10"
+      },
+      "dependencies": {
+        "tar": {
+          "version": "4.4.10",
+          "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.10.tgz",
+          "integrity": "sha512-g2SVs5QIxvo6OLp0GudTqEf05maawKUxXru104iaayWA09551tFCTI8f1Asb4lPfkBr91k07iL4c11XO3/b0tA==",
+          "requires": {
+            "chownr": "^1.1.1",
+            "fs-minipass": "^1.2.5",
+            "minipass": "^2.3.5",
+            "minizlib": "^1.2.1",
+            "mkdirp": "^0.5.0",
+            "safe-buffer": "^5.1.2",
+            "yallist": "^3.0.3"
+          }
+        }
+      }
+    },
     "big.js": {
       "version": "5.2.2",
       "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz",
@@ -1591,6 +1617,14 @@
         "os-filter-obj": "^1.0.0"
       }
     },
+    "bindings": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
+      "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
+      "requires": {
+        "file-uri-to-path": "1.0.0"
+      }
+    },
     "bl": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz",
@@ -4186,6 +4220,11 @@
         }
       }
     },
+    "expand-template": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
+      "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg=="
+    },
     "expect-ct": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/expect-ct/-/expect-ct-0.2.0.tgz",
@@ -4458,6 +4497,11 @@
         "typedarray-to-buffer": "^3.1.5"
       }
     },
+    "file-uri-to-path": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
+      "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="
+    },
     "filelist": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.1.tgz",
@@ -4978,6 +5022,11 @@
         "logalot": "^2.0.0"
       }
     },
+    "github-from-package": {
+      "version": "0.0.0",
+      "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
+      "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4="
+    },
     "glob": {
       "version": "5.0.15",
       "resolved": "https://registry.npmjs.org/glob/-/glob-5.0.15.tgz",
@@ -7243,6 +7292,11 @@
         }
       }
     },
+    "mkdirp-classic": {
+      "version": "0.5.3",
+      "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
+      "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
+    },
     "modify-filename": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/modify-filename/-/modify-filename-1.1.0.tgz",
@@ -7876,6 +7930,11 @@
       "integrity": "sha512-s/snB+WGm6uwi0WjsZdaVcuf3KJXlfGl2LcxgwkEwJF0D/BWzVWAZW/XY4bFaiR7s0Jk3FPvlnepg1H1b1UwlA==",
       "dev": true
     },
+    "napi-build-utils": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz",
+      "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg=="
+    },
     "needle": {
       "version": "2.4.1",
       "resolved": "https://registry.npmjs.org/needle/-/needle-2.4.1.tgz",
@@ -7923,9 +7982,9 @@
       "integrity": "sha512-0L9FvHG3nfnnmaEQPjT9xhfN4ISk0A8/2j4M37Np4mcDesJjHgEUfgPhdCyZuFI954tjokaIj/A3NdpFNdEh4Q=="
     },
     "node-abi": {
-      "version": "2.16.0",
+      "version": "2.18.0",
-      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.16.0.tgz",
+      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.18.0.tgz",
-      "integrity": "sha512-+sa0XNlWDA6T+bDLmkCUYn6W5k5W6BPRL6mqzSCs6H/xUgtl4D5x2fORKDzopKiU6wsyn/+wXlRXwXeSp+mtoA==",
+      "integrity": "sha512-yi05ZoiuNNEbyT/xXfSySZE+yVnQW6fxPZuFbLyS1s6b5Kw3HzV2PHOM4XR+nsjzkHxByK+2Wg+yCQbe35l8dw==",
       "requires": {
         "semver": "^5.4.1"
       },
@@ -8037,6 +8096,11 @@
       "resolved": "https://registry.npmjs.org/node-status-codes/-/node-status-codes-1.0.0.tgz",
       "integrity": "sha1-WuVUHQJGRdMqWPzdyc7s6nrjrC8="
     },
+    "noop-logger": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz",
+      "integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI="
+    },
     "nopt": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz",
@@ -9393,6 +9457,43 @@
         "is-number-like": "^1.0.3"
       }
     },
+    "prebuild-install": {
+      "version": "5.3.4",
+      "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.4.tgz",
+      "integrity": "sha512-AkKN+pf4fSEihjapLEEj8n85YIw/tN6BQqkhzbDc0RvEZGdkpJBGMUYx66AAMcPG2KzmPQS7Cm16an4HVBRRMA==",
+      "requires": {
+        "detect-libc": "^1.0.3",
+        "expand-template": "^2.0.3",
+        "github-from-package": "0.0.0",
+        "minimist": "^1.2.3",
+        "mkdirp": "^0.5.1",
+        "napi-build-utils": "^1.0.1",
+        "node-abi": "^2.7.0",
+        "noop-logger": "^0.1.1",
+        "npmlog": "^4.0.1",
+        "pump": "^3.0.0",
+        "rc": "^1.2.7",
+        "simple-get": "^3.0.3",
+        "tar-fs": "^2.0.0",
+        "tunnel-agent": "^0.6.0",
+        "which-pm-runs": "^1.0.0"
+      },
+      "dependencies": {
+        "minimist": {
+          "version": "1.2.5",
+          "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
+          "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
+        },
+        "tunnel-agent": {
+          "version": "0.6.0",
+          "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
+          "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
+          "requires": {
+            "safe-buffer": "^5.0.1"
+          }
+        }
+      }
+    },
     "prelude-ls": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",
@@ -10225,6 +10326,36 @@
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
       "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0="
     },
+    "simple-concat": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.0.tgz",
+      "integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY="
+    },
+    "simple-get": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz",
+      "integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==",
+      "requires": {
+        "decompress-response": "^4.2.0",
+        "once": "^1.3.1",
+        "simple-concat": "^1.0.0"
+      },
+      "dependencies": {
+        "decompress-response": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz",
+          "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==",
+          "requires": {
+            "mimic-response": "^2.0.0"
+          }
+        },
+        "mimic-response": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz",
+          "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA=="
+        }
+      }
+    },
     "simple-node-logger": {
       "version": "18.12.24",
       "resolved": "https://registry.npmjs.org/simple-node-logger/-/simple-node-logger-18.12.24.tgz",
@@ -10633,6 +10764,17 @@
         "yallist": "^3.0.3"
       }
     },
+    "tar-fs": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.0.tgz",
+      "integrity": "sha512-9uW5iDvrIMCVpvasdFHW0wJPez0K4JnMZtsuIeDI7HyMGJNxmDZDOCQROr7lXyS+iL/QMpj07qcjGYTSdRFXUg==",
+      "requires": {
+        "chownr": "^1.1.1",
+        "mkdirp-classic": "^0.5.2",
+        "pump": "^3.0.0",
+        "tar-stream": "^2.0.0"
+      }
+    },
     "tar-stream": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.2.tgz",
@@ -11777,6 +11919,11 @@
       "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz",
       "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho="
     },
+    "which-pm-runs": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz",
+      "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs="
+    },
     "wide-align": {
       "version": "1.1.3",
       "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz",

package.json

@@ -23,6 +23,7 @@
   "dependencies": {
     "async-mutex": "0.2.2",
     "axios": "0.19.2",
+    "better-sqlite3": "^7.1.0",
     "body-parser": "1.19.0",
     "cls-hooked": "4.2.2",
     "commonmark": "0.29.1",
@@ -54,7 +55,7 @@
     "jimp": "0.10.3",
     "mime-types": "2.1.27",
     "multer": "1.4.2",
-    "node-abi": "2.16.0",
+    "node-abi": "2.18.0",
     "open": "7.0.3",
     "portscanner": "2.2.0",
     "rand-token": "1.0.1",

src/routes/api/export.js

@@ -23,7 +23,11 @@ async function exportBranch(req, res) {
 
     try {
         if (type === 'subtree' && (format === 'html' || format === 'markdown')) {
+            const start = Date.now();
+
             await zipExportService.exportToZip(taskContext, branch, format, res);
+
+            console.log("Export took", Date.now() - start, "ms");
         }
         else if (type === 'single') {
             await singleExportService.exportSingleNote(taskContext, branch, format, res);

src/routes/api/import.js

@@ -50,7 +50,11 @@ async function importToBranch(req) {
         if (extension === '.tar' && options.explodeArchives) {
             note = await tarImportService.importTar(taskContext, file.buffer, parentNote);
         } else if (extension === '.zip' && options.explodeArchives) {
+            const start = Date.now();
+
             note = await zipImportService.importZip(taskContext, file.buffer, parentNote);
+
+            console.log("Import took", Date.now() - start, "ms");
         } else if (extension === '.opml' && options.explodeArchives) {
             note = await opmlImportService.importOpml(taskContext, file.buffer, parentNote);
         } else if (extension === '.enex' && options.explodeArchives) {
@@ -85,4 +89,4 @@ async function importToBranch(req) {
 
 module.exports = {
     importToBranch
-};
+};

src/services/sql.js

@@ -33,7 +33,7 @@ async function insert(tableName, rec, replace = false) {
 
     const res = await execute(query, Object.values(rec));
 
-    return res.lastID;
+    return res.lastInsertRowid;
 }
 
 async function replace(tableName, rec) {
@@ -49,34 +49,46 @@ async function upsert(tableName, primaryKey, rec) {
 
     const columns = keys.join(", ");
 
-    let i = 0;
+    const questionMarks = keys.map(colName => "@" + colName).join(", ");
 
-    const questionMarks = keys.map(p => ":" + i++).join(", ");
+    const updateMarks = keys.map(colName => `${colName} = @${colName}`).join(", ");
-
-    i = 0;
-
-    const updateMarks = keys.map(key => `${key} = :${i++}`).join(", ");
 
     const query = `INSERT INTO ${tableName} (${columns}) VALUES (${questionMarks}) 
                    ON CONFLICT (${primaryKey}) DO UPDATE SET ${updateMarks}`;
 
-    await execute(query, Object.values(rec));
+    for (const idx in rec) {
+        if (rec[idx] === true || rec[idx] === false) {
+            rec[idx] = rec[idx] ? 1 : 0;
+        }
+    }
+
+    await execute(query, rec);
 }
 
-async function beginTransaction() {
+const statementCache = {};
-    return await dbConnection.run("BEGIN");
+
+function stmt(sql) {
+    if (!(sql in statementCache)) {
+        statementCache[sql] = dbConnection.prepare(sql);
+    }
+
+    return statementCache[sql];
 }
 
-async function commit() {
+function beginTransaction() {
-    return await dbConnection.run("COMMIT");
+    return stmt("BEGIN").run();
 }
 
-async function rollback() {
+function commit() {
-    return await dbConnection.run("ROLLBACK");
+    return stmt("COMMIT").run();
+}
+
+function rollback() {
+    return stmt("ROLLBACK").run();
 }
 
 async function getRow(query, params = []) {
-    return await wrap(async db => db.get(query, ...params), query);
+    return wrap(() => stmt(query).get(params), query);
 }
 
 async function getRowOrNull(query, params = []) {
@@ -105,18 +117,25 @@ async function getManyRows(query, params) {
         const curParams = params.slice(0, Math.min(params.length, PARAM_LIMIT));
         params = params.slice(curParams.length);
 
+        const curParamsObj = {};
+
+        let j = 1;
+        for (const param of curParams) {
+            curParamsObj['param' + j++] = param;
+        }
+
         let i = 1;
-        const questionMarks = curParams.map(() => "?" + i++).join(",");
+        const questionMarks = curParams.map(() => ":param" + i++).join(",");
         const curQuery = query.replace(/\?\?\?/g, questionMarks);
 
-        results = results.concat(await getRows(curQuery, curParams));
+        results = results.concat(await getRows(curQuery, curParamsObj));
     }
 
     return results;
 }
 
 async function getRows(query, params = []) {
-    return await wrap(async db => db.all(query, ...params), query);
+    return wrap(() => stmt(query).all(params), query);
 }
 
 async function getMap(query, params = []) {
@@ -152,11 +171,11 @@ async function getColumn(query, params = []) {
 async function execute(query, params = []) {
     await startTransactionIfNecessary();
 
-    return await wrap(async db => db.run(query, ...params), query);
+    return wrap(() => stmt(query).run(params), query);
 }
 
 async function executeWithoutTransaction(query, params = []) {
-    await dbConnection.run(query, ...params);
+    await dbConnection.run(query, params);
 }
 
 async function executeMany(query, params) {
@@ -169,10 +188,10 @@ async function executeMany(query, params) {
 async function executeScript(query) {
     await startTransactionIfNecessary();
 
-    return await wrap(async db => db.exec(query), query);
+    return wrap(() => stmt.run(query), query);
 }
 
-async function wrap(func, query) {
+function wrap(func, query) {
     if (!dbConnection) {
         throw new Error("DB connection not initialized yet");
     }
@@ -182,7 +201,7 @@ async function wrap(func, query) {
     try {
         const startTimestamp = Date.now();
 
-        const result = await func(dbConnection);
+        const result = func(dbConnection);
 
         const milliseconds = Date.now() - startTimestamp;
         if (milliseconds >= 300) {

src/services/sql_init.js

@@ -1,8 +1,6 @@
 const log = require('./log');
 const dataDir = require('./data_dir');
 const fs = require('fs');
-const sqlite = require('sqlite');
-const sqlite3 = require('sqlite3');
 const resourceDir = require('./resource_dir');
 const appInfo = require('./app_info');
 const sql = require('./sql');
@@ -12,28 +10,14 @@ const optionService = require('./options');
 const port = require('./port');
 const Option = require('../entities/option');
 const TaskContext = require('./task_context.js');
+const Database = require('better-sqlite3');
 
-const dbConnection = new Promise(async (resolve, reject) => {
+const dbConnection = new Database(dataDir.DOCUMENT_PATH);
-    const db = await sqlite.open({
+dbConnection.pragma('journal_mode = WAL');
-        filename: dataDir.DOCUMENT_PATH,
-        driver: sqlite3.Database
-    });
 
-    db.run('PRAGMA journal_mode = WAL;');
+sql.setDbConnection(dbConnection);
 
-    sql.setDbConnection(db);
+const dbReady = initDbConnection();
-
-    resolve();
-});
-
-let dbReadyResolve = null;
-const dbReady = new Promise(async (resolve, reject) => {
-    dbReadyResolve = resolve;
-
-    await dbConnection;
-
-    initDbConnection();
-});
 
 async function schemaExists() {
     const tableResults = await sql.getRows("SELECT name FROM sqlite_master WHERE type='table' AND name='options'");
@@ -78,7 +62,6 @@ async function initDbConnection() {
         await require('./options_init').initStartupOptions();
 
         log.info("DB ready.");
-        dbReadyResolve();
     });
 }
 
@@ -189,7 +172,6 @@ dbReady.then(async () => {
 
 module.exports = {
     dbReady,
-    dbConnection,
     schemaExists,
     isDbInitialized,
     initDbConnection,
@@ -197,4 +179,4 @@ module.exports = {
     createInitialDatabase,
     createDatabaseForSync,
     dbInitialized
-};
+};