Merge remote-tracking branch 'origin/optimize-tree'

src/entities/note.js

@@ -777,7 +777,7 @@ class Note extends Entity {
      * @returns {NoteRevision[]}
      */
     getRevisions() {
-        return this.repository.getEntities("SELECT * FROM note_revisions WHERE noteId = ?", [this.noteId]);
+        return this.repository.getEntities("SELECT * FROM note_revisions WHERE isErased = 0 AND noteId = ?", [this.noteId]);
     }
 
     /**

src/public/app/dialogs/options/other.js

@@ -51,6 +51,12 @@ const TPL = `
         <label for="erase-notes-after-time-in-seconds">Erase notes after X seconds</label>
         <input class="form-control" id="erase-notes-after-time-in-seconds" type="number" min="0">
     </div>
+    
+    <p>You can also trigger erasing manually:</p>
+    
+    <button id="erase-deleted-notes-now-button" class="btn">Erase deleted notes now</button>
+    
+    <br/><br/>
 </div>
 
 <div>
@@ -117,6 +123,13 @@ export default class ProtectedSessionOptions {
             return false;
         });
 
+        this.$eraseDeletedNotesButton = $("#erase-deleted-notes-now-button");
+        this.$eraseDeletedNotesButton.on('click', () => {
+            server.post('notes/erase-deleted-notes-now').then(() => {
+                toastService.showMessage("Deleted notes have been erased.");
+            });
+        });
+
         this.$protectedSessionTimeout = $("#protected-session-timeout-in-seconds");
 
         this.$protectedSessionTimeout.on('change', () => {

src/public/app/entities/note_short.js

@@ -75,15 +75,17 @@ class NoteShort {
         this.parentToBranch[parentNoteId] = branchId;
     }
 
-    addChild(childNoteId, branchId) {
+    addChild(childNoteId, branchId, sort = true) {
         if (!this.children.includes(childNoteId)) {
             this.children.push(childNoteId);
         }
 
         this.childToBranch[childNoteId] = branchId;
 
+        if (sort) {
             this.sortChildren();
         }
+    }
 
     sortChildren() {
         const branchIdPos = {};

src/public/app/services/tree_cache.js

@@ -21,9 +21,6 @@ class TreeCache {
     async loadInitialTree() {
         const resp = await server.get('tree');
 
-        // FIXME: we need to do this to cover for ascendants of template notes which are not loaded
-        await this.loadParents(resp, false);
-
         // clear the cache only directly before adding new content which is important for e.g. switching to protected session
 
         /** @type {Object.<string, NoteShort>} */
@@ -44,50 +41,18 @@ class TreeCache {
     async loadSubTree(subTreeNoteId) {
         const resp = await server.get('tree?subTreeNoteId=' + subTreeNoteId);
 
-        await this.loadParents(resp, true);
-
         this.addResp(resp);
 
         return this.notes[subTreeNoteId];
     }
 
-    async loadParents(resp, additiveLoad) {
-        const noteIds = new Set(resp.notes.map(note => note.noteId));
-        const missingNoteIds = [];
-        const existingNotes = additiveLoad ? this.notes : {};
-
-        for (const branch of resp.branches) {
-            if (!(branch.parentNoteId in existingNotes) && !noteIds.has(branch.parentNoteId) && branch.parentNoteId !== 'none') {
-                missingNoteIds.push(branch.parentNoteId);
-            }
-        }
-
-        for (const attr of resp.attributes) {
-            if (attr.type === 'relation' && attr.name === 'template' && !(attr.value in existingNotes) && !noteIds.has(attr.value)) {
-                missingNoteIds.push(attr.value);
-            }
-
-            if (!(attr.noteId in existingNotes) && !noteIds.has(attr.noteId)) {
-                missingNoteIds.push(attr.noteId);
-            }
-        }
-
-        if (missingNoteIds.length > 0) {
-            const newResp = await server.post('tree/load', { noteIds: missingNoteIds });
-
-            resp.notes = resp.notes.concat(newResp.notes);
-            resp.branches = resp.branches.concat(newResp.branches);
-            resp.attributes = resp.attributes.concat(newResp.attributes);
-
-            await this.loadParents(resp, additiveLoad);
-        }
-    }
-
     addResp(resp) {
         const noteRows = resp.notes;
         const branchRows = resp.branches;
         const attributeRows = resp.attributes;
 
+        const noteIdsToSort = new Set();
+
         for (const noteRow of noteRows) {
             const {noteId} = noteRow;
 
@@ -154,7 +119,9 @@ class TreeCache {
             const parentNote = this.notes[branch.parentNoteId];
 
             if (parentNote) {
-                parentNote.addChild(branch.noteId, branch.branchId);
+                parentNote.addChild(branch.noteId, branch.branchId, false);
+
+                noteIdsToSort.add(parentNote.noteId);
             }
         }
 
@@ -179,6 +146,11 @@ class TreeCache {
                 }
             }
         }
+
+        // sort all of them at once, this avoids repeated sorts (#1480)
+        for (const noteId of noteIdsToSort) {
+            this.notes[noteId].sortChildren();
+        }
     }
 
     async reloadNotes(noteIds) {
@@ -190,7 +162,6 @@ class TreeCache {
 
         const resp = await server.post('tree/load', { noteIds });
 
-        await this.loadParents(resp, true);
         this.addResp(resp);
 
         const searchNoteIds = [];

src/public/app/widgets/note_detail.js

@@ -249,13 +249,22 @@ export default class NoteDetailWidget extends TabAwareWidget {
 
         this.$widget.find('.note-detail-printable:visible').printThis({
             header: $("<h2>").text(this.note && this.note.title).prop('outerHTML'),
-            footer: "<script>document.body.className += ' ck-content printed-content';</script>",
+            footer: `
+<script src="libraries/katex/katex.min.js"></script>
+<script src="libraries/katex/auto-render.min.js"></script>
+<script>
+    document.body.className += ' ck-content printed-content';
+    
+    renderMathInElement(document.body, {});
+</script>
+`,
             importCSS: false,
             loadCSS: [
                 "libraries/codemirror/codemirror.css",
                 "libraries/ckeditor/ckeditor-content.css",
                 "libraries/ckeditor/ckeditor-content.css",
                 "libraries/bootstrap/css/bootstrap.min.css",
+                "libraries/katex/katex.min.css",
                 "stylesheets/print.css",
                 "stylesheets/relation_map.css",
                 "stylesheets/themes.css"

src/routes/api/notes.js

@@ -193,6 +193,10 @@ function duplicateSubtree(req) {
     return noteService.duplicateSubtree(noteId, parentNoteId);
 }
 
+function eraseDeletedNotesNow() {
+    noteService.eraseDeletedNotesNow();
+}
+
 module.exports = {
     getNote,
     updateNote,
@@ -204,5 +208,6 @@ module.exports = {
     setNoteTypeMime,
     getRelationMap,
     changeTitle,
-    duplicateSubtree
+    duplicateSubtree,
+    eraseDeletedNotesNow
 };

src/routes/api/recent_changes.js

@@ -23,7 +23,8 @@ function getRecentChanges(req) {
             note_revisions.dateCreated AS date
         FROM 
             note_revisions
-            JOIN notes USING(noteId)`);
+            JOIN notes USING(noteId)
+        WHERE note_revisions.isErased = 0`);
 
     for (const noteRevision of noteRevisions) {
         if (noteCacheService.isInAncestor(noteRevision.noteId, ancestorNoteId)) {

src/routes/api/setup.js

@@ -38,6 +38,8 @@ function saveSyncSeed(req) {
         }]
     }
 
+    log.info("Saved sync seed.");
+
     sqlInit.createDatabaseForSync(options);
 }
 

src/routes/api/tree.js

@@ -5,14 +5,15 @@ const optionService = require('../../services/options');
 const treeService = require('../../services/tree');
 
 function getNotesAndBranchesAndAttributes(noteIds) {
-    noteIds = Array.from(new Set(noteIds));
+    const notes = treeService.getNotesIncludingAscendants(noteIds);
-    const notes = treeService.getNotes(noteIds);
 
-    noteIds = notes.map(note => note.noteId);
+    noteIds = new Set(notes.map(note => note.noteId));
+
+    sql.fillNoteIdList(noteIds);
 
     // joining child note to filter out not completely synchronised notes which would then cause errors later
     // cannot do that with parent because of root note's 'none' parent
-    const branches = sql.getManyRows(` 
+    const branches = sql.getRows(` 
         SELECT 
             branches.branchId,
             branches.noteId,
@@ -20,28 +21,45 @@ function getNotesAndBranchesAndAttributes(noteIds) {
             branches.notePosition,
             branches.prefix,
             branches.isExpanded
-        FROM branches
+        FROM param_list
+        JOIN branches ON param_list.paramId = branches.noteId OR param_list.paramId = branches.parentNoteId
         JOIN notes AS child ON child.noteId = branches.noteId
-        WHERE branches.isDeleted = 0
+        WHERE branches.isDeleted = 0`);
-          AND (branches.noteId IN (???) OR parentNoteId IN (???))`, noteIds);
+
+    const attributes = sql.getRows(`
+        SELECT
+            attributes.attributeId,
+            attributes.noteId,
+            attributes.type,
+            attributes.name,
+            attributes.value,
+            attributes.position,
+            attributes.isInheritable
+        FROM param_list
+        JOIN attributes ON attributes.noteId = param_list.paramId 
+                        OR (attributes.type = 'relation' AND attributes.value = param_list.paramId)
+        WHERE attributes.isDeleted = 0`);
+
+    // we don't really care about the direction of the relation
+    const missingTemplateNoteIds = attributes
+        .filter(attr => attr.type === 'relation'
+                && attr.name === 'template'
+                && !noteIds.has(attr.value))
+        .map(attr => attr.value);
+
+    if (missingTemplateNoteIds.length > 0) {
+        const templateData = getNotesAndBranchesAndAttributes(missingTemplateNoteIds);
+
+        // there are going to be duplicates with simple concatenation, however:
+        // 1) shouldn't matter for the frontend which will update the entity twice
+        // 2) there shouldn't be many duplicates. There isn't that many templates
+        addArrays(notes, templateData.notes);
+        addArrays(branches, templateData.branches);
+        addArrays(attributes, templateData.attributes);
+    }
 
     // sorting in memory is faster
     branches.sort((a, b) => a.notePosition - b.notePosition < 0 ? -1 : 1);
-
-    const attributes = sql.getManyRows(`
-        SELECT
-            attributeId,
-            noteId,
-            type,
-            name,
-            value,
-            position,
-            isInheritable
-        FROM attributes
-        WHERE isDeleted = 0 
-          AND (noteId IN (???) OR (type = 'relation' AND value IN (???)))`, noteIds);
-
-    // sorting in memory is faster
     attributes.sort((a, b) => a.position - b.position < 0 ? -1 : 1);
 
     return {
@@ -51,6 +69,16 @@ function getNotesAndBranchesAndAttributes(noteIds) {
     };
 }
 
+// should be fast based on https://stackoverflow.com/a/64826145/944162
+// in this case it is assumed that target is potentially much larger than elementsToAdd
+function addArrays(target, elementsToAdd) {
+    while (elementsToAdd.length) {
+        target.push(elementsToAdd.shift());
+    }
+
+    return target;
+}
+
 function getTree(req) {
     const subTreeNoteId = req.query.subTreeNoteId || 'root';
 
@@ -64,25 +92,8 @@ function getTree(req) {
                   JOIN treeWithDescendants ON branches.parentNoteId = treeWithDescendants.noteId
                 WHERE treeWithDescendants.isExpanded = 1 
                   AND branches.isDeleted = 0
-            ),
-            treeWithDescendantsAndAscendants AS (
-                SELECT noteId FROM treeWithDescendants
-                UNION
-                SELECT branches.parentNoteId FROM branches
-                  JOIN treeWithDescendantsAndAscendants ON branches.noteId = treeWithDescendantsAndAscendants.noteId
-                WHERE branches.isDeleted = 0
-                  AND branches.parentNoteId != ?
-            ),
-            treeWithDescendantsAscendantsAndTemplates AS (
-                SELECT noteId FROM treeWithDescendantsAndAscendants
-                UNION
-                SELECT attributes.value FROM attributes
-                   JOIN treeWithDescendantsAscendantsAndTemplates ON attributes.noteId = treeWithDescendantsAscendantsAndTemplates.noteId
-                WHERE attributes.isDeleted = 0
-                    AND attributes.type = 'relation'
-                    AND attributes.name = 'template'
             )
-        SELECT noteId FROM treeWithDescendantsAscendantsAndTemplates`, [subTreeNoteId, subTreeNoteId]);
+        SELECT noteId FROM treeWithDescendants`, [subTreeNoteId]);
 
     noteIds.push(subTreeNoteId);
 

src/routes/routes.js

@@ -154,6 +154,7 @@ function register(app) {
     route(GET, '/api/notes/:noteId/revisions/:noteRevisionId/download', [auth.checkApiAuthOrElectron], noteRevisionsApiRoute.downloadNoteRevision);
     apiRoute(PUT, '/api/notes/:noteId/restore-revision/:noteRevisionId', noteRevisionsApiRoute.restoreNoteRevision);
     apiRoute(POST, '/api/notes/relation-map', notesApiRoute.getRelationMap);
+    apiRoute(POST, '/api/notes/erase-deleted-notes-now', notesApiRoute.eraseDeletedNotesNow);
     apiRoute(PUT, '/api/notes/:noteId/change-title', notesApiRoute.changeTitle);
     apiRoute(POST, '/api/notes/:noteId/duplicate/:parentNoteId', notesApiRoute.duplicateSubtree);
 

src/services/consistency_checks.js

@@ -651,7 +651,7 @@ class ConsistencyChecks {
         // root branch should always be expanded
         sql.execute("UPDATE branches SET isExpanded = 1 WHERE branchId = 'root'");
 
-        if (this.unrecoveredConsistencyErrors) {
+        if (!this.unrecoveredConsistencyErrors) {
             // we run this only if basic checks passed since this assumes basic data consistency
 
             this.checkTreeCycles();

src/services/handlers.js

@@ -70,6 +70,10 @@ eventService.subscribe(eventService.ENTITY_CREATED, ({ entityName, entity }) =>
                 if (templateNoteContent) {
                     note.setContent(templateNoteContent);
                 }
+
+                note.type = templateNote.type;
+                note.mime = templateNote.mime;
+                note.save();
             }
 
             noteService.duplicateSubtreeWithoutRoot(templateNote.noteId, note.noteId);

src/services/note_revisions.js

@@ -2,6 +2,7 @@
 
 const NoteRevision = require('../entities/note_revision');
 const dateUtils = require('../services/date_utils');
+const log = require('../services/log');
 
 /**
  * @param {Note} note
@@ -9,6 +10,7 @@ const dateUtils = require('../services/date_utils');
 function protectNoteRevisions(note) {
     for (const revision of note.getRevisions()) {
         if (note.isProtected !== revision.isProtected) {
+            try {
                 const content = revision.getContent();
 
                 revision.isProtected = note.isProtected;
@@ -18,6 +20,12 @@ function protectNoteRevisions(note) {
 
                 revision.save();
             }
+            catch (e) {
+                log.error("Could not un/protect note revision ID = " + revision.noteRevisionId);
+
+                throw e;
+            }
+        }
     }
 }
 

src/services/notes.js

@@ -183,6 +183,7 @@ function protectNoteRecursively(note, protect, includingSubTree, taskContext) {
 }
 
 function protectNote(note, protect) {
+    try {
         if (protect !== note.isProtected) {
             const content = note.getContent();
 
@@ -195,6 +196,12 @@ function protectNote(note, protect) {
         }
 
         noteRevisionService.protectNoteRevisions(note);
+    }
+    catch (e) {
+        log.error("Could not un/protect note ID = " + note.noteId);
+
+        throw e;
+    }
 }
 
 function findImageLinks(content, foundLinks) {
@@ -459,7 +466,7 @@ function saveNoteRevision(note) {
     const revisionCutoff = dateUtils.utcDateStr(new Date(now.getTime() - noteRevisionSnapshotTimeInterval * 1000));
 
     const existingNoteRevisionId = sql.getValue(
-        "SELECT noteRevisionId FROM note_revisions WHERE noteId = ? AND utcDateCreated >= ?", [note.noteId, revisionCutoff]);
+        "SELECT noteRevisionId FROM note_revisions WHERE noteId = ? AND isErased = 0 AND utcDateCreated >= ?", [note.noteId, revisionCutoff]);
 
     const msSinceDateCreated = now.getTime() - dateUtils.parseDateTime(note.utcDateCreated).getTime();
 
@@ -666,8 +673,10 @@ function scanForLinks(note) {
     }
 }
 
-function eraseDeletedNotes() {
+function eraseDeletedNotes(eraseNotesAfterTimeInSeconds = null) {
-    const eraseNotesAfterTimeInSeconds = optionService.getOptionInt('eraseNotesAfterTimeInSeconds');
+    if (eraseNotesAfterTimeInSeconds === null) {
+        eraseNotesAfterTimeInSeconds = optionService.getOptionInt('eraseNotesAfterTimeInSeconds');
+    }
 
     const cutoffDate = new Date(Date.now() - eraseNotesAfterTimeInSeconds * 1000);
 
@@ -717,6 +726,10 @@ function eraseDeletedNotes() {
     log.info(`Erased notes: ${JSON.stringify(noteIdsToErase)}`);
 }
 
+function eraseDeletedNotesNow() {
+    eraseDeletedNotes(0);
+}
+
 // do a replace in str - all keys should be replaced by the corresponding values
 function replaceByMap(str, mapObj) {
     const re = new RegExp(Object.keys(mapObj).join("|"),"g");
@@ -823,9 +836,9 @@ function getNoteIdMapping(origNote) {
 
 sqlInit.dbReady.then(() => {
     // first cleanup kickoff 5 minutes after startup
-    setTimeout(cls.wrap(eraseDeletedNotes), 5 * 60 * 1000);
+    setTimeout(cls.wrap(() => eraseDeletedNotes()), 5 * 60 * 1000);
 
-    setInterval(cls.wrap(eraseDeletedNotes), 4 * 3600 * 1000);
+    setInterval(cls.wrap(() => eraseDeletedNotes()), 4 * 3600 * 1000);
 });
 
 module.exports = {
@@ -839,5 +852,6 @@ module.exports = {
     duplicateSubtree,
     duplicateSubtreeWithoutRoot,
     getUndeletedParentBranches,
-    triggerNoteTitleChanged
+    triggerNoteTitleChanged,
+    eraseDeletedNotesNow
 };

src/services/sql.js

@@ -236,6 +236,29 @@ function transactional(func) {
     return ret;
 }
 
+function fillNoteIdList(noteIds, truncate = true) {
+    if (noteIds.length === 0) {
+        return;
+    }
+
+    if (truncate) {
+        execute("DELETE FROM param_list");
+    }
+
+    noteIds = Array.from(new Set(noteIds));
+
+    if (noteIds.length > 30000) {
+        fillNoteIdList(noteIds.slice(30000), false);
+
+        noteIds = noteIds.slice(0, 30000);
+    }
+
+    // doing it manually to avoid this showing up on the sloq query list
+    const s = stmt(`INSERT INTO param_list VALUES ` + noteIds.map(noteId => `(?)`).join(','), noteIds);
+
+    s.run(noteIds);
+}
+
 module.exports = {
     dbConnection,
     insert,
@@ -253,5 +276,6 @@ module.exports = {
     executeMany,
     executeScript,
     transactional,
-    upsert
+    upsert,
+    fillNoteIdList
 };

src/services/sql_init.js

@@ -40,6 +40,8 @@ async function initDbConnection() {
 
     require('./options_init').initStartupOptions();
 
+    sql.execute('CREATE TEMP TABLE "param_list" (`paramId` TEXT NOT NULL PRIMARY KEY)');
+
     dbReady.resolve();
 }
 

src/services/tree.js

@@ -6,6 +6,41 @@ const Branch = require('../entities/branch');
 const entityChangesService = require('./entity_changes.js');
 const protectedSessionService = require('./protected_session');
 
+function getNotesIncludingAscendants(noteIds) {
+    noteIds = Array.from(new Set(noteIds));
+
+    sql.fillNoteIdList(noteIds);
+
+    // we return also deleted notes which have been specifically asked for
+
+    const notes = sql.getRows(`
+        WITH RECURSIVE
+            treeWithAscendants AS (
+                SELECT paramId AS noteId FROM param_list
+                UNION
+                SELECT branches.parentNoteId FROM branches
+                  JOIN treeWithAscendants ON branches.noteId = treeWithAscendants.noteId
+                WHERE branches.isDeleted = 0
+            )
+        SELECT 
+          noteId,
+          title,
+          isProtected,
+          type,
+          mime,
+          isDeleted
+        FROM notes
+        JOIN treeWithAscendants USING(noteId)`);
+
+    protectedSessionService.decryptNotes(notes);
+
+    notes.forEach(note => {
+        note.isProtected = !!note.isProtected
+    });
+
+    return notes;
+}
+
 function getNotes(noteIds) {
     // we return also deleted notes which have been specifically asked for
     const notes = sql.getManyRows(`
@@ -190,6 +225,7 @@ function setNoteToParent(noteId, prefix, parentNoteId) {
 
 module.exports = {
     getNotes,
+    getNotesIncludingAscendants,
     validateParentChild,
     sortNotesAlphabetically,
     setNoteToParent