note cache refactoring

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "trilium",
-  "version": "0.42.1",
+  "version": "0.42.2",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -2218,9 +2218,9 @@
       }
     },
     "cli-spinners": {
-      "version": "2.2.0",
+      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.3.0.tgz",
-      "integrity": "sha512-tgU3fKwzYjiLEQgPMD9Jt+JjHVL9kW93FiIMX/l7rivvOD4/LL0Mf7gda3+4U2KJBloybwgj5KEoQgGRioMiKQ==",
+      "integrity": "sha512-Xs2Hf2nzrvJMFKimOR7YR0QwZ8fc0u98kdtwN1eNAZzNQgH3vK2pXzff6GJtKh7S5hoJ87ECiAiZFS2fb5Ii2w==",
       "dev": true
     },
     "cli-table3": {
@@ -3802,9 +3802,9 @@
       }
     },
     "electron-rebuild": {
-      "version": "1.10.1",
+      "version": "1.11.0",
-      "resolved": "https://registry.npmjs.org/electron-rebuild/-/electron-rebuild-1.10.1.tgz",
+      "resolved": "https://registry.npmjs.org/electron-rebuild/-/electron-rebuild-1.11.0.tgz",
-      "integrity": "sha512-KSqp0Xiu7CCvKL2aEdPp/vNe2Rr11vaO8eM/wq9gQJTY02UjtAJ3l7WLV7Mf8oR+UJReJO8SWOWs/FozqK8ggA==",
+      "integrity": "sha512-cn6AqZBQBVtaEyj5jZW1/LOezZZ22PA1HvhEP7asvYPJ8PDF4i4UFt9be4i9T7xJKiSiomXvY5Fd+dSq3FXZxA==",
       "dev": true,
       "requires": {
         "colors": "^1.3.3",
@@ -3877,9 +3877,9 @@
           }
         },
         "yargs": {
-          "version": "14.2.2",
+          "version": "14.2.3",
-          "resolved": "https://registry.npmjs.org/yargs/-/yargs-14.2.2.tgz",
+          "resolved": "https://registry.npmjs.org/yargs/-/yargs-14.2.3.tgz",
-          "integrity": "sha512-/4ld+4VV5RnrynMhPZJ/ZpOCGSCeghMykZ3BhdFBDa9Wy/RH6uEGNWDJog+aUlq+9OM1CFTgtYRW5Is1Po9NOA==",
+          "integrity": "sha512-ZbotRWhF+lkjijC/VhmOT9wSgyBQ7+zr13+YLkhfsSiTriYsMzkTUFP18pFhWwBeMa5gUc1MzbhrO6/VB7c9Xg==",
           "dev": true,
           "requires": {
             "cliui": "^5.0.0",
@@ -3892,13 +3892,13 @@
             "string-width": "^3.0.0",
             "which-module": "^2.0.0",
             "y18n": "^4.0.0",
-            "yargs-parser": "^15.0.0"
+            "yargs-parser": "^15.0.1"
           }
         },
         "yargs-parser": {
-          "version": "15.0.0",
+          "version": "15.0.1",
-          "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-15.0.0.tgz",
+          "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-15.0.1.tgz",
-          "integrity": "sha512-xLTUnCMc4JhxrPEPUYD5IBR1mWCK/aT6+RJ/K29JY2y1vD+FhtgKK0AXRWvI262q3QSffAQuTouFIKUuHX89wQ==",
+          "integrity": "sha512-0OAMV2mAZQrs3FkNpDQcBk1x5HXb8X4twADss4S0Iuk+2dGnLOE/fRHrsYm542GduMveyA77OF4wrNJuanRCWw==",
           "dev": true,
           "requires": {
             "camelcase": "^5.0.0",
@@ -9929,9 +9929,9 @@
       }
     },
     "rxjs": {
-      "version": "6.5.4",
+      "version": "6.5.5",
-      "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.4.tgz",
+      "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.5.tgz",
-      "integrity": "sha512-naMQXcgEo3csAEGvw/NydRA0fuS2nDZJiw1YUWFKU7aPPAPGZEsD4Iimit96qwCieH6y614MCLYwdkrWx7z/7Q==",
+      "integrity": "sha512-WfQI+1gohdf0Dai/Bbmk5L5ItH5tYqm3ki2c5GdWhKjalzjg93N3avFjVStyZZz+A2Em+ZxKH5bNghw9UeylGQ==",
       "dev": true,
       "requires": {
         "tslib": "^1.9.0"

package.json

@@ -81,7 +81,7 @@
     "electron": "9.0.0-beta.24",
     "electron-builder": "22.6.0",
     "electron-packager": "14.2.1",
-    "electron-rebuild": "1.10.1",
+    "electron-rebuild": "1.11.0",
     "jsdoc": "3.6.4",
     "lorem-ipsum": "2.0.3",
     "webpack": "5.0.0-beta.16",

src/entities/attribute.js

@@ -8,13 +8,13 @@ const sql = require('../services/sql');
 /**
  * Attribute is key value pair owned by a note.
  *
- * @property {string} attributeId
+ * @property {string} attributeId - immutable
- * @property {string} noteId
+ * @property {string} noteId - immutable
- * @property {string} type
+ * @property {string} type - immutable
- * @property {string} name
+ * @property {string} name - immutable
  * @property {string} value
  * @property {int} position
- * @property {boolean} isInheritable
+ * @property {boolean} isInheritable - immutable
  * @property {boolean} isDeleted
  * @property {string|null} deleteId - ID identifying delete transaction
  * @property {string} utcDateCreated
@@ -108,14 +108,14 @@ class Attribute extends Entity {
         delete pojo.__note;
     }
 
-    createClone(type, name, value) {
+    createClone(type, name, value, isInheritable) {
         return new Attribute({
             noteId: this.noteId,
             type: type,
             name: name,
             value: value,
             position: this.position,
-            isInheritable: this.isInheritable,
+            isInheritable: isInheritable,
             isDeleted: false,
             utcDateCreated: this.utcDateCreated,
             utcDateModified: this.utcDateModified

src/entities/branch.js

@@ -9,9 +9,9 @@ const sql = require('../services/sql');
  * Branch represents note's placement in the tree - it's essentially pair of noteId and parentNoteId.
  * Each note can have multiple (at least one) branches, meaning it can be placed into multiple places in the tree.
  *
- * @property {string} branchId - primary key
+ * @property {string} branchId - primary key, immutable
- * @property {string} noteId
+ * @property {string} noteId - immutable
- * @property {string} parentNoteId
+ * @property {string} parentNoteId - immutable
  * @property {int} notePosition
  * @property {string} prefix
  * @property {boolean} isExpanded
@@ -77,4 +77,4 @@ class Branch extends Entity {
     }
 }
 
-module.exports = Branch;
+module.exports = Branch;

src/routes/api/attributes.js

@@ -98,10 +98,11 @@ async function updateNoteAttributes(req) {
 
             if (attribute.type !== attributeEntity.type
                 || attribute.name !== attributeEntity.name
-                || (attribute.type === 'relation' && attribute.value !== attributeEntity.value)) {
+                || (attribute.type === 'relation' && attribute.value !== attributeEntity.value)
+                || attribute.isInheritable !== attributeEntity.isInheritable) {
 
                 if (attribute.type !== 'relation' || !!attribute.value.trim()) {
-                    const newAttribute = attributeEntity.createClone(attribute.type, attribute.name, attribute.value);
+                    const newAttribute = attributeEntity.createClone(attribute.type, attribute.name, attribute.value, attribute.isInheritable);
                     await newAttribute.save();
                 }
 

src/routes/api/similar_notes.js

@@ -12,7 +12,7 @@ async function getSimilarNotes(req) {
         return [404, `Note ${noteId} not found.`];
     }
 
-    const results = await noteCacheService.findSimilarNotes(note.title);
+    const results = await noteCacheService.findSimilarNotes(noteId);
 
     return results
         .filter(note => note.noteId !== noteId);
@@ -20,4 +20,4 @@ async function getSimilarNotes(req) {
 
 module.exports = {
     getSimilarNotes
-};
+};

src/services/note_cache.js

@@ -35,46 +35,75 @@ class Note {
         this.children = [];
         /** @param {Attribute[]} */
         this.ownedAttributes = [];
+
+        /** @param {Attribute[]|null} */
+        this.attributeCache = null;
+        /** @param {Attribute[]|null} */
+        this.templateAttributeCache = null;
+        /** @param {Attribute[]|null} */
+        this.inheritableAttributeCache = null;
+
+        /** @param {string|null} */
+        this.fulltextCache = null;
     }
 
     /** @return {Attribute[]} */
     get attributes() {
-        if (!(this.noteId in noteAttributeCache)) {
+        if (!this.attributeCache) {
-            const attrArrs = [
+            const parentAttributes = this.ownedAttributes.slice();
-                this.ownedAttributes
-            ];
-
-            for (const templateAttr of this.ownedAttributes.filter(oa => oa.type === 'relation' && oa.name === 'template')) {
-                const templateNote = notes[templateAttr.value];
-
-                if (templateNote) {
-                    attrArrs.push(templateNote.attributes);
-                }
-            }
 
             if (this.noteId !== 'root') {
                 for (const parentNote of this.parents) {
-                    attrArrs.push(parentNote.inheritableAttributes);
+                    parentAttributes.push(...parentNote.inheritableAttributes);
                 }
             }
 
-            noteAttributeCache[this.noteId] = attrArrs.flat();
+            const templateAttributes = [];
+
+            for (const ownedAttr of parentAttributes) { // parentAttributes so we process also inherited templates
+                if (ownedAttr.type === 'relation' && ownedAttr.name === 'template') {
+                    const templateNote = notes[ownedAttr.value];
+
+                    if (templateNote) {
+                        templateAttributes.push(...templateNote.attributes);
+                    }
+                }
+            }
+
+            this.attributeCache = parentAttributes.concat(templateAttributes);
+            this.inheritableAttributeCache = [];
+            this.templateAttributeCache = [];
+
+            for (const attr of this.attributeCache) {
+                if (attr.isInheritable) {
+                    this.inheritableAttributeCache.push(attr);
+                }
+
+                if (attr.type === 'relation' && attr.name === 'template') {
+                    this.templateAttributeCache.push(attr);
+                }
+            }
         }
 
-        return noteAttributeCache[this.noteId];
+        return this.attributeCache;
-    }
-
-    addSubTreeNoteIdsTo(noteIdSet) {
-        noteIdSet.add(this.noteId);
-
-        for (const child of this.children) {
-            child.addSubTreeNoteIdsTo(noteIdSet);
-        }
     }
 
     /** @return {Attribute[]} */
     get inheritableAttributes() {
-        return this.attributes.filter(attr => attr.isInheritable);
+        if (!this.inheritableAttributeCache) {
+            this.attributes; // will refresh also this.inheritableAttributeCache
+        }
+
+        return this.inheritableAttributeCache;
+    }
+
+    /** @return {Attribute[]} */
+    get templateAttributes() {
+        if (!this.templateAttributeCache) {
+            this.attributes; // will refresh also this.templateAttributeCache
+        }
+
+        return this.templateAttributeCache;
     }
 
     hasAttribute(type, name) {
@@ -94,6 +123,63 @@ class Note {
     resortParents() {
         this.parents.sort((a, b) => a.hasInheritableOwnedArchivedLabel ? 1 : -1);
     }
+
+    get fulltext() {
+        if (!this.fulltextCache) {
+            this.fulltextCache = this.title.toLowerCase();
+
+            for (const attr of this.attributes) {
+                // it's best to use space as separator since spaces are filtered from the search string by the tokenization into words
+                this.fulltextCache += ' ' + attr.name.toLowerCase();
+
+                if (attr.value) {
+                    this.fulltextCache += ' ' + attr.value.toLowerCase();
+                }
+            }
+        }
+
+        return this.fulltextCache;
+    }
+
+    invalidateThisCache() {
+        this.fulltextCache = null;
+
+        this.attributeCache = null;
+        this.templateAttributeCache = null;
+        this.inheritableAttributeCache = null;
+    }
+
+    invalidateSubtreeCaches() {
+        this.invalidateThisCache();
+
+        for (const childNote of this.children) {
+            childNote.invalidateSubtreeCaches();
+        }
+
+        for (const templateAttr of this.templateAttributes) {
+            const targetNote = templateAttr.targetNote;
+
+            if (targetNote) {
+                targetNote.invalidateSubtreeCaches();
+            }
+        }
+    }
+
+    invalidateSubtreeFulltext() {
+        this.fulltextCache = null;
+
+        for (const childNote of this.children) {
+            childNote.invalidateSubtreeFulltext();
+        }
+
+        for (const templateAttr of this.templateAttributes) {
+            const targetNote = templateAttr.targetNote;
+
+            if (targetNote) {
+                targetNote.invalidateSubtreeFulltext();
+            }
+        }
+    }
 }
 
 class Branch {
@@ -137,47 +223,16 @@ class Attribute {
         /** @param {boolean} */
         this.isInheritable = !!row.isInheritable;
     }
-}
 
-/** @type {Object.<String, String>} */
+    get isAffectingSubtree() {
-let fulltext = {};
+        return this.isInheritable
-
+            || (this.type === 'relation' && this.name === 'template');
-/** @type {Object.<String, AttributeMeta>} */
-let attributeMetas = {};
-
-class AttributeMeta {
-    constructor(attribute) {
-        this.type = attribute.type;
-        this.name = attribute.name;
-        this.isInheritable = attribute.isInheritable;
-        this.attributeIds = new Set(attribute.attributeId);
     }
 
-    addAttribute(attribute) {
+    get targetNote() {
-        this.attributeIds.add(attribute.attributeId);
+        if (this.type === 'relation') {
-        this.isInheritable = this.isInheritable || attribute.isInheritable;
+            return notes[this.value];
-    }
-
-    updateAttribute(attribute) {
-        if (attribute.isDeleted) {
-            this.attributeIds.delete(attribute.attributeId);
         }
-        else {
-            this.attributeIds.add(attribute.attributeId);
-        }
-
-        this.isInheritable = !!this.attributeIds.find(attributeId => attributes[attributeId].isInheritable);
-    }
-}
-
-function addToAttributeMeta(attribute) {
-    const key = `${attribute.type}-${attribute.name}`;
-
-    if (!(key in attributeMetas)) {
-        attributeMetas[key] = new AttributeMeta(attribute);
-    }
-    else {
-        attributeMetas[key].addAttribute(attribute);
     }
 }
 
@@ -186,9 +241,6 @@ let loadedPromiseResolve;
 /** Is resolved after the initial load */
 let loadedPromise = new Promise(res => loadedPromiseResolve = res);
 
-// key is 'childNoteId-parentNoteId' as a replacement for branchId which we don't use here
-let prefixes = {};
-
 async function getMappedRows(query, cb) {
     const map = {};
     const results = await sql.getRows(query, []);
@@ -202,17 +254,6 @@ async function getMappedRows(query, cb) {
     return map;
 }
 
-function updateFulltext(note) {
-    let ft = note.title.toLowerCase();
-
-    for (const attr of note.attributes) {
-        ft += '|' + attr.name.toLowerCase();
-        ft += '|' + attr.value.toLowerCase();
-    }
-
-    fulltext[note.noteId] = ft;
-}
-
 async function load() {
     notes = await getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`,
         row => new Note(row));
@@ -225,8 +266,6 @@ async function load() {
 
     for (const attr of Object.values(attributes)) {
         notes[attr.noteId].ownedAttributes.push(attr);
-
-        addToAttributeMeta(attributes);
     }
 
     for (const branch of Object.values(branches)) {
@@ -250,10 +289,6 @@ async function load() {
         await decryptProtectedNotes();
     }
 
-    for (const note of Object.values(notes)) {
-        updateFulltext(note);
-    }
-
     loaded = true;
     loadedPromiseResolve();
 }
@@ -325,38 +360,21 @@ function highlightResults(results, allTokens) {
  * Returns noteIds which have at least one matching tokens
  *
  * @param tokens
- * @return {Set<String>}
+ * @return {String[]}
  */
 function getCandidateNotes(tokens) {
-    const candidateNoteIds = new Set();
+    const candidateNotes = [];
 
-    for (const token of tokens) {
+    for (const note of Object.values(notes)) {
-        for (const noteId in fulltext) {
+        for (const token of tokens) {
-            if (!fulltext[noteId].includes(token)) {
+            if (note.fulltext.includes(token)) {
-                continue;
+                candidateNotes.push(note);
+                break;
             }
-
-            candidateNoteIds.add(noteId);
-            const note = notes[noteId];
-            const inheritableAttrs = note.ownedAttributes.filter(attr => attr.isInheritable);
-
-            searchingAttrs:
-                // for matching inheritable attributes, include the whole note subtree to the candidates
-                for (const attr of inheritableAttrs) {
-                    const lcName = attr.name.toLowerCase();
-                    const lcValue = attr.value.toLowerCase();
-
-                    for (const token of tokens) {
-                        if (lcName.includes(token) || lcValue.includes(token)) {
-                            note.addSubTreeNoteIdsTo(candidateNoteIds);
-
-                            break searchingAttrs;
-                        }
-                    }
-                }
         }
     }
-    return candidateNoteIds;
+
+    return candidateNotes;
 }
 
 async function findNotes(query) {
@@ -370,18 +388,16 @@ async function findNotes(query) {
         .split(/[ -]/)
         .filter(token => token !== '/'); // '/' is used as separator
 
-    const candidateNoteIds = getCandidateNotes(allTokens);
+    const candidateNotes = getCandidateNotes(allTokens);
 
     // now we have set of noteIds which match at least one token
 
     let results = [];
     const tokens = allTokens.slice();
 
-    for (const noteId of candidateNoteIds) {
+    for (const note of candidateNotes) {
-        const note = notes[noteId];
-
         // autocomplete should be able to find notes by their noteIds as well (only leafs)
-        if (noteId === query) {
+        if (note.noteId === query) {
             search(note, [], [], results);
             continue;
         }
@@ -415,7 +431,7 @@ async function findNotes(query) {
             if (foundTokens.length > 0) {
                 const remainingTokens = tokens.filter(token => !foundTokens.includes(token));
 
-                search(parentNote, remainingTokens, [noteId], results);
+                search(parentNote, remainingTokens, [note.noteId], results);
             }
         }
     }
@@ -678,11 +694,11 @@ function getNotePath(noteId) {
     }
 }
 
-function evaluateSimilarity(text, note, results) {
+function evaluateSimilarity(sourceNote, candidateNote, results) {
-    let coeff = stringSimilarity.compareTwoStrings(text, note.title);
+    let coeff = stringSimilarity.compareTwoStrings(sourceNote.fulltext, candidateNote.fulltext);
 
     if (coeff > 0.4) {
-        const notePath = getSomePath(note);
+        const notePath = getSomePath(candidateNote);
 
         // this takes care of note hoisting
         if (!notePath) {
@@ -693,7 +709,7 @@ function evaluateSimilarity(text, note, results) {
             coeff -= 0.2; // archived penalization
         }
 
-        results.push({coeff, notePath, noteId: note.noteId});
+        results.push({coeff, notePath, noteId: candidateNote.noteId});
     }
 }
 
@@ -707,16 +723,22 @@ function setImmediatePromise() {
     });
 }
 
-async function findSimilarNotes(title) {
+async function findSimilarNotes(noteId) {
     const results = [];
     let i = 0;
 
+    const origNote = notes[noteId];
+
+    if (!origNote) {
+        return [];
+    }
+
     for (const note of Object.values(notes)) {
         if (note.isProtected && !note.isDecrypted) {
             continue;
         }
 
-        evaluateSimilarity(title, note, results);
+        evaluateSimilarity(origNote, note, results);
 
         i++;
 
@@ -744,9 +766,12 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED
             delete notes[noteId];
         }
         else if (noteId in notes) {
+            const note = notes[noteId];
+
             // we can assume we have protected session since we managed to update
-            notes[noteId].title = entity.title;
+            note.title = entity.title;
-            notes[noteId].isDecrypted = true;
+            note.isDecrypted = true;
+            note.fulltextCache = null;
         }
         else {
             notes[noteId] = new Note(entity);
@@ -760,6 +785,10 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED
 
             if (childNote) {
                 childNote.parents = childNote.parents.filter(parent => parent.noteId !== parentNoteId);
+
+                if (childNote.parents.length > 0) {
+                    childNote.invalidateSubtreeCaches();
+                }
             }
 
             const parentNote = notes[parentNoteId];
@@ -787,30 +816,46 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED
     }
     else if (entityName === 'attributes') {
         const {attributeId, noteId} = entity;
+        const note = notes[noteId];
+        const attr = attributes[attributeId];
 
         if (entity.isDeleted) {
-            const note = notes[noteId];
+            if (note && attr) {
-
-            if (note) {
                 note.ownedAttributes = note.ownedAttributes.filter(attr => attr.attributeId !== attributeId);
+
+                if (attr.isAffectingSubtree) {
+                    note.invalidateSubtreeCaches();
+                }
             }
 
-            delete attributes[entity.attributeId];
+            delete attributes[attributeId];
         }
         else if (attributeId in attributes) {
             const attr = attributes[attributeId];
 
-            // attr name cannot change
+            // attr name and isInheritable are immutable
             attr.value = entity.value;
-            attr.isInheritable = entity.isInheritable;
+
+            if (attr.isAffectingSubtree) {
+                note.invalidateSubtreeFulltext();
+            }
+            else {
+                note.fulltextCache = null;
+            }
         }
         else {
-            attributes[attributeId] = new Attribute(entity);
+            const attr = new Attribute(entity);
-
+            attributes[attributeId] = attr;
-            const note = notes[noteId];
 
             if (note) {
-                note.ownedAttributes.push(attributes[attributeId]);
+                note.ownedAttributes.push(attr);
+
+                if (attr.isAffectingSubtree) {
+                    note.invalidateSubtreeCaches();
+                }
+                else {
+                    this.invalidateThisCache();
+                }
             }
         }
     }