move entity hash to entity_changes table WIP

src/entities/attribute.js

@@ -109,10 +109,8 @@ class Attribute extends Entity {
 
         super.beforeSaving();
 
-        if (this.isChanged) {
         this.utcDateModified = dateUtils.utcNowDateTime();
     }
-    }
 
     createClone(type, name, value, isInheritable) {
         return new Attribute({

src/entities/branch.js

@@ -57,10 +57,8 @@ class Branch extends Entity {
 
         super.beforeSaving();
 
-        if (this.isChanged) {
         this.utcDateModified = dateUtils.utcNowDateTime();
     }
-    }
 
     createClone(parentNoteId, notePosition) {
         return new Branch({

src/entities/entity.js

@@ -22,11 +22,6 @@ class Entity {
 
     beforeSaving() {
         this.generateIdIfNecessary();
-
-        const origHash = this.hash;
-
-        this.hash = this.generateHash();
-        this.isChanged = origHash !== this.hash;
     }
 
     generateIdIfNecessary() {

src/entities/note.js

@@ -143,8 +143,7 @@ class Note extends Entity {
             noteId: this.noteId,
             content: content,
             dateModified: dateUtils.localNowDateTime(),
-            utcDateModified: dateUtils.utcNowDateTime(),
+            utcDateModified: dateUtils.utcNowDateTime()
-            hash: utils.hash(this.noteId + "|" + content.toString())
         };
 
         if (this.isProtected) {
@@ -158,7 +157,9 @@ class Note extends Entity {
 
         sql.upsert("note_contents", "noteId", pojo);
 
-        entityChangesService.addNoteContentEntityChange(this.noteId);
+        const hash = utils.hash(this.noteId + "|" + content.toString());
+
+        entityChangesService.addEntityChange('note_contents', this.noteId, hash);
     }
 
     setJsonContent(content) {
@@ -904,11 +905,9 @@ class Note extends Entity {
 
         super.beforeSaving();
 
-        if (this.isChanged) {
         this.dateModified = dateUtils.localNowDateTime();
         this.utcDateModified = dateUtils.utcNowDateTime();
     }
-    }
 
     // cannot be static!
     updatePojo(pojo) {

src/entities/note_revision.js

@@ -106,8 +106,7 @@ class NoteRevision extends Entity {
         const pojo = {
             noteRevisionId: this.noteRevisionId,
             content: content,
-            utcDateModified: dateUtils.utcNowDateTime(),
+            utcDateModified: dateUtils.utcNowDateTime()
-            hash: utils.hash(this.noteRevisionId + "|" + content)
         };
 
         if (this.isProtected) {
@@ -121,16 +120,16 @@ class NoteRevision extends Entity {
 
         sql.upsert("note_revision_contents", "noteRevisionId", pojo);
 
-        entityChangesService.addNoteRevisionContentEntityChange(this.noteRevisionId);
+        const hash = utils.hash(this.noteRevisionId + "|" + content);
+
+        entityChangesService.addEntityChange('note_revision_contents', this.noteRevisionId, hash);
     }
 
     beforeSaving() {
         super.beforeSaving();
 
-        if (this.isChanged) {
         this.utcDateModified = dateUtils.utcNowDateTime();
     }
-    }
 
     // cannot be static!
     updatePojo(pojo) {

src/entities/option.js

@@ -32,10 +32,8 @@ class Option extends Entity {
 
         super.beforeSaving();
 
-        if (this.isChanged) {
         this.utcDateModified = dateUtils.utcNowDateTime();
     }
-    }
 }
 
 module.exports = Option;

src/routes/api/sync.js

@@ -86,32 +86,32 @@ function forceNoteSync(req) {
     const now = dateUtils.utcNowDateTime();
 
     sql.execute(`UPDATE notes SET utcDateModified = ? WHERE noteId = ?`, [now, noteId]);
-    entityChangesService.addNoteEntityChange(noteId);
+    entityChangesService.moveEntityChangeToTop('notes', noteId);
 
     sql.execute(`UPDATE note_contents SET utcDateModified = ? WHERE noteId = ?`, [now, noteId]);
-    entityChangesService.addNoteContentEntityChange(noteId);
+    entityChangesService.moveEntityChangeToTop('note_contents', noteId);
 
     for (const branchId of sql.getColumn("SELECT branchId FROM branches WHERE noteId = ?", [noteId])) {
         sql.execute(`UPDATE branches SET utcDateModified = ? WHERE branchId = ?`, [now, branchId]);
 
-        entityChangesService.addBranchEntityChange(branchId);
+        entityChangesService.moveEntityChangeToTop('branches', branchId);
     }
 
     for (const attributeId of sql.getColumn("SELECT attributeId FROM attributes WHERE noteId = ?", [noteId])) {
         sql.execute(`UPDATE attributes SET utcDateModified = ? WHERE attributeId = ?`, [now, attributeId]);
 
-        entityChangesService.addAttributeEntityChange(attributeId);
+        entityChangesService.moveEntityChangeToTop('attributes', attributeId);
     }
 
     for (const noteRevisionId of sql.getColumn("SELECT noteRevisionId FROM note_revisions WHERE noteId = ?", [noteId])) {
         sql.execute(`UPDATE note_revisions SET utcDateModified = ? WHERE noteRevisionId = ?`, [now, noteRevisionId]);
-        entityChangesService.addNoteRevisionEntityChange(noteRevisionId);
+        entityChangesService.moveEntityChangeToTop('note_revisions', noteRevisionId);
 
         sql.execute(`UPDATE note_revision_contents SET utcDateModified = ? WHERE noteRevisionId = ?`, [now, noteRevisionId]);
-        entityChangesService.addNoteRevisionContentEntityChange(noteRevisionId);
+        entityChangesService.moveEntityChangeToTop('note_revision_contents', noteRevisionId);
     }
 
-    entityChangesService.addRecentNoteEntityChange(noteId);
+    entityChangesService.moveEntityChangeToTop('recent_changes', noteId);
 
     log.info("Forcing note sync for " + noteId);
 

src/services/app_info.js

@@ -4,8 +4,8 @@ const build = require('./build');
 const packageJson = require('../../package');
 const {TRILIUM_DATA_DIR} = require('./data_dir');
 
-const APP_DB_VERSION = 172;
+const APP_DB_VERSION = 173;
-const SYNC_VERSION = 16;
+const SYNC_VERSION = 17;
 const CLIPPER_PROTOCOL_VERSION = "1.0";
 
 module.exports = {

src/services/consistency_checks.js

@@ -297,11 +297,10 @@ class ConsistencyChecks {
                         sql.upsert("note_contents", "noteId", {
                             noteId: noteId,
                             content: null,
-                            hash: "consistency_checks",
                             utcDateModified: dateUtils.utcNowDateTime()
                         });
 
-                        entityChangesService.addNoteContentEntityChange(noteId);
+                        entityChangesService.addEntityChange('note_contents', noteId, "consistency_checks");
                     }
                     else {
                         // empty string might be wrong choice for some note types but it's a best guess
@@ -566,7 +565,9 @@ class ConsistencyChecks {
           entity_changes.id IS NULL AND ` + (entityName === 'options' ? 'options.isSynced = 1' : '1'),
             ({entityId}) => {
                 if (this.autoFix) {
-                    entityChangesService.addEntityChange(entityName, entityId);
+                    const entity = repository.getEntity(`SELECT * FROM ${entityName} WHERE ${key} = ?`, [entityId]);
+
+                    entityChangesService.addEntityChange(entityName, entityId, entity.generateHash());
 
                     logFix(`Created missing entity change for entityName=${entityName}, entityId=${entityId}`);
                 } else {

src/services/content_hash.js

@@ -34,23 +34,33 @@ function getSectorHashes(tableName, primaryKeyName, whereBranch) {
 function getEntityHashes() {
     const startTime = new Date();
 
-    const hashes = {
+    const hashRows = sql.getRows(`SELECT entityName, entityId, hash FROM entity_changes`);
-        notes: getSectorHashes(Note.entityName, Note.primaryKeyName),
+
-        note_contents: getSectorHashes("note_contents", "noteId"),
+    // sorting is faster in memory
-        branches: getSectorHashes(Branch.entityName, Branch.primaryKeyName),
+    // sorting by entityId is enough, hashes will be segmented by entityName later on anyway
-        note_revisions: getSectorHashes(NoteRevision.entityName, NoteRevision.primaryKeyName),
+    hashRows.sort((a, b) => a.entityId < b.entityId ? -1 : 1);
-        note_revision_contents: getSectorHashes("note_revision_contents", "noteRevisionId"),
+
-        recent_notes: getSectorHashes(RecentNote.entityName, RecentNote.primaryKeyName),
+    const hashMap = {};
-        options: getSectorHashes(Option.entityName, Option.primaryKeyName, "isSynced = 1"),
+
-        attributes: getSectorHashes(Attribute.entityName, Attribute.primaryKeyName),
+    for (const {entityName, entityId, hash} of hashRows) {
-        api_tokens: getSectorHashes(ApiToken.entityName, ApiToken.primaryKeyName),
+        const entityHashMap = hashMap[entityName] = hashMap[entityName] || {};
-    };
+
+        const sector = entityId[0];
+
+        entityHashMap[sector] = (entityHashMap[sector] || "") + hash
+    }
+
+    for (const entityHashMap of Object.values(hashMap)) {
+        for (const key in entityHashMap) {
+            entityHashMap[key] = utils.hash(entityHashMap[key]);
+        }
+    }
 
     const elapsedTimeMs = Date.now() - startTime.getTime();
 
     log.info(`Content hash computation took ${elapsedTimeMs}ms`);
 
-    return hashes;
+    return hashMap;
 }
 
 function checkContentHashes(otherHashes) {

src/services/entity_changes.js

@@ -1,4 +1,5 @@
 const sql = require('./sql');
+const repository = require('repository');
 const sourceIdService = require('./source_id');
 const dateUtils = require('./date_utils');
 const log = require('./log');
@@ -6,10 +7,11 @@ const cls = require('./cls');
 
 let maxEntityChangeId = 0;
 
-function insertEntityChange(entityName, entityId, sourceId = null, isSynced = true) {
+function insertEntityChange(entityName, entityId, hash, sourceId = null, isSynced = true) {
     const entityChange = {
         entityName: entityName,
         entityId: entityId,
+        hash: hash,
         utcChangedDate: dateUtils.utcNowDateTime(),
         sourceId: sourceId || cls.getSourceId() || sourceIdService.getCurrentSourceId(),
         isSynced: isSynced ? 1 : 0
@@ -22,12 +24,18 @@ function insertEntityChange(entityName, entityId, sourceId = null, isSynced = tr
     return entityChange;
 }
 
-function addEntityChange(entityName, entityId, sourceId, isSynced) {
+function addEntityChange(entityName, entityId, hash, sourceId, isSynced) {
-    const sync = insertEntityChange(entityName, entityId, sourceId, isSynced);
+    const sync = insertEntityChange(entityName, entityId, hash, sourceId, isSynced);
 
     cls.addSyncRow(sync);
 }
 
+function moveEntityChangeToTop(entityName, entityId) {
+    const entityChange = sql.getRow(`SELECT * FROM entity_changes WHERE entityName = ? AND entityId = ?`, [entityName, entityId]);
+
+    addEntityChange(entityName, entityId, entityChange.hash, null, entityChange.isSynced);
+}
+
 function addEntityChangesForSector(entityName, entityPrimaryKey, sector) {
     const startTime = Date.now();
 
@@ -35,15 +43,14 @@ function addEntityChangesForSector(entityName, entityPrimaryKey, sector) {
         const entityIds = sql.getColumn(`SELECT ${entityPrimaryKey} FROM ${entityName} WHERE SUBSTR(${entityPrimaryKey}, 1, 1) = ?`, [sector]);
 
         for (const entityId of entityIds) {
-            if (entityName === 'options') {
+            // retrieving entity one by one to avoid memory issues with note_contents
-                const isSynced = sql.getValue(`SELECT isSynced FROM options WHERE name = ?`, [entityId]);
+            const entity = repository.getEntity(`SELECT * FROM ${entityName} WHERE ${entityPrimaryKey} = ?`, [entityId]);
 
-                if (!isSynced) {
+            if (entityName === 'options' && !entity.isSynced) {
-                    continue;
+                continue
-                }
             }
 
-            insertEntityChange(entityName, entityId, 'content-check', true);
+            insertEntityChange(entityName, entityId, entity.generateHash(), 'content-check', true);
         }
     });
 
@@ -112,16 +119,8 @@ function fillAllEntityChanges() {
 }
 
 module.exports = {
-    addNoteEntityChange: (noteId, sourceId) => addEntityChange("notes", noteId, sourceId),
-    addNoteContentEntityChange: (noteId, sourceId) => addEntityChange("note_contents", noteId, sourceId),
-    addBranchEntityChange: (branchId, sourceId) => addEntityChange("branches", branchId, sourceId),
     addNoteReorderingEntityChange: (parentNoteId, sourceId) => addEntityChange("note_reordering", parentNoteId, sourceId),
-    addNoteRevisionEntityChange: (noteRevisionId, sourceId) => addEntityChange("note_revisions", noteRevisionId, sourceId),
+    moveEntityChangeToTop,
-    addNoteRevisionContentEntityChange: (noteRevisionId, sourceId) => addEntityChange("note_revision_contents", noteRevisionId, sourceId),
-    addOptionEntityChange: (name, sourceId, isSynced) => addEntityChange("options", name, sourceId, isSynced),
-    addRecentNoteEntityChange: (noteId, sourceId) => addEntityChange("recent_notes", noteId, sourceId),
-    addAttributeEntityChange: (attributeId, sourceId) => addEntityChange("attributes", attributeId, sourceId),
-    addApiTokenEntityChange: (apiTokenId, sourceId) => addEntityChange("api_tokens", apiTokenId, sourceId),
     addEntityChange,
     fillAllEntityChanges,
     addEntityChangesForSector,

src/services/repository.js

@@ -101,9 +101,6 @@ function updateEntity(entity) {
         entity.updatePojo(clone);
     }
 
-    // indicates whether entity actually changed
-    delete clone.isChanged;
-
     for (const key in clone) {
         // !isBuffer is for images and attachments
         if (clone[key] !== null && typeof clone[key] === 'object' && !Buffer.isBuffer(clone[key])) {
@@ -116,7 +113,6 @@ function updateEntity(entity) {
 
         const primaryKey = entity[primaryKeyName];
 
-        if (entity.isChanged) {
         const isSynced = entityName !== 'options' || entity.isSynced;
 
         entityChangesService.addEntityChange(entityName, primaryKey, null, isSynced);
@@ -133,7 +129,6 @@ function updateEntity(entity) {
 
             eventService.emit(entity.isDeleted ? eventService.ENTITY_DELETED : eventService.ENTITY_CHANGED, eventPayload);
         }
-        }
     });
 }
 

src/services/sync_update.js

@@ -9,38 +9,38 @@ function updateEntity(entityChange, entity, sourceId) {
         return false;
     }
 
-    const {entityName} = entityChange;
+    const {entityName, hash} = entityChange;
     let updated;
 
     if (entityName === 'notes') {
-        updated = updateNote(entity, sourceId);
+        updated = updateNote(entity, hash, sourceId);
     }
     else if (entityName === 'note_contents') {
-        updated = updateNoteContent(entity, sourceId);
+        updated = updateNoteContent(entity, hash, sourceId);
     }
     else if (entityName === 'branches') {
-        updated = updateBranch(entity, sourceId);
+        updated = updateBranch(entity, hash, sourceId);
     }
     else if (entityName === 'note_revisions') {
-        updated = updateNoteRevision(entity, sourceId);
+        updated = updateNoteRevision(entity, hash, sourceId);
     }
     else if (entityName === 'note_revision_contents') {
-        updated = updateNoteRevisionContent(entity, sourceId);
+        updated = updateNoteRevisionContent(entity, hash, sourceId);
     }
     else if (entityName === 'note_reordering') {
         updated = updateNoteReordering(entityChange.entityId, entity, sourceId);
     }
     else if (entityName === 'options') {
-        updated = updateOptions(entity, sourceId);
+        updated = updateOptions(entity, hash, sourceId);
     }
     else if (entityName === 'recent_notes') {
-        updated = updateRecentNotes(entity, sourceId);
+        updated = updateRecentNotes(entity, hash, sourceId);
     }
     else if (entityName === 'attributes') {
-        updated = updateAttribute(entity, sourceId);
+        updated = updateAttribute(entity, hash, sourceId);
     }
     else if (entityName === 'api_tokens') {
-        updated = updateApiToken(entity, sourceId);
+        updated = updateApiToken(entity, hash, sourceId);
     }
     else {
         throw new Error(`Unrecognized entity type ${entityName}`);
@@ -79,14 +79,14 @@ function shouldWeUpdateEntity(localEntity, remoteEntity) {
     return false;
 }
 
-function updateNote(remoteEntity, sourceId) {
+function updateNote(remoteEntity, hash, sourceId) {
     const localEntity = sql.getRow("SELECT * FROM notes WHERE noteId = ?", [remoteEntity.noteId]);
 
     if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
         sql.transactional(() => {
             sql.replace("notes", remoteEntity);
 
-            entityChangesService.addNoteEntityChange(remoteEntity.noteId, sourceId);
+            entityChangesService.addEntityChange('notes', remoteEntity.noteId, hash, sourceId);
         });
 
         return true;
@@ -95,7 +95,7 @@ function updateNote(remoteEntity, sourceId) {
     return false;
 }
 
-function updateNoteContent(remoteEntity, sourceId) {
+function updateNoteContent(remoteEntity, hash, sourceId) {
     const localEntity = sql.getRow("SELECT * FROM note_contents WHERE noteId = ?", [remoteEntity.noteId]);
 
     if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
@@ -111,7 +111,7 @@ function updateNoteContent(remoteEntity, sourceId) {
         sql.transactional(() => {
             sql.replace("note_contents", remoteEntity);
 
-            entityChangesService.addNoteContentEntityChange(remoteEntity.noteId, sourceId);
+            entityChangesService.addEntityChange("note_contents", remoteEntity.noteId, hash, sourceId);
         });
 
         return true;
@@ -120,7 +120,7 @@ function updateNoteContent(remoteEntity, sourceId) {
     return false;
 }
 
-function updateBranch(remoteEntity, sourceId) {
+function updateBranch(remoteEntity, hash, sourceId) {
     const localEntity = sql.getRowOrNull("SELECT * FROM branches WHERE branchId = ?", [remoteEntity.branchId]);
 
     if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
@@ -133,7 +133,7 @@ function updateBranch(remoteEntity, sourceId) {
 
             sql.replace('branches', remoteEntity);
 
-            entityChangesService.addBranchEntityChange(remoteEntity.branchId, sourceId);
+            entityChangesService.addEntityChange('branches', remoteEntity.branchId, hash, sourceId);
         });
 
         return true;
@@ -142,21 +142,21 @@ function updateBranch(remoteEntity, sourceId) {
     return false;
 }
 
-function updateNoteRevision(remoteEntity, sourceId) {
+function updateNoteRevision(remoteEntity, hash, sourceId) {
     const localEntity = sql.getRowOrNull("SELECT * FROM note_revisions WHERE noteRevisionId = ?", [remoteEntity.noteRevisionId]);
 
     sql.transactional(() => {
         if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
             sql.replace('note_revisions', remoteEntity);
 
-            entityChangesService.addNoteRevisionEntityChange(remoteEntity.noteRevisionId, sourceId);
+            entityChangesService.addEntityChange('note_revisions', remoteEntity.noteRevisionId, hash, sourceId);
 
             log.info("Update/sync note revision " + remoteEntity.noteRevisionId);
         }
     });
 }
 
-function updateNoteRevisionContent(remoteEntity, sourceId) {
+function updateNoteRevisionContent(remoteEntity, hash, sourceId) {
     const localEntity = sql.getRowOrNull("SELECT * FROM note_revision_contents WHERE noteRevisionId = ?", [remoteEntity.noteRevisionId]);
 
     if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
@@ -165,7 +165,7 @@ function updateNoteRevisionContent(remoteEntity, sourceId) {
 
             sql.replace('note_revision_contents', remoteEntity);
 
-            entityChangesService.addNoteRevisionContentEntityChange(remoteEntity.noteRevisionId, sourceId);
+            entityChangesService.addEntityChange('note_revision_contents', remoteEntity.noteRevisionId, hash, sourceId);
         });
 
         return true;
@@ -180,13 +180,13 @@ function updateNoteReordering(entityId, remote, sourceId) {
             sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [remote[key], key]);
         }
 
-        entityChangesService.addNoteReorderingEntityChange(entityId, sourceId);
+        entityChangesService.addEntityChange('note_reordering', entityId, 'none', sourceId);
     });
 
     return true;
 }
 
-function updateOptions(remoteEntity, sourceId) {
+function updateOptions(remoteEntity, hash, sourceId) {
     const localEntity = sql.getRowOrNull("SELECT * FROM options WHERE name = ?", [remoteEntity.name]);
 
     if (localEntity && !localEntity.isSynced) {
@@ -197,7 +197,7 @@ function updateOptions(remoteEntity, sourceId) {
         sql.transactional(() => {
             sql.replace('options', remoteEntity);
 
-            entityChangesService.addOptionEntityChange(remoteEntity.name, sourceId, true);
+            entityChangesService.addEntityChange('options', remoteEntity.name, hash, sourceId, true);
         });
 
         return true;
@@ -206,14 +206,14 @@ function updateOptions(remoteEntity, sourceId) {
     return false;
 }
 
-function updateRecentNotes(remoteEntity, sourceId) {
+function updateRecentNotes(remoteEntity, hash, sourceId) {
     const localEntity = sql.getRowOrNull("SELECT * FROM recent_notes WHERE noteId = ?", [remoteEntity.noteId]);
 
     if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
         sql.transactional(() => {
             sql.replace('recent_notes', remoteEntity);
 
-            entityChangesService.addRecentNoteEntityChange(remoteEntity.noteId, sourceId);
+            entityChangesService.addEntityChange('recent_notes', remoteEntity.noteId, hash, sourceId);
         });
 
         return true;
@@ -222,14 +222,14 @@ function updateRecentNotes(remoteEntity, sourceId) {
     return false;
 }
 
-function updateAttribute(remoteEntity, sourceId) {
+function updateAttribute(remoteEntity, hash, sourceId) {
     const localEntity = sql.getRow("SELECT * FROM attributes WHERE attributeId = ?", [remoteEntity.attributeId]);
 
     if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
         sql.transactional(() => {
             sql.replace("attributes", remoteEntity);
 
-            entityChangesService.addAttributeEntityChange(remoteEntity.attributeId, sourceId);
+            entityChangesService.addEntityChange('attributes', remoteEntity.attributeId, hash, sourceId);
         });
 
         return true;
@@ -238,14 +238,14 @@ function updateAttribute(remoteEntity, sourceId) {
     return false;
 }
 
-function updateApiToken(entity, sourceId) {
+function updateApiToken(entity, hash, sourceId) {
     const apiTokenId = sql.getRow("SELECT * FROM api_tokens WHERE apiTokenId = ?", [entity.apiTokenId]);
 
     if (!apiTokenId) {
         sql.transactional(() => {
             sql.replace("api_tokens", entity);
 
-            entityChangesService.addApiTokenEntityChange(entity.apiTokenId, sourceId);
+            entityChangesService.addEntityChange('api_tokens',entity.apiTokenId, hash, sourceId);
         });
 
         return true;