Dockery/Trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-10-31 02:16:05 +01:00
Code Issues Packages Projects Releases Activity

moving isErased into entity_changes

Browse Source
This commit is contained in:
zadam
2020-12-14 13:15:32 +01:00
parent 0a896cc19c
commit 20c7c657da
5 changed files with 101 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
db/migrations/0173__move_hash_to_entity_changes.sql
View File

@@ -37,6 +37,9 @@ CREATE TABLE IF NOT EXISTS "mig_api_tokens"
INSERT INTO mig_api_tokens (apiTokenId, token, utcDateCreated, isDeleted)
SELECT apiTokenId, token, utcDateCreated, isDeleted FROM api_tokens;
DROP TABLE api_tokens;
ALTER TABLE mig_api_tokens RENAME TO api_tokens;
CREATE TABLE IF NOT EXISTS "mig_attributes"
(
attributeId TEXT not null primary key,

65
db/migrations/0174__isErased_in_entity_changes.sql Normal file
View File

@@ -0,0 +1,65 @@
CREATE TABLE IF NOT EXISTS "mig_entity_changes" (
`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
`entityName` TEXT NOT NULL,
`entityId` TEXT NOT NULL,
`hash` TEXT NOT NULL,
`sourceId` TEXT NOT NULL,
`isErased` INT NOT NULL,
`utcDateChanged` TEXT NOT NULL,
`isSynced` INTEGER NOT NULL);
INSERT INTO mig_entity_changes (entityName, entityId, hash, sourceId, isSynced, utcDateChanged, isErased)
SELECT entityName, entityId, '', sourceId, isSynced, 'now', 0 FROM entity_changes;
UPDATE mig_entity_changes SET isErased = (SELECT isErased FROM notes WHERE noteId = entityId) WHERE entityName = 'notes';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM notes WHERE noteId = entityId) WHERE entityName = 'notes';
UPDATE mig_entity_changes SET isErased = (
SELECT isErased
FROM attributes
JOIN notes USING(noteId)
WHERE attributeId = entityId
) WHERE entityName = 'attributes';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM attributes WHERE attributeId = entityId) WHERE entityName = 'attributes';
UPDATE mig_entity_changes SET isErased = (
SELECT isErased
FROM branches
JOIN notes USING(noteId)
WHERE branchId = entityId
) WHERE entityName = 'branches';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM branches WHERE branchId = entityId) WHERE entityName = 'branches';
UPDATE mig_entity_changes SET isErased = (
SELECT isErased
FROM note_revisions
WHERE noteRevisionId = entityId
) WHERE entityName = 'note_revisions';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM note_revisions WHERE noteRevisionId = entityId) WHERE entityName = 'note_revisions';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateCreated FROM api_tokens WHERE apiTokenId = entityId) WHERE entityName = 'api_tokens';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM note_contents WHERE noteId = entityId) WHERE entityName = 'note_contents';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM note_revision_contents WHERE noteRevisionId = entityId) WHERE entityName = 'note_revision_contents';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM options WHERE name = entityId) WHERE entityName = 'options';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateCreated FROM recent_notes WHERE noteId = entityId) WHERE entityName = 'options';
DROP TABLE entity_changes;
ALTER TABLE mig_entity_changes RENAME TO entity_changes;
CREATE UNIQUE INDEX `IDX_entityChanges_entityName_entityId` ON "entity_changes" (
`entityName`,
`entityId`
);
DELETE FROM attributes WHERE noteId IN (SELECT noteId FROM notes WHERE isErased = 1);
DELETE FROM branches WHERE noteId IN (SELECT noteId FROM notes WHERE isErased = 1);
DELETE FROM note_contents WHERE noteId IN (SELECT noteId FROM notes WHERE isErased = 1);
DELETE FROM note_revision_contents WHERE noteRevisionId IN (
SELECT noteRevisionId FROM note_revisions WHERE isErased = 1
);
DELETE FROM note_revisions WHERE isErased = 1;
DELETE FROM notes WHERE isErased = 1;

3
src/services/consistency_checks.js
View File

@@ -582,7 +582,8 @@ class ConsistencyChecks {
entity_changes
LEFT JOIN ${entityName} ON entityId = ${key}
WHERE
entity_changes.entityName = '${entityName}'
entity_changes.isErased = 0
AND entity_changes.entityName = '${entityName}'
AND ${key} IS NULL`,
({id, entityId}) => {
if (this.autoFix) {

10
src/services/entity_changes.js
View File

@@ -22,8 +22,8 @@ function insertEntityChange(entityName, entityId, hash, sourceId = null, isSynce
return entityChange;
}
function addEntityChange(entityName, entityId, hash, sourceId, isSynced) {
const sync = insertEntityChange(entityName, entityId, hash, sourceId, isSynced);
function addEntityChange(entityChange, sourceId, isSynced) {
const sync = insertEntityChange(entityChange.entityName, entityChange.entityId, entityChange.hash, sourceId, isSynced);
cls.addSyncRow(sync);
}
@@ -59,7 +59,9 @@ function cleanupSyncRowsForMissingEntities(entityName, entityPrimaryKey) {
sql.execute(`
DELETE
FROM entity_changes
WHERE sync.entityName = '${entityName}'
WHERE
isErased = 0
AND sync.entityName = '${entityName}'
AND sync.entityId NOT IN (SELECT ${entityPrimaryKey} FROM ${entityName})`);
}
@@ -101,7 +103,7 @@ function fillEntityChanges(entityName, entityPrimaryKey, condition = '') {
function fillAllEntityChanges() {
sql.transactional(() => {
sql.execute("DELETE FROM entity_changes");
sql.execute("DELETE FROM entity_changes WHERE isErased = 0");
fillEntityChanges("notes", "noteId");
fillEntityChanges("note_contents", "noteId");

222
src/services/sync_update.js
View File

@@ -12,38 +12,11 @@ function updateEntity(entityChange, entity, sourceId) {
const {entityName, hash} = entityChange;
let updated;
if (entityName === 'notes') {
updated = updateNote(entity, hash, sourceId);
}
else if (entityName === 'note_contents') {
updated = updateNoteContent(entity, hash, sourceId);
}
else if (entityName === 'branches') {
updated = updateBranch(entity, hash, sourceId);
}
else if (entityName === 'note_revisions') {
updated = updateNoteRevision(entity, hash, sourceId);
}
else if (entityName === 'note_revision_contents') {
updated = updateNoteRevisionContent(entity, hash, sourceId);
}
else if (entityName === 'note_reordering') {
updated = updateNoteReordering(entityChange.entityId, entity, sourceId);
}
else if (entityName === 'options') {
updated = updateOptions(entity, hash, sourceId);
}
else if (entityName === 'recent_notes') {
updated = updateRecentNotes(entity, hash, sourceId);
}
else if (entityName === 'attributes') {
updated = updateAttribute(entity, hash, sourceId);
}
else if (entityName === 'api_tokens') {
updated = updateApiToken(entity, hash, sourceId);
if (entityName === 'note_reordering') {
updated = updateNoteReordering(entityChange, entity, sourceId);
}
else {
throw new Error(`Unrecognized entity type ${entityName}`);
updated = updateNormalEntity(entityChange, entity, sourceId);
}
// currently making exception for protected notes and note revisions because here
@@ -59,59 +32,30 @@ function updateEntity(entityChange, entity, sourceId) {
return updated;
}
function shouldWeUpdateEntity(localEntity, remoteEntity) {
if (!localEntity) {
return true;
}
function updateNormalEntity(entityChange, entity, sourceId) {
const {utcDateChanged, hash} = sql.getRow(`
SELECT utcDateChanged, hash
FROM entity_changes
WHERE entityName = ? AND entityId = ?`, [entityChange.entityName, entityChange.entityId]);
const localDate = localEntity.utcDateModified || localEntity.utcDateCreated;
const remoteDate = remoteEntity.utcDateModified || remoteEntity.utcDateCreated;
if (localDate < remoteDate) {
return true;
}
// this can happen in case of sync error when hashes are different but dates are the same - we should still update
if (localEntity.hash !== remoteEntity.hash && localDate === remoteDate) {
return true;
}
return false;
}
function updateNote(remoteEntity, hash, sourceId) {
const localEntity = sql.getRow("SELECT * FROM notes WHERE noteId = ?", [remoteEntity.noteId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.transactional(() => {
sql.replace("notes", remoteEntity);
entityChangesService.addEntityChange('notes', remoteEntity.noteId, hash, sourceId);
});
return true;
}
return false;
}
function updateNoteContent(remoteEntity, hash, sourceId) {
const localEntity = sql.getRow("SELECT * FROM note_contents WHERE noteId = ?", [remoteEntity.noteId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
if (utcDateChanged < entityChange.utcDateChanged
|| hash !== entityChange.hash // sync error, we should still update
) {
if (['note_contents', 'note_revision_contents'].includes(entityChange.entityName)) {
// we always use Buffer object which is different from normal saving - there we use simple string type for "string notes"
// the problem is that in general it's not possible to whether a note_content is string note or note (syncs can arrive out of order)
remoteEntity.content = remoteEntity.content === null ? null : Buffer.from(remoteEntity.content, 'base64');
entity.content = entity.content === null ? null : Buffer.from(entity.content, 'base64');
if (remoteEntity.content && remoteEntity.content.byteLength === 0) {
if (entity.content && entity.content.byteLength === 0) {
// there seems to be a bug which causes empty buffer to be stored as NULL which is then picked up as inconsistency
remoteEntity.content = "";
entity.content = "";
}
}
sql.transactional(() => {
sql.replace("note_contents", remoteEntity);
sql.replace(entityChange.entityName, entity);
entityChangesService.addEntityChange("note_contents", remoteEntity.noteId, hash, sourceId);
entityChangesService.addEntityChange(entityChange, sourceId);
});
return true;
@@ -120,138 +64,16 @@ function updateNoteContent(remoteEntity, hash, sourceId) {
return false;
}
function updateBranch(remoteEntity, hash, sourceId) {
const localEntity = sql.getRowOrNull("SELECT * FROM branches WHERE branchId = ?", [remoteEntity.branchId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
function updateNoteReordering(entityChange, entity, sourceId) {
sql.transactional(() => {
// isExpanded is not synced unless it's a new branch instance
// otherwise in case of full new sync we'll get all branches (even root) collapsed.
if (localEntity) {
delete remoteEntity.isExpanded;
for (const key in entity) {
sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [entity[key], key]);
}
sql.replace('branches', remoteEntity);
entityChangesService.addEntityChange('branches', remoteEntity.branchId, hash, sourceId);
entityChangesService.addEntityChange(entityChange, sourceId);
});
return true;
}
return false;
}
function updateNoteRevision(remoteEntity, hash, sourceId) {
const localEntity = sql.getRowOrNull("SELECT * FROM note_revisions WHERE noteRevisionId = ?", [remoteEntity.noteRevisionId]);
sql.transactional(() => {
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.replace('note_revisions', remoteEntity);
entityChangesService.addEntityChange('note_revisions', remoteEntity.noteRevisionId, hash, sourceId);
log.info("Update/sync note revision " + remoteEntity.noteRevisionId);
}
});
}
function updateNoteRevisionContent(remoteEntity, hash, sourceId) {
const localEntity = sql.getRowOrNull("SELECT * FROM note_revision_contents WHERE noteRevisionId = ?", [remoteEntity.noteRevisionId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.transactional(() => {
remoteEntity.content = remoteEntity.content === null ? null : Buffer.from(remoteEntity.content, 'base64');
sql.replace('note_revision_contents', remoteEntity);
entityChangesService.addEntityChange('note_revision_contents', remoteEntity.noteRevisionId, hash, sourceId);
});
return true;
}
return false;
}
function updateNoteReordering(entityId, remote, sourceId) {
sql.transactional(() => {
for (const key in remote) {
sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [remote[key], key]);
}
entityChangesService.addEntityChange('note_reordering', entityId, 'none', sourceId);
});
return true;
}
function updateOptions(remoteEntity, hash, sourceId) {
const localEntity = sql.getRowOrNull("SELECT * FROM options WHERE name = ?", [remoteEntity.name]);
if (localEntity && !localEntity.isSynced) {
return;
}
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.transactional(() => {
sql.replace('options', remoteEntity);
entityChangesService.addEntityChange('options', remoteEntity.name, hash, sourceId, true);
});
return true;
}
return false;
}
function updateRecentNotes(remoteEntity, hash, sourceId) {
const localEntity = sql.getRowOrNull("SELECT * FROM recent_notes WHERE noteId = ?", [remoteEntity.noteId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.transactional(() => {
sql.replace('recent_notes', remoteEntity);
entityChangesService.addEntityChange('recent_notes', remoteEntity.noteId, hash, sourceId);
});
return true;
}
return false;
}
function updateAttribute(remoteEntity, hash, sourceId) {
const localEntity = sql.getRow("SELECT * FROM attributes WHERE attributeId = ?", [remoteEntity.attributeId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.transactional(() => {
sql.replace("attributes", remoteEntity);
entityChangesService.addEntityChange('attributes', remoteEntity.attributeId, hash, sourceId);
});
return true;
}
return false;
}
function updateApiToken(entity, hash, sourceId) {
const apiTokenId = sql.getRow("SELECT * FROM api_tokens WHERE apiTokenId = ?", [entity.apiTokenId]);
if (!apiTokenId) {
sql.transactional(() => {
sql.replace("api_tokens", entity);
entityChangesService.addEntityChange('api_tokens',entity.apiTokenId, hash, sourceId);
});
return true;
}
return false;
}
module.exports = {