Dockery/Trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-10-31 02:16:05 +01:00
Code Issues Packages Projects Releases Activity

sync fixes

Browse Source
This commit is contained in:
zadam
2022-01-09 21:25:15 +01:00
parent 96f4230bc1
commit 2d2641dbd7
12 changed files with 90 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
db/migrations/0191__add_memberId_to_entity_changes.sql Normal file
View File

@@ -0,0 +1,24 @@
CREATE TABLE IF NOT EXISTS "mig_entity_changes" (
`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
`entityName` TEXT NOT NULL,
`entityId` TEXT NOT NULL,
`hash` TEXT NOT NULL,
`isErased` INT NOT NULL,
`changeId` TEXT NOT NULL,
`componentId` TEXT NOT NULL,
`instanceId` TEXT NOT NULL,
`isSynced` INTEGER NOT NULL,
`utcDateChanged` TEXT NOT NULL
);
INSERT INTO mig_entity_changes (id, entityName, entityId, hash, isErased, changeId, componentId, instanceId, isSynced, utcDateChanged)
SELECT id, entityName, entityId, hash, isErased, changeId, '', '', isSynced, utcDateChanged FROM entity_changes;
DROP TABLE entity_changes;
ALTER TABLE mig_entity_changes RENAME TO entity_changes;
CREATE UNIQUE INDEX `IDX_entityChanges_entityName_entityId` ON "entity_changes" (
`entityName`,
`entityId`
);

1
db/migrations/0192__add_index_to_changeId.sql Normal file
View File

@@ -0,0 +1 @@
CREATE INDEX `IDX_entity_changes_changeId` ON `entity_changes` (`changeId`);

3
db/schema.sql
View File

@@ -6,7 +6,7 @@ CREATE TABLE IF NOT EXISTS "entity_changes" (
`isErased` INT NOT NULL, `isErased` INT NOT NULL,
`changeId` TEXT NOT NULL, `changeId` TEXT NOT NULL,
`componentId` TEXT NOT NULL, `componentId` TEXT NOT NULL,
`memberId` TEXT NOT NULL, `instanceId` TEXT NOT NULL,
`isSynced` INTEGER NOT NULL, `isSynced` INTEGER NOT NULL,
`utcDateChanged` TEXT NOT NULL `utcDateChanged` TEXT NOT NULL
); );
@@ -98,6 +98,7 @@ CREATE INDEX `IDX_note_revisions_utcDateCreated` ON `note_revisions` (`utcDateCr
CREATE INDEX `IDX_note_revisions_utcDateLastEdited` ON `note_revisions` (`utcDateLastEdited`); CREATE INDEX `IDX_note_revisions_utcDateLastEdited` ON `note_revisions` (`utcDateLastEdited`);
CREATE INDEX `IDX_note_revisions_dateCreated` ON `note_revisions` (`dateCreated`); CREATE INDEX `IDX_note_revisions_dateCreated` ON `note_revisions` (`dateCreated`);
CREATE INDEX `IDX_note_revisions_dateLastEdited` ON `note_revisions` (`dateLastEdited`); CREATE INDEX `IDX_note_revisions_dateLastEdited` ON `note_revisions` (`dateLastEdited`);
CREATE INDEX `IDX_entity_changes_changeId` ON `entity_changes` (`changeId`);
CREATE INDEX IDX_attributes_name_value CREATE INDEX IDX_attributes_name_value
on attributes (name, value); on attributes (name, value);
CREATE INDEX IDX_attributes_noteId_index CREATE INDEX IDX_attributes_noteId_index

2
src/public/app/setup.js
View File

@@ -23,8 +23,6 @@ function SetupModel() {
this.syncProxy = ko.observable(); this.syncProxy = ko.observable();
this.password = ko.observable(); this.password = ko.observable();
this.instanceType = utils.isElectron() ? "desktop" : "server";
this.setupTypeSelected = () => !!this.setupType(); this.setupTypeSelected = () => !!this.setupType();
this.selectSetupType = () => { this.selectSetupType = () => {

4
src/routes/api/login.js
View File

@@ -3,7 +3,7 @@
const options = require('../../services/options'); const options = require('../../services/options');
const utils = require('../../services/utils'); const utils = require('../../services/utils');
const dateUtils = require('../../services/date_utils'); const dateUtils = require('../../services/date_utils');
const memberId = require('../../services/member_id'); const instanceId = require('../../services/member_id');
const passwordEncryptionService = require('../../services/password_encryption'); const passwordEncryptionService = require('../../services/password_encryption');
const protectedSessionService = require('../../services/protected_session'); const protectedSessionService = require('../../services/protected_session');
const appInfo = require('../../services/app_info'); const appInfo = require('../../services/app_info');
@@ -47,7 +47,7 @@ function loginSync(req) {
req.session.loggedIn = true; req.session.loggedIn = true;
return { return {
memberId: memberId, instanceId: instanceId,
maxEntityChangeId: sql.getValue("SELECT COALESCE(MAX(id), 0) FROM entity_changes WHERE isSynced = 1") maxEntityChangeId: sql.getValue("SELECT COALESCE(MAX(id), 0) FROM entity_changes WHERE isSynced = 1")
}; };
} }

29
src/routes/api/sync.js
View File

@@ -124,7 +124,7 @@ function getChanged(req) {
const startTime = Date.now(); const startTime = Date.now();
let lastEntityChangeId = parseInt(req.query.lastEntityChangeId); let lastEntityChangeId = parseInt(req.query.lastEntityChangeId);
const clientMemberId = req.query.memberId; const clientinstanceId = req.query.instanceId;
let filteredEntityChanges = []; let filteredEntityChanges = [];
while (filteredEntityChanges.length === 0) { while (filteredEntityChanges.length === 0) {
@@ -140,19 +140,28 @@ function getChanged(req) {
break; break;
} }
filteredEntityChanges = entityChanges.filter(ec => ec.memberId !== clientMemberId); filteredEntityChanges = entityChanges.filter(ec => ec.instanceId !== clientinstanceId);
if (filteredEntityChanges.length === 0) {
lastEntityChangeId = entityChanges[entityChanges.length - 1].id;
}
} }
const entityChangesRecords = syncService.getEntityChangesRecords(filteredEntityChanges); const entityChangeRecords = syncService.getEntityChangeRecords(filteredEntityChanges);
if (entityChangesRecords.length > 0) { if (entityChangeRecords.length > 0) {
lastEntityChangeId = entityChangesRecords[entityChangesRecords.length - 1].entityChange.id; lastEntityChangeId = entityChangeRecords[entityChangeRecords.length - 1].entityChange.id;
} }
const ret = { const ret = {
entityChanges: entityChangesRecords, entityChanges: entityChangeRecords,
maxEntityChangeId: sql.getValue('SELECT COALESCE(MAX(id), 0) FROM entity_changes WHERE isSynced = 1'), lastEntityChangeId,
lastEntityChangeId outstandingPullCount: sql.getValue(`
SELECT COUNT(id)
FROM entity_changes
WHERE isSynced = 1
AND instanceId != ?
AND id > ?`, [clientinstanceId, lastEntityChangeId])
}; };
if (ret.entityChanges.length > 0) { if (ret.entityChanges.length > 0) {
@@ -197,10 +206,10 @@ function update(req) {
} }
} }
const {entities, memberId} = body; const {entities, instanceId} = body;
for (const {entityChange, entity} of entities) { for (const {entityChange, entity} of entities) {
syncUpdateService.updateEntity(entityChange, entity, memberId); syncUpdateService.updateEntity(entityChange, entity, instanceId);
} }
} }

2
src/services/app_info.js
View File

@@ -4,7 +4,7 @@ const build = require('./build');
const packageJson = require('../../package'); const packageJson = require('../../package');
const {TRILIUM_DATA_DIR} = require('./data_dir'); const {TRILIUM_DATA_DIR} = require('./data_dir');
const APP_DB_VERSION = 191; const APP_DB_VERSION = 192;
const SYNC_VERSION = 25; const SYNC_VERSION = 25;
const CLIPPER_PROTOCOL_VERSION = "1.0"; const CLIPPER_PROTOCOL_VERSION = "1.0";

12
src/services/entity_changes.js
View File

@@ -3,13 +3,13 @@ const dateUtils = require('./date_utils');
const log = require('./log'); const log = require('./log');
const cls = require('./cls'); const cls = require('./cls');
const utils = require('./utils'); const utils = require('./utils');
const memberId = require('./member_id'); const instanceId = require('./member_id');
const becca = require("../becca/becca"); const becca = require("../becca/becca");
let maxEntityChangeId = 0; let maxEntityChangeId = 0;
function addEntityChangeWithMemberId(origEntityChange, memberId) { function addEntityChangeWithinstanceId(origEntityChange, instanceId) {
const ec = {...origEntityChange, memberId}; const ec = {...origEntityChange, instanceId};
return addEntityChange(ec); return addEntityChange(ec);
} }
@@ -24,7 +24,7 @@ function addEntityChange(origEntityChange) {
} }
ec.componentId = ec.componentId || cls.getComponentId() || ""; ec.componentId = ec.componentId || cls.getComponentId() || "";
ec.memberId = ec.memberId || memberId; ec.instanceId = ec.instanceId || instanceId;
ec.isSynced = ec.isSynced ? 1 : 0; ec.isSynced = ec.isSynced ? 1 : 0;
ec.isErased = ec.isErased ? 1 : 0; ec.isErased = ec.isErased ? 1 : 0;
ec.id = sql.replace("entity_changes", ec); ec.id = sql.replace("entity_changes", ec);
@@ -43,7 +43,7 @@ function addNoteReorderingEntityChange(parentNoteId, componentId) {
utcDateChanged: dateUtils.utcNowDateTime(), utcDateChanged: dateUtils.utcNowDateTime(),
isSynced: true, isSynced: true,
componentId, componentId,
memberId: memberId instanceId: instanceId
}); });
const eventService = require('./events'); const eventService = require('./events');
@@ -146,7 +146,7 @@ module.exports = {
addNoteReorderingEntityChange, addNoteReorderingEntityChange,
moveEntityChangeToTop, moveEntityChangeToTop,
addEntityChange, addEntityChange,
addEntityChangeWithMemberId, addEntityChangeWithinstanceId,
fillAllEntityChanges, fillAllEntityChanges,
addEntityChangesForSector, addEntityChangesForSector,
getMaxEntityChangeId: () => maxEntityChangeId getMaxEntityChangeId: () => maxEntityChangeId

4
src/services/member_id.js
View File

@@ -1,5 +1,5 @@
const utils = require('./utils'); const utils = require('./utils');
const memberId = utils.randomString(12); const instanceId = utils.randomString(12);
module.exports = memberId; module.exports = instanceId;

47
src/services/sync.js
View File

@@ -4,7 +4,7 @@ const log = require('./log');
const sql = require('./sql'); const sql = require('./sql');
const optionService = require('./options'); const optionService = require('./options');
const utils = require('./utils'); const utils = require('./utils');
const memberId = require('./member_id'); const instanceId = require('./member_id');
const dateUtils = require('./date_utils'); const dateUtils = require('./date_utils');
const syncUpdateService = require('./sync_update'); const syncUpdateService = require('./sync_update');
const contentHashService = require('./content_hash'); const contentHashService = require('./content_hash');
@@ -107,11 +107,11 @@ async function doLogin() {
hash: hash hash: hash
}); });
if (resp.memberId === memberId) { if (resp.instanceId === instanceId) {
throw new Error(`Sync server has member ID ${resp.memberId} which is also local. This usually happens when the sync client is (mis)configured to sync with itself (URL points back to client) instead of the correct sync server.`); throw new Error(`Sync server has member ID ${resp.instanceId} which is also local. This usually happens when the sync client is (mis)configured to sync with itself (URL points back to client) instead of the correct sync server.`);
} }
syncContext.memberId = resp.memberId; syncContext.instanceId = resp.instanceId;
const lastSyncedPull = getLastSyncedPull(); const lastSyncedPull = getLastSyncedPull();
@@ -131,22 +131,21 @@ async function pullChanges(syncContext) {
while (true) { while (true) {
const lastSyncedPull = getLastSyncedPull(); const lastSyncedPull = getLastSyncedPull();
const changesUri = `/api/sync/changed?memberId=${memberId}&lastEntityChangeId=${lastSyncedPull}`; const changesUri = `/api/sync/changed?instanceId=${instanceId}&lastEntityChangeId=${lastSyncedPull}`;
const startDate = Date.now(); const startDate = Date.now();
const resp = await syncRequest(syncContext, 'GET', changesUri); const resp = await syncRequest(syncContext, 'GET', changesUri);
const {entityChanges, lastEntityChangeId} = resp;
outstandingPullCount = resp.outstandingPullCount;
const pulledDate = Date.now(); const pulledDate = Date.now();
outstandingPullCount = Math.max(0, resp.maxEntityChangeId - lastSyncedPull);
const {entityChanges, lastEntityChangeId} = resp;
sql.transactional(() => { sql.transactional(() => {
for (const {entityChange, entity} of entityChanges) { for (const {entityChange, entity} of entityChanges) {
const changeAppliedAlready = !entityChange.changeId const changeAppliedAlready = entityChange.changeId
|| !!sql.getValue("SELECT id FROM entity_changes WHERE changeId = ?", [entityChange.changeId]); && !!sql.getValue("SELECT id FROM entity_changes WHERE changeId = ?", [entityChange.changeId]);
if (!changeAppliedAlready) { if (!changeAppliedAlready) {
if (!atLeastOnePullApplied) { // send only for first if (!atLeastOnePullApplied) { // send only for first
@@ -155,10 +154,8 @@ async function pullChanges(syncContext) {
atLeastOnePullApplied = true; atLeastOnePullApplied = true;
} }
syncUpdateService.updateEntity(entityChange, entity, syncContext.memberId); syncUpdateService.updateEntity(entityChange, entity, syncContext.instanceId);
} }
outstandingPullCount = Math.max(0, resp.maxEntityChangeId - entityChange.id);
} }
if (lastSyncedPull !== lastEntityChangeId) { if (lastSyncedPull !== lastEntityChangeId) {
@@ -191,7 +188,7 @@ async function pushChanges(syncContext) {
} }
const filteredEntityChanges = entityChanges.filter(entityChange => { const filteredEntityChanges = entityChanges.filter(entityChange => {
if (entityChange.memberId === syncContext.memberId) { if (entityChange.instanceId === syncContext.instanceId) {
// this may set lastSyncedPush beyond what's actually sent (because of size limit) // this may set lastSyncedPush beyond what's actually sent (because of size limit)
// so this is applied to the database only if there's no actual update // so this is applied to the database only if there's no actual update
lastSyncedPush = entityChange.id; lastSyncedPush = entityChange.id;
@@ -211,12 +208,12 @@ async function pushChanges(syncContext) {
continue; continue;
} }
const entityChangesRecords = getEntityChangesRecords(filteredEntityChanges); const entityChangesRecords = getEntityChangeRecords(filteredEntityChanges);
const startDate = new Date(); const startDate = new Date();
await syncRequest(syncContext, 'PUT', '/api/sync/update', { await syncRequest(syncContext, 'PUT', '/api/sync/update', {
entities: entityChangesRecords, entities: entityChangesRecords,
memberId instanceId
}); });
ws.syncPushInProgress(); ws.syncPushInProgress();
@@ -253,6 +250,11 @@ async function checkContentHash(syncContext) {
const failedChecks = contentHashService.checkContentHashes(resp.entityHashes); const failedChecks = contentHashService.checkContentHashes(resp.entityHashes);
process.exit(0);
throw new Error("AAAA");
return;
if (failedChecks.length > 0) { if (failedChecks.length > 0) {
// before requeuing sectors make sure the entity changes are correct // before requeuing sectors make sure the entity changes are correct
const consistencyChecks = require("./consistency_checks"); const consistencyChecks = require("./consistency_checks");
@@ -332,7 +334,7 @@ function getEntityChangeRow(entityName, entityId) {
} }
} }
function getEntityChangesRecords(entityChanges) { function getEntityChangeRecords(entityChanges) {
const records = []; const records = [];
let length = 0; let length = 0;
@@ -345,13 +347,6 @@ function getEntityChangesRecords(entityChanges) {
const entity = getEntityChangeRow(entityChange.entityName, entityChange.entityId); const entity = getEntityChangeRow(entityChange.entityName, entityChange.entityId);
if (entityChange.entityName === 'options' && !entity.isSynced) {
// if non-synced entities should count towards "lastSyncedPush"
records.push({entityChange});
continue;
}
const record = { entityChange, entity }; const record = { entityChange, entity };
records.push(record); records.push(record);
@@ -423,7 +418,7 @@ require("../becca/becca_loader").beccaLoaded.then(() => {
module.exports = { module.exports = {
sync, sync,
login, login,
getEntityChangesRecords, getEntityChangeRecords,
getOutstandingPullCount, getOutstandingPullCount,
getMaxEntityChangeId getMaxEntityChangeId
}; };

22
src/services/sync_update.js
View File

@@ -4,12 +4,12 @@ const entityChangesService = require('./entity_changes');
const eventService = require('./events'); const eventService = require('./events');
const entityConstructor = require("../becca/entity_constructor"); const entityConstructor = require("../becca/entity_constructor");
function updateEntity(entityChange, entityRow, memberId) { function updateEntity(entityChange, entityRow, instanceId) {
// can be undefined for options with isSynced=false // can be undefined for options with isSynced=false
if (!entityRow) { if (!entityRow) {
if (entityChange.isSynced) { if (entityChange.isSynced) {
if (entityChange.isErased) { if (entityChange.isErased) {
eraseEntity(entityChange, memberId); eraseEntity(entityChange, instanceId);
} }
else { else {
log.info(`Encountered synced non-erased entity change without entity: ${JSON.stringify(entityChange)}`); log.info(`Encountered synced non-erased entity change without entity: ${JSON.stringify(entityChange)}`);
@@ -23,8 +23,8 @@ function updateEntity(entityChange, entityRow, memberId) {
} }
const updated = entityChange.entityName === 'note_reordering' const updated = entityChange.entityName === 'note_reordering'
? updateNoteReordering(entityChange, entityRow, memberId) ? updateNoteReordering(entityChange, entityRow, instanceId)
: updateNormalEntity(entityChange, entityRow, memberId); : updateNormalEntity(entityChange, entityRow, instanceId);
if (updated) { if (updated) {
if (entityRow.isDeleted) { if (entityRow.isDeleted) {
@@ -42,7 +42,7 @@ function updateEntity(entityChange, entityRow, memberId) {
} }
} }
function updateNormalEntity(remoteEntityChange, entity, memberId) { function updateNormalEntity(remoteEntityChange, entity, instanceId) {
const localEntityChange = sql.getRow(` const localEntityChange = sql.getRow(`
SELECT utcDateChanged, hash, isErased SELECT utcDateChanged, hash, isErased
FROM entity_changes FROM entity_changes
@@ -54,7 +54,7 @@ function updateNormalEntity(remoteEntityChange, entity, memberId) {
sql.execute(`DELETE FROM ${remoteEntityChange.entityName} WHERE ${primaryKey} = ?`, remoteEntityChange.entityId); sql.execute(`DELETE FROM ${remoteEntityChange.entityName} WHERE ${primaryKey} = ?`, remoteEntityChange.entityId);
entityChangesService.addEntityChangeWithMemberId(remoteEntityChange, memberId); entityChangesService.addEntityChangeWithinstanceId(remoteEntityChange, instanceId);
}); });
return true; return true;
@@ -71,7 +71,7 @@ function updateNormalEntity(remoteEntityChange, entity, memberId) {
sql.transactional(() => { sql.transactional(() => {
sql.replace(remoteEntityChange.entityName, entity); sql.replace(remoteEntityChange.entityName, entity);
entityChangesService.addEntityChangeWithMemberId(remoteEntityChange, memberId); entityChangesService.addEntityChangeWithinstanceId(remoteEntityChange, instanceId);
}); });
return true; return true;
@@ -80,13 +80,13 @@ function updateNormalEntity(remoteEntityChange, entity, memberId) {
return false; return false;
} }
function updateNoteReordering(entityChange, entity, memberId) { function updateNoteReordering(entityChange, entity, instanceId) {
sql.transactional(() => { sql.transactional(() => {
for (const key in entity) { for (const key in entity) {
sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [entity[key], key]); sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [entity[key], key]);
} }
entityChangesService.addEntityChangeWithMemberId(entityChange, memberId); entityChangesService.addEntityChangeWithinstanceId(entityChange, instanceId);
}); });
return true; return true;
@@ -105,7 +105,7 @@ function handleContent(content) {
return content; return content;
} }
function eraseEntity(entityChange, memberId) { function eraseEntity(entityChange, instanceId) {
const {entityName, entityId} = entityChange; const {entityName, entityId} = entityChange;
if (!["notes", "note_contents", "branches", "attributes", "note_revisions", "note_revision_contents"].includes(entityName)) { if (!["notes", "note_contents", "branches", "attributes", "note_revisions", "note_revision_contents"].includes(entityName)) {
@@ -119,7 +119,7 @@ function eraseEntity(entityChange, memberId) {
eventService.emit(eventService.ENTITY_DELETE_SYNCED, { entityName, entityId }); eventService.emit(eventService.ENTITY_DELETE_SYNCED, { entityName, entityId });
entityChangesService.addEntityChangeWithMemberId(entityChange, memberId); entityChangesService.addEntityChangeWithinstanceId(entityChange, instanceId);
} }
module.exports = { module.exports = {

4
src/views/setup.ejs
View File

@@ -135,9 +135,7 @@
<div class="alert alert-success">Sync has been correctly set up. It will take some time for the initial sync to finish. Once it's done, you'll be redirected to the login page.</div> <div class="alert alert-success">Sync has been correctly set up. It will take some time for the initial sync to finish. Once it's done, you'll be redirected to the login page.</div>
<div data-bind="if: instanceType == 'desktop'"> <div>Outstanding sync items: <strong id="outstanding-syncs">N/A</strong></div>
Outstanding sync items: <strong id="outstanding-syncs">N/A</strong>
</div>
</div> </div>
</div> </div>
</div> </div>