Dockery/Trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-10-31 02:16:05 +01:00
Code Issues Packages Projects Releases Activity

moving out note revision content into separate table, refactoring, WIP

Browse Source
This commit is contained in:
zadam
2019-10-31 21:58:34 +01:00
parent 1a182d1b58
commit cf53cbf1dd
10 changed files with 172 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
db/migrations/0150__note_revision_contents.sql Normal file
View File

@@ -0,0 +1,33 @@
CREATE TABLE IF NOT EXISTS "note_revisions_mig" (`noteRevisionId` TEXT NOT NULL PRIMARY KEY,
`noteId` TEXT NOT NULL,
`title` TEXT,
`content` TEXT,
`isProtected` INT NOT NULL DEFAULT 0,
`utcDateLastEdited` TEXT NOT NULL,
`utcDateCreated` TEXT NOT NULL,
`utcDateModified` TEXT NOT NULL,
`dateLastEdited` TEXT NOT NULL,
`dateCreated` TEXT NOT NULL,
type TEXT DEFAULT '' NOT NULL,
mime TEXT DEFAULT '' NOT NULL,
hash TEXT DEFAULT '' NOT NULL);
CREATE TABLE IF NOT EXISTS "note_revision_contents" (`noteRevisionId` TEXT NOT NULL PRIMARY KEY,
`content` TEXT,
hash TEXT DEFAULT '' NOT NULL,
`utcDateModified` TEXT NOT NULL);
INSERT INTO note_revision_contents (noteRevisionId, content, hash, utcDateModified)
SELECT noteRevisionId, content, hash, utcDateModified FROM note_revisions;
INSERT INTO note_revisions_mig (noteRevisionId, noteId, title, isProtected, utcDateLastEdited, utcDateCreated, utcDateModified, dateLastEdited, dateCreated, type, mime, hash)
SELECT noteRevisionId, noteId, title, isProtected, utcDateModifiedFrom, utcDateModifiedTo, utcDateModifiedTo, dateModifiedFrom, dateModifiedTo, type, mime, hash FROM note_revisions;
DROP TABLE note_revisions;
ALTER TABLE note_revisions_mig RENAME TO note_revisions;
CREATE INDEX `IDX_note_revisions_noteId` ON `note_revisions` (`noteId`);
CREATE INDEX `IDX_note_revisions_utcDateCreated` ON `note_revisions` (`utcDateCreated`);
CREATE INDEX `IDX_note_revisions_utcDateLastEdited` ON `note_revisions` (`utcDateLastEdited`);
CREATE INDEX `IDX_note_revisions_dateCreated` ON `note_revisions` (`dateCreated`);
CREATE INDEX `IDX_note_revisions_dateLastEdited` ON `note_revisions` (`dateLastEdited`);

16
src/entities/note.js
View File

@@ -14,8 +14,6 @@ const LABEL_DEFINITION = 'label-definition';
const RELATION = 'relation';
const RELATION_DEFINITION = 'relation-definition';
const STRING_MIME_TYPES = ["application/x-javascript"];
/**
* This represents a Note which is a central object in the Trilium Notes project.
*
@@ -44,7 +42,7 @@ class Note extends Entity {
super(row);
this.isProtected = !!this.isProtected;
/* true if content (meaning any kind of potentially encrypted content) is either not encrypted
/* true if content is either not encrypted
* or encrypted, but with available protected session (so effectively decrypted) */
this.isContentAvailable = true;
@@ -53,7 +51,7 @@ class Note extends Entity {
this.isContentAvailable = protectedSessionService.isProtectedSessionAvailable();
if (this.isContentAvailable) {
protectedSessionService.decryptNote(this);
this.title = protectedSessionService.decrypt(this.title);
}
else {
this.title = "[protected]";
@@ -88,7 +86,7 @@ class Note extends Entity {
if (this.isProtected) {
if (this.isContentAvailable) {
protectedSessionService.decryptNoteContent(this);
this.content = this.content === null ? null : protectedSessionService.decrypt(this.content);
}
else {
this.content = "";
@@ -129,7 +127,7 @@ class Note extends Entity {
if (this.isProtected) {
if (this.isContentAvailable) {
protectedSessionService.encryptNoteContent(pojo);
pojo.content = protectedSessionService.encrypt(pojo.content);
}
else {
throw new Error(`Cannot update content of noteId=${this.noteId} since we're out of protected session.`);
@@ -171,9 +169,7 @@ class Note extends Entity {
/** @returns {boolean} true if the note has string content (not binary) */
isStringNote() {
return ["text", "code", "relation-map", "search"].includes(this.type)
|| this.mime.startsWith('text/')
|| STRING_MIME_TYPES.includes(this.mime);
return utils.isStringNote(this.type, this.mime);
}
/** @returns {string} JS script environment - either "frontend" or "backend" */
@@ -746,7 +742,7 @@ class Note extends Entity {
updatePojo(pojo) {
if (pojo.isProtected) {
if (this.isContentAvailable) {
protectedSessionService.encryptNote(pojo);
pojo.title = protectedSessionService.encrypt(pojo.title);
}
else {
// updating protected note outside of protected session means we will keep original ciphertexts

106
src/entities/note_revision.js
View File

@@ -3,6 +3,9 @@
const Entity = require('./entity');
const protectedSessionService = require('../services/protected_session');
const repository = require('../services/repository');
const utils = require('../services/utils');
const dateUtils = require('../services/date_utils');
const syncTableService = require('../services/sync_table');
/**
* NoteRevision represents snapshot of note's title and content at some point in the past. It's used for seamless note versioning.
@@ -12,7 +15,6 @@ const repository = require('../services/repository');
* @param {string} type
* @param {string} mime
* @param {string} title
* @param {string} content
* @param {string} isProtected
* @param {string} dateModifiedFrom
* @param {string} dateModifiedTo
@@ -24,7 +26,7 @@ const repository = require('../services/repository');
class NoteRevision extends Entity {
static get entityName() { return "note_revisions"; }
static get primaryKeyName() { return "noteRevisionId"; }
static get hashedProperties() { return ["noteRevisionId", "noteId", "title", "content", "isProtected", "dateModifiedFrom", "dateModifiedTo", "utcDateModifiedFrom", "utcDateModifiedTo"]; }
static get hashedProperties() { return ["noteRevisionId", "noteId", "title", "isProtected", "dateModifiedFrom", "dateModifiedTo", "utcDateModifiedFrom", "utcDateModifiedTo"]; }
constructor(row) {
super(row);
@@ -32,7 +34,12 @@ class NoteRevision extends Entity {
this.isProtected = !!this.isProtected;
if (this.isProtected) {
protectedSessionService.decryptNoteRevision(this);
if (protectedSessionService.isProtectedSessionAvailable()) {
this.title = protectedSessionService.decrypt(this.title);
}
else {
this.title = "[Protected]";
}
}
}
@@ -40,12 +47,97 @@ class NoteRevision extends Entity {
return await repository.getEntity("SELECT * FROM notes WHERE noteId = ?", [this.noteId]);
}
beforeSaving() {
if (this.isProtected) {
protectedSessionService.encryptNoteRevision(this);
/** @returns {boolean} true if the note has string content (not binary) */
isStringNote() {
return utils.isStringNote(this.type, this.mime);
}
super.beforeSaving();
/*
* Note revision content has quite special handling - it's not a separate entity, but a lazily loaded
* part of NoteRevision entity with it's own sync. Reason behind this hybrid design is that
* content can be quite large and it's not necessary to load it / fill memory for any note access even
* if we don't need a content, especially for bulk operations like search.
*
* This is the same approach as is used for Note's content.
*/
/** @returns {Promise<*>} */
async getContent(silentNotFoundError = false) {
if (this.content === undefined) {
const res = await sql.getRow(`SELECT content, hash FROM note_revision_contents WHERE noteRevisionId = ?`, [this.noteRevisionId]);
if (!res) {
if (silentNotFoundError) {
return undefined;
}
else {
throw new Error("Cannot find note revision content for noteRevisionId=" + this.noteRevisionId);
}
}
this.content = res.content;
if (this.isProtected) {
if (protectedSessionService.isProtectedSessionAvailable()) {
this.content = protectedSessionService.decrypt(this.content);
}
else {
this.content = "";
}
}
if (this.isStringNote()) {
this.content = this.content === null
? ""
: this.content.toString("UTF-8");
}
}
return this.content;
}
/** @returns {Promise} */
async setContent(content) {
// force updating note itself so that dateChanged is represented correctly even for the content
this.forcedChange = true;
await this.save();
this.content = content;
const pojo = {
noteRevisionId: this.noteRevisionId,
content: content,
utcDateModified: dateUtils.utcNowDateTime(),
hash: utils.hash(this.noteRevisionId + "|" + content)
};
if (this.isProtected) {
if (protectedSessionService.isProtectedSessionAvailable()) {
pojo.content = protectedSessionService.encrypt(pojo.content);
}
else {
throw new Error(`Cannot update content of noteRevisionId=${this.noteRevisionId} since we're out of protected session.`);
}
}
await sql.upsert("note_revision_contents", "noteRevisionId", pojo);
await syncTableService.addNoteContentSync(this.noteId);
}
// cannot be static!
updatePojo(pojo) {
if (pojo.isProtected) {
if (protectedSessionService.isProtectedSessionAvailable()) {
pojo.title = protectedSessionService.encrypt(pojo.title);
}
else {
// updating protected note outside of protected session means we will keep original ciphertexts
delete pojo.title;
}
}
delete pojo.content;
}
}

4
src/routes/api/recent_changes.js
View File

@@ -41,8 +41,8 @@ async function getRecentChanges() {
for (const change of recentChanges) {
if (change.current_isProtected) {
if (protectedSessionService.isProtectedSessionAvailable()) {
change.title = protectedSessionService.decryptNoteTitle(change.noteId, change.title);
change.current_title = protectedSessionService.decryptNoteTitle(change.noteId, change.current_title);
change.title = protectedSessionService.decrypt(change.title);
change.current_title = protectedSessionService.decrypt(change.current_title);
}
else {
change.title = change.current_title = "[Protected]";

4
src/services/app_info.js
View File

@@ -4,8 +4,8 @@ const build = require('./build');
const packageJson = require('../../package');
const {TRILIUM_DATA_DIR} = require('./data_dir');
const APP_DB_VERSION = 149;
const SYNC_VERSION = 10;
const APP_DB_VERSION = 150;
const SYNC_VERSION = 11;
const CLIPPER_PROTOCOL_VERSION = "1.0";
module.exports = {

2
src/services/note_cache.js
View File

@@ -54,7 +54,7 @@ async function loadProtectedNotes() {
protectedNoteTitles = await sql.getMap(`SELECT noteId, title FROM notes WHERE isDeleted = 0 AND isProtected = 1`);
for (const noteId in protectedNoteTitles) {
protectedNoteTitles[noteId] = protectedSessionService.decryptNoteTitle(noteId, protectedNoteTitles[noteId]);
protectedNoteTitles[noteId] = protectedSessionService.decrypt(protectedNoteTitles[noteId]);
}
}

83
src/services/protected_session.js
View File

@@ -34,93 +34,28 @@ function isProtectedSessionAvailable() {
return !!dataKeyMap[protectedSessionId];
}
function decryptNoteTitle(noteId, encryptedTitle) {
const dataKey = getDataKey();
try {
return dataEncryptionService.decryptString(dataKey, encryptedTitle);
}
catch (e) {
e.message = `Cannot decrypt note title for noteId=${noteId}: ` + e.message;
throw e;
}
}
function decryptNote(note) {
if (!note.isProtected) {
return;
}
if (note.title) {
note.title = decryptNoteTitle(note.noteId, note.title);
}
}
function decryptNoteContent(note) {
try {
if (note.content != null) {
note.content = dataEncryptionService.decrypt(getDataKey(), note.content);
}
}
catch (e) {
e.message = `Cannot decrypt content for noteId=${note.noteId}: ` + e.message;
throw e;
}
}
function decryptNotes(notes) {
for (const note of notes) {
decryptNote(note);
if (note.isProtected) {
note.title = decrypt(note.title);
}
}
}
function decryptNoteRevision(hist) {
const dataKey = getDataKey();
if (!hist.isProtected) {
return;
function encrypt(plainText) {
return dataEncryptionService.encrypt(getDataKey(), plainText);
}
try {
if (hist.title) {
hist.title = dataEncryptionService.decryptString(dataKey, hist.title.toString());
}
if (hist.content) {
hist.content = dataEncryptionService.decryptString(dataKey, hist.content.toString());
}
}
catch (e) {
throw new Error(`Decryption failed for note ${hist.noteId}, revision ${hist.noteRevisionId}: ` + e.message + " " + e.stack);
}
}
function encryptNote(note) {
note.title = dataEncryptionService.encrypt(getDataKey(), note.title);
}
function encryptNoteContent(note) {
note.content = dataEncryptionService.encrypt(getDataKey(), note.content);
}
function encryptNoteRevision(revision) {
const dataKey = getDataKey();
revision.title = dataEncryptionService.encrypt(dataKey, revision.title);
revision.content = dataEncryptionService.encrypt(dataKey, revision.content);
function decrypt(cipherText) {
return dataEncryptionService.encrypt(getDataKey(), cipherText);
}
module.exports = {
setDataKey,
getDataKey,
isProtectedSessionAvailable,
decryptNoteTitle,
decryptNote,
decryptNoteContent,
encrypt,
decrypt,
decryptNotes,
decryptNoteRevision,
encryptNote,
encryptNoteContent,
encryptNoteRevision,
setProtectedSessionId
};

54
src/services/sync_table.js
View File

@@ -4,42 +4,6 @@ const dateUtils = require('./date_utils');
const log = require('./log');
const cls = require('./cls');
async function addNoteSync(noteId, sourceId) {
await addEntitySync("notes", noteId, sourceId)
}
async function addNoteContentSync(noteId, sourceId) {
await addEntitySync("note_contents", noteId, sourceId)
}
async function addBranchSync(branchId, sourceId) {
await addEntitySync("branches", branchId, sourceId)
}
async function addNoteReorderingSync(parentNoteId, sourceId) {
await addEntitySync("note_reordering", parentNoteId, sourceId)
}
async function addNoteRevisionSync(noteRevisionId, sourceId) {
await addEntitySync("note_revisions", noteRevisionId, sourceId);
}
async function addOptionsSync(name, sourceId) {
await addEntitySync("options", name, sourceId);
}
async function addRecentNoteSync(noteId, sourceId) {
await addEntitySync("recent_notes", noteId, sourceId);
}
async function addAttributeSync(attributeId, sourceId) {
await addEntitySync("attributes", attributeId, sourceId);
}
async function addApiTokenSync(apiTokenId, sourceId) {
await addEntitySync("api_tokens", apiTokenId, sourceId);
}
async function addEntitySync(entityName, entityId, sourceId) {
await sql.replace("sync", {
entityName: entityName,
@@ -107,15 +71,15 @@ async function fillAllSyncRows() {
}
module.exports = {
addNoteSync,
addNoteContentSync,
addBranchSync,
addNoteReorderingSync,
addNoteRevisionSync,
addOptionsSync,
addRecentNoteSync,
addAttributeSync,
addApiTokenSync,
addNoteSync: async (noteId, sourceId) => await addEntitySync("notes", noteId, sourceId),
addNoteContentSync: async (noteId, sourceId) => await addEntitySync("note_contents", noteId, sourceId),
addBranchSync: async (branchId, sourceId) => await addEntitySync("branches", branchId, sourceId),
addNoteReorderingSync: async (parentNoteId, sourceId) => await addEntitySync("note_reordering", parentNoteId, sourceId),
addNoteRevisionSync: async (noteRevisionId, sourceId) => await addEntitySync("note_revisions", noteRevisionId, sourceId),
addOptionsSync: async (name, sourceId) => await addEntitySync("options", name, sourceId),
addRecentNoteSync: async (noteId, sourceId) => await addEntitySync("recent_notes", noteId, sourceId),
addAttributeSync: async (attributeId, sourceId) => await addEntitySync("attributes", attributeId, sourceId),
addApiTokenSync: async (apiTokenId, sourceId) => await addEntitySync("api_tokens", apiTokenId, sourceId),
addEntitySync,
fillAllSyncRows
};

4
src/services/sync_update.js
View File

@@ -117,9 +117,9 @@ async function updateNoteRevision(entity, sourceId) {
async function updateNoteReordering(entityId, entity, sourceId) {
await sql.transactional(async () => {
Object.keys(entity).forEach(async key => {
for (const key in entity) {
await sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [entity[key], key]);
});
}
await syncTableService.addNoteReorderingSync(entityId, sourceId);
});

11
src/services/utils.js
View File

@@ -154,6 +154,14 @@ function getContentDisposition(filename) {
return `file; filename="${sanitizedFilename}"; filename*=UTF-8''${sanitizedFilename}`;
}
const STRING_MIME_TYPES = ["application/x-javascript"];
function isStringNote(type, mime) {
return ["text", "code", "relation-map", "search"].includes(type)
|| mime.startsWith('text/')
|| STRING_MIME_TYPES.includes(mime);
}
module.exports = {
randomSecureToken,
randomString,
@@ -177,5 +185,6 @@ module.exports = {
escapeRegExp,
crash,
sanitizeFilenameForHeader,
getContentDisposition
getContentDisposition,
isStringNote
};