Dockery/Trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-10-31 02:16:05 +01:00
Code Issues Packages Projects Releases Activity

Merge remote-tracking branch 'origin/optimize-tree'

Browse Source
This commit is contained in:
zadam
2020-12-10 22:54:16 +01:00
parent 8f1b2fa226 5010175b84
commit 0b6fa4ab06
17 changed files with 208 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/entities/note.js
View File

@@ -777,7 +777,7 @@ class Note extends Entity {
* @returns {NoteRevision[]}
*/
getRevisions() {
return this.repository.getEntities("SELECT * FROM note_revisions WHERE noteId = ?", [this.noteId]);
return this.repository.getEntities("SELECT * FROM note_revisions WHERE isErased = 0 AND noteId = ?", [this.noteId]);
}
/**

13
src/public/app/dialogs/options/other.js
View File

@@ -51,6 +51,12 @@ const TPL = `
<label for="erase-notes-after-time-in-seconds">Erase notes after X seconds</label>
<input class="form-control" id="erase-notes-after-time-in-seconds" type="number" min="0">
</div>
<p>You can also trigger erasing manually:</p>
<button id="erase-deleted-notes-now-button" class="btn">Erase deleted notes now</button>
<br/><br/>
</div>
<div>
@@ -117,6 +123,13 @@ export default class ProtectedSessionOptions {
return false;
});
this.$eraseDeletedNotesButton = $("#erase-deleted-notes-now-button");
this.$eraseDeletedNotesButton.on('click', () => {
server.post('notes/erase-deleted-notes-now').then(() => {
toastService.showMessage("Deleted notes have been erased.");
});
});
this.$protectedSessionTimeout = $("#protected-session-timeout-in-seconds");
this.$protectedSessionTimeout.on('change', () => {

4
src/public/app/entities/note_short.js
View File

@@ -75,15 +75,17 @@ class NoteShort {
this.parentToBranch[parentNoteId] = branchId;
}
addChild(childNoteId, branchId) {
addChild(childNoteId, branchId, sort = true) {
if (!this.children.includes(childNoteId)) {
this.children.push(childNoteId);
}
this.childToBranch[childNoteId] = branchId;
if (sort) {
this.sortChildren();
}
}
sortChildren() {
const branchIdPos = {};

49
src/public/app/services/tree_cache.js
View File

@@ -21,9 +21,6 @@ class TreeCache {
async loadInitialTree() {
const resp = await server.get('tree');
// FIXME: we need to do this to cover for ascendants of template notes which are not loaded
await this.loadParents(resp, false);
// clear the cache only directly before adding new content which is important for e.g. switching to protected session
/** @type {Object.<string, NoteShort>} */
@@ -44,50 +41,18 @@ class TreeCache {
async loadSubTree(subTreeNoteId) {
const resp = await server.get('tree?subTreeNoteId=' + subTreeNoteId);
await this.loadParents(resp, true);
this.addResp(resp);
return this.notes[subTreeNoteId];
}
async loadParents(resp, additiveLoad) {
const noteIds = new Set(resp.notes.map(note => note.noteId));
const missingNoteIds = [];
const existingNotes = additiveLoad ? this.notes : {};
for (const branch of resp.branches) {
if (!(branch.parentNoteId in existingNotes) && !noteIds.has(branch.parentNoteId) && branch.parentNoteId !== 'none') {
missingNoteIds.push(branch.parentNoteId);
}
}
for (const attr of resp.attributes) {
if (attr.type === 'relation' && attr.name === 'template' && !(attr.value in existingNotes) && !noteIds.has(attr.value)) {
missingNoteIds.push(attr.value);
}
if (!(attr.noteId in existingNotes) && !noteIds.has(attr.noteId)) {
missingNoteIds.push(attr.noteId);
}
}
if (missingNoteIds.length > 0) {
const newResp = await server.post('tree/load', { noteIds: missingNoteIds });
resp.notes = resp.notes.concat(newResp.notes);
resp.branches = resp.branches.concat(newResp.branches);
resp.attributes = resp.attributes.concat(newResp.attributes);
await this.loadParents(resp, additiveLoad);
}
}
addResp(resp) {
const noteRows = resp.notes;
const branchRows = resp.branches;
const attributeRows = resp.attributes;
const noteIdsToSort = new Set();
for (const noteRow of noteRows) {
const {noteId} = noteRow;
@@ -154,7 +119,9 @@ class TreeCache {
const parentNote = this.notes[branch.parentNoteId];
if (parentNote) {
parentNote.addChild(branch.noteId, branch.branchId);
parentNote.addChild(branch.noteId, branch.branchId, false);
noteIdsToSort.add(parentNote.noteId);
}
}
@@ -179,6 +146,11 @@ class TreeCache {
}
}
}
// sort all of them at once, this avoids repeated sorts (#1480)
for (const noteId of noteIdsToSort) {
this.notes[noteId].sortChildren();
}
}
async reloadNotes(noteIds) {
@@ -190,7 +162,6 @@ class TreeCache {
const resp = await server.post('tree/load', { noteIds });
await this.loadParents(resp, true);
this.addResp(resp);
const searchNoteIds = [];

11
src/public/app/widgets/note_detail.js
View File

@@ -249,13 +249,22 @@ export default class NoteDetailWidget extends TabAwareWidget {
this.$widget.find('.note-detail-printable:visible').printThis({
header: $("<h2>").text(this.note && this.note.title).prop('outerHTML'),
footer: "<script>document.body.className += ' ck-content printed-content';</script>",
footer: `
<script src="libraries/katex/katex.min.js"></script>
<script src="libraries/katex/auto-render.min.js"></script>
<script>
document.body.className += ' ck-content printed-content';
renderMathInElement(document.body, {});
</script>
`,
importCSS: false,
loadCSS: [
"libraries/codemirror/codemirror.css",
"libraries/ckeditor/ckeditor-content.css",
"libraries/ckeditor/ckeditor-content.css",
"libraries/bootstrap/css/bootstrap.min.css",
"libraries/katex/katex.min.css",
"stylesheets/print.css",
"stylesheets/relation_map.css",
"stylesheets/themes.css"

7
src/routes/api/notes.js
View File

@@ -193,6 +193,10 @@ function duplicateSubtree(req) {
return noteService.duplicateSubtree(noteId, parentNoteId);
}
function eraseDeletedNotesNow() {
noteService.eraseDeletedNotesNow();
}
module.exports = {
getNote,
updateNote,
@@ -204,5 +208,6 @@ module.exports = {
setNoteTypeMime,
getRelationMap,
changeTitle,
duplicateSubtree
duplicateSubtree,
eraseDeletedNotesNow
};

3
src/routes/api/recent_changes.js
View File

@@ -23,7 +23,8 @@ function getRecentChanges(req) {
note_revisions.dateCreated AS date
FROM
note_revisions
JOIN notes USING(noteId)`);
JOIN notes USING(noteId)
WHERE note_revisions.isErased = 0`);
for (const noteRevision of noteRevisions) {
if (noteCacheService.isInAncestor(noteRevision.noteId, ancestorNoteId)) {

2
src/routes/api/setup.js
View File

@@ -38,6 +38,8 @@ function saveSyncSeed(req) {
}]
}
log.info("Saved sync seed.");
sqlInit.createDatabaseForSync(options);
}

91
src/routes/api/tree.js
View File

@@ -5,14 +5,15 @@ const optionService = require('../../services/options');
const treeService = require('../../services/tree');
function getNotesAndBranchesAndAttributes(noteIds) {
noteIds = Array.from(new Set(noteIds));
const notes = treeService.getNotes(noteIds);
const notes = treeService.getNotesIncludingAscendants(noteIds);
noteIds = notes.map(note => note.noteId);
noteIds = new Set(notes.map(note => note.noteId));
sql.fillNoteIdList(noteIds);
// joining child note to filter out not completely synchronised notes which would then cause errors later
// cannot do that with parent because of root note's 'none' parent
const branches = sql.getManyRows(`
const branches = sql.getRows(`
SELECT
branches.branchId,
branches.noteId,
@@ -20,28 +21,45 @@ function getNotesAndBranchesAndAttributes(noteIds) {
branches.notePosition,
branches.prefix,
branches.isExpanded
FROM branches
FROM param_list
JOIN branches ON param_list.paramId = branches.noteId OR param_list.paramId = branches.parentNoteId
JOIN notes AS child ON child.noteId = branches.noteId
WHERE branches.isDeleted = 0
AND (branches.noteId IN (???) OR parentNoteId IN (???))`, noteIds);
WHERE branches.isDeleted = 0`);
const attributes = sql.getRows(`
SELECT
attributes.attributeId,
attributes.noteId,
attributes.type,
attributes.name,
attributes.value,
attributes.position,
attributes.isInheritable
FROM param_list
JOIN attributes ON attributes.noteId = param_list.paramId
OR (attributes.type = 'relation' AND attributes.value = param_list.paramId)
WHERE attributes.isDeleted = 0`);
// we don't really care about the direction of the relation
const missingTemplateNoteIds = attributes
.filter(attr => attr.type === 'relation'
&& attr.name === 'template'
&& !noteIds.has(attr.value))
.map(attr => attr.value);
if (missingTemplateNoteIds.length > 0) {
const templateData = getNotesAndBranchesAndAttributes(missingTemplateNoteIds);
// there are going to be duplicates with simple concatenation, however:
// 1) shouldn't matter for the frontend which will update the entity twice
// 2) there shouldn't be many duplicates. There isn't that many templates
addArrays(notes, templateData.notes);
addArrays(branches, templateData.branches);
addArrays(attributes, templateData.attributes);
}
// sorting in memory is faster
branches.sort((a, b) => a.notePosition - b.notePosition < 0 ? -1 : 1);
const attributes = sql.getManyRows(`
SELECT
attributeId,
noteId,
type,
name,
value,
position,
isInheritable
FROM attributes
WHERE isDeleted = 0
AND (noteId IN (???) OR (type = 'relation' AND value IN (???)))`, noteIds);
// sorting in memory is faster
attributes.sort((a, b) => a.position - b.position < 0 ? -1 : 1);
return {
@@ -51,6 +69,16 @@ function getNotesAndBranchesAndAttributes(noteIds) {
};
}
// should be fast based on https://stackoverflow.com/a/64826145/944162
// in this case it is assumed that target is potentially much larger than elementsToAdd
function addArrays(target, elementsToAdd) {
while (elementsToAdd.length) {
target.push(elementsToAdd.shift());
}
return target;
}
function getTree(req) {
const subTreeNoteId = req.query.subTreeNoteId || 'root';
@@ -64,25 +92,8 @@ function getTree(req) {
JOIN treeWithDescendants ON branches.parentNoteId = treeWithDescendants.noteId
WHERE treeWithDescendants.isExpanded = 1
AND branches.isDeleted = 0
),
treeWithDescendantsAndAscendants AS (
SELECT noteId FROM treeWithDescendants
UNION
SELECT branches.parentNoteId FROM branches
JOIN treeWithDescendantsAndAscendants ON branches.noteId = treeWithDescendantsAndAscendants.noteId
WHERE branches.isDeleted = 0
AND branches.parentNoteId != ?
),
treeWithDescendantsAscendantsAndTemplates AS (
SELECT noteId FROM treeWithDescendantsAndAscendants
UNION
SELECT attributes.value FROM attributes
JOIN treeWithDescendantsAscendantsAndTemplates ON attributes.noteId = treeWithDescendantsAscendantsAndTemplates.noteId
WHERE attributes.isDeleted = 0
AND attributes.type = 'relation'
AND attributes.name = 'template'
)
SELECT noteId FROM treeWithDescendantsAscendantsAndTemplates`, [subTreeNoteId, subTreeNoteId]);
SELECT noteId FROM treeWithDescendants`, [subTreeNoteId]);
noteIds.push(subTreeNoteId);

1
src/routes/routes.js
View File

@@ -154,6 +154,7 @@ function register(app) {
route(GET, '/api/notes/:noteId/revisions/:noteRevisionId/download', [auth.checkApiAuthOrElectron], noteRevisionsApiRoute.downloadNoteRevision);
apiRoute(PUT, '/api/notes/:noteId/restore-revision/:noteRevisionId', noteRevisionsApiRoute.restoreNoteRevision);
apiRoute(POST, '/api/notes/relation-map', notesApiRoute.getRelationMap);
apiRoute(POST, '/api/notes/erase-deleted-notes-now', notesApiRoute.eraseDeletedNotesNow);
apiRoute(PUT, '/api/notes/:noteId/change-title', notesApiRoute.changeTitle);
apiRoute(POST, '/api/notes/:noteId/duplicate/:parentNoteId', notesApiRoute.duplicateSubtree);

2
src/services/consistency_checks.js
View File

@@ -651,7 +651,7 @@ class ConsistencyChecks {
// root branch should always be expanded
sql.execute("UPDATE branches SET isExpanded = 1 WHERE branchId = 'root'");
if (this.unrecoveredConsistencyErrors) {
if (!this.unrecoveredConsistencyErrors) {
// we run this only if basic checks passed since this assumes basic data consistency
this.checkTreeCycles();

4
src/services/handlers.js
View File

@@ -70,6 +70,10 @@ eventService.subscribe(eventService.ENTITY_CREATED, ({ entityName, entity }) =>
if (templateNoteContent) {
note.setContent(templateNoteContent);
}
note.type = templateNote.type;
note.mime = templateNote.mime;
note.save();
}
noteService.duplicateSubtreeWithoutRoot(templateNote.noteId, note.noteId);

8
src/services/note_revisions.js
View File

@@ -2,6 +2,7 @@
const NoteRevision = require('../entities/note_revision');
const dateUtils = require('../services/date_utils');
const log = require('../services/log');
/**
* @param {Note} note
@@ -9,6 +10,7 @@ const dateUtils = require('../services/date_utils');
function protectNoteRevisions(note) {
for (const revision of note.getRevisions()) {
if (note.isProtected !== revision.isProtected) {
try {
const content = revision.getContent();
revision.isProtected = note.isProtected;
@@ -18,6 +20,12 @@ function protectNoteRevisions(note) {
revision.save();
}
catch (e) {
log.error("Could not un/protect note revision ID = " + revision.noteRevisionId);
throw e;
}
}
}
}

26
src/services/notes.js
View File

@@ -183,6 +183,7 @@ function protectNoteRecursively(note, protect, includingSubTree, taskContext) {
}
function protectNote(note, protect) {
try {
if (protect !== note.isProtected) {
const content = note.getContent();
@@ -196,6 +197,12 @@ function protectNote(note, protect) {
noteRevisionService.protectNoteRevisions(note);
}
catch (e) {
log.error("Could not un/protect note ID = " + note.noteId);
throw e;
}
}
function findImageLinks(content, foundLinks) {
const re = /src="[^"]*api\/images\/([a-zA-Z0-9]+)\//g;
@@ -459,7 +466,7 @@ function saveNoteRevision(note) {
const revisionCutoff = dateUtils.utcDateStr(new Date(now.getTime() - noteRevisionSnapshotTimeInterval * 1000));
const existingNoteRevisionId = sql.getValue(
"SELECT noteRevisionId FROM note_revisions WHERE noteId = ? AND utcDateCreated >= ?", [note.noteId, revisionCutoff]);
"SELECT noteRevisionId FROM note_revisions WHERE noteId = ? AND isErased = 0 AND utcDateCreated >= ?", [note.noteId, revisionCutoff]);
const msSinceDateCreated = now.getTime() - dateUtils.parseDateTime(note.utcDateCreated).getTime();
@@ -666,8 +673,10 @@ function scanForLinks(note) {
}
}
function eraseDeletedNotes() {
const eraseNotesAfterTimeInSeconds = optionService.getOptionInt('eraseNotesAfterTimeInSeconds');
function eraseDeletedNotes(eraseNotesAfterTimeInSeconds = null) {
if (eraseNotesAfterTimeInSeconds === null) {
eraseNotesAfterTimeInSeconds = optionService.getOptionInt('eraseNotesAfterTimeInSeconds');
}
const cutoffDate = new Date(Date.now() - eraseNotesAfterTimeInSeconds * 1000);
@@ -717,6 +726,10 @@ function eraseDeletedNotes() {
log.info(`Erased notes: ${JSON.stringify(noteIdsToErase)}`);
}
function eraseDeletedNotesNow() {
eraseDeletedNotes(0);
}
// do a replace in str - all keys should be replaced by the corresponding values
function replaceByMap(str, mapObj) {
const re = new RegExp(Object.keys(mapObj).join("|"),"g");
@@ -823,9 +836,9 @@ function getNoteIdMapping(origNote) {
sqlInit.dbReady.then(() => {
// first cleanup kickoff 5 minutes after startup
setTimeout(cls.wrap(eraseDeletedNotes), 5 * 60 * 1000);
setTimeout(cls.wrap(() => eraseDeletedNotes()), 5 * 60 * 1000);
setInterval(cls.wrap(eraseDeletedNotes), 4 * 3600 * 1000);
setInterval(cls.wrap(() => eraseDeletedNotes()), 4 * 3600 * 1000);
});
module.exports = {
@@ -839,5 +852,6 @@ module.exports = {
duplicateSubtree,
duplicateSubtreeWithoutRoot,
getUndeletedParentBranches,
triggerNoteTitleChanged
triggerNoteTitleChanged,
eraseDeletedNotesNow
};

26
src/services/sql.js
View File

@@ -236,6 +236,29 @@ function transactional(func) {
return ret;
}
function fillNoteIdList(noteIds, truncate = true) {
if (noteIds.length === 0) {
return;
}
if (truncate) {
execute("DELETE FROM param_list");
}
noteIds = Array.from(new Set(noteIds));
if (noteIds.length > 30000) {
fillNoteIdList(noteIds.slice(30000), false);
noteIds = noteIds.slice(0, 30000);
}
// doing it manually to avoid this showing up on the sloq query list
const s = stmt(`INSERT INTO param_list VALUES ` + noteIds.map(noteId => `(?)`).join(','), noteIds);
s.run(noteIds);
}
module.exports = {
dbConnection,
insert,
@@ -253,5 +276,6 @@ module.exports = {
executeMany,
executeScript,
transactional,
upsert
upsert,
fillNoteIdList
};

2
src/services/sql_init.js
View File

@@ -40,6 +40,8 @@ async function initDbConnection() {
require('./options_init').initStartupOptions();
sql.execute('CREATE TEMP TABLE "param_list" (`paramId` TEXT NOT NULL PRIMARY KEY)');
dbReady.resolve();
}

36
src/services/tree.js
View File

@@ -6,6 +6,41 @@ const Branch = require('../entities/branch');
const entityChangesService = require('./entity_changes.js');
const protectedSessionService = require('./protected_session');
function getNotesIncludingAscendants(noteIds) {
noteIds = Array.from(new Set(noteIds));
sql.fillNoteIdList(noteIds);
// we return also deleted notes which have been specifically asked for
const notes = sql.getRows(`
WITH RECURSIVE
treeWithAscendants AS (
SELECT paramId AS noteId FROM param_list
UNION
SELECT branches.parentNoteId FROM branches
JOIN treeWithAscendants ON branches.noteId = treeWithAscendants.noteId
WHERE branches.isDeleted = 0
)
SELECT
noteId,
title,
isProtected,
type,
mime,
isDeleted
FROM notes
JOIN treeWithAscendants USING(noteId)`);
protectedSessionService.decryptNotes(notes);
notes.forEach(note => {
note.isProtected = !!note.isProtected
});
return notes;
}
function getNotes(noteIds) {
// we return also deleted notes which have been specifically asked for
const notes = sql.getManyRows(`
@@ -190,6 +225,7 @@ function setNoteToParent(noteId, prefix, parentNoteId) {
module.exports = {
getNotes,
getNotesIncludingAscendants,
validateParentChild,
sortNotesAlphabetically,
setNoteToParent