Dockery/Trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-11-17 18:50:41 +01:00
Code Issues Packages Projects Releases Activity

preparing 0.59 without ocr/pdf, userguide, note ancillaries

Browse Source
This commit is contained in:
zadam
2023-02-17 14:49:45 +01:00
parent 42e08284b0
commit 6f7b554cdc
57 changed files with 813 additions and 2246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/becca/becca.js
View File

@@ -121,14 +121,6 @@ class Becca {
return row ? new BNoteRevision(row) : null;
}
/** @returns {BNoteAncillary|null} */
getNoteAncillary(noteAncillaryId) {
const row = sql.getRow("SELECT * FROM note_ancillaries WHERE noteAncillaryId = ?", [noteAncillaryId]);
const BNoteAncillary = require("./entities/bnote_ancillary"); // avoiding circular dependency problems
return row ? new BNoteAncillary(row) : null;
}
/** @returns {BOption|null} */
getOption(name) {
return this.options[name];
@@ -151,8 +143,6 @@ class Becca {
if (entityName === 'note_revisions') {
return this.getNoteRevision(entityId);
} else if (entityName === 'note_ancillaries') {
return this.getNoteAncillary(entityId);
}
const camelCaseEntityName = entityName.toLowerCase().replace(/(_[a-z])/g,

4
src/becca/entities/bbranch.js
View File

@@ -198,10 +198,6 @@ class BBranch extends AbstractBeccaEntity {
relation.markAsDeleted(deleteId);
}
for (const noteAncillary of note.getNoteAncillaries()) {
noteAncillary.markAsDeleted(deleteId);
}
note.markAsDeleted(deleteId);
return true;

43
src/becca/entities/bnote.js
View File

@@ -8,7 +8,6 @@ const dateUtils = require('../../services/date_utils');
const entityChangesService = require('../../services/entity_changes');
const AbstractBeccaEntity = require("./abstract_becca_entity");
const BNoteRevision = require("./bnote_revision");
const BNoteAncillary = require("./bnote_ancillary");
const TaskContext = require("../../services/task_context");
const dayjs = require("dayjs");
const utc = require('dayjs/plugin/utc');
@@ -19,7 +18,7 @@ const LABEL = 'label';
const RELATION = 'relation';
/**
* Trilium's main entity which can represent text note, image, code note, file ancillary etc.
* Trilium's main entity which can represent text note, image, code note, file attachment etc.
*
* @extends AbstractBeccaEntity
*/
@@ -337,7 +336,7 @@ class BNote extends AbstractBeccaEntity {
return this.mime === "application/json";
}
/** @returns {boolean} true if this note is JavaScript (code or ancillary) */
/** @returns {boolean} true if this note is JavaScript (code or attachment) */
isJavaScript() {
return (this.type === "code" || this.type === "file" || this.type === 'launcher')
&& (this.mime.startsWith("application/javascript")
@@ -1136,19 +1135,6 @@ class BNote extends AbstractBeccaEntity {
.map(row => new BNoteRevision(row));
}
/** @returns {BNoteAncillary[]} */
getNoteAncillaries() {
return sql.getRows("SELECT * FROM note_ancillaries WHERE noteId = ? AND isDeleted = 0", [this.noteId])
.map(row => new BNoteAncillary(row));
}
/** @returns {BNoteAncillary|undefined} */
getNoteAncillaryByName(name) {
return sql.getRows("SELECT * FROM note_ancillaries WHERE noteId = ? AND name = ? AND isDeleted = 0", [this.noteId, name])
.map(row => new BNoteAncillary(row))
[0];
}
/**
* @returns {string[][]} - array of notePaths (each represented by array of noteIds constituting the particular note path)
*/
@@ -1478,31 +1464,6 @@ class BNote extends AbstractBeccaEntity {
return noteRevision;
}
/**
* @returns {BNoteAncillary}
*/
saveNoteAncillary(name, mime, content) {
let noteAncillary = this.getNoteAncillaryByName(name);
if (noteAncillary
&& noteAncillary.mime === mime
&& noteAncillary.contentCheckSum === noteAncillary.calculateCheckSum(content)) {
return noteAncillary; // no change
}
noteAncillary = new BNoteAncillary({
noteId: this.noteId,
name,
mime,
isProtected: this.isProtected
});
noteAncillary.setContent(content);
return noteAncillary;
}
beforeSaving() {
super.beforeSaving();

161
src/becca/entities/bnote_ancillary.js
View File

@@ -1,161 +0,0 @@
"use strict";
const protectedSessionService = require('../../services/protected_session');
const utils = require('../../services/utils');
const sql = require('../../services/sql');
const dateUtils = require('../../services/date_utils');
const becca = require('../becca');
const entityChangesService = require('../../services/entity_changes');
const AbstractBeccaEntity = require("./abstract_becca_entity");
/**
* NoteAncillary represent data related/attached to the note. Conceptually similar to attributes, but intended for
* larger amounts of data and generally not accessible to the user.
*
* @extends AbstractBeccaEntity
*/
class BNoteAncillary extends AbstractBeccaEntity {
static get entityName() { return "note_ancillaries"; }
static get primaryKeyName() { return "noteAncillaryId"; }
static get hashedProperties() { return ["noteAncillaryId", "noteId", "name", "content", "utcDateModified"]; }
constructor(row) {
super();
if (!row.noteId) {
throw new Error("'noteId' must be given to initialize a NoteAncillary entity");
}
if (!row.name) {
throw new Error("'name' must be given to initialize a NoteAncillary entity");
}
/** @type {string} needs to be set at the initialization time since it's used in the .setContent() */
this.noteAncillaryId = row.noteAncillaryId || `${this.noteId}_${this.name}`;
/** @type {string} */
this.noteId = row.noteId;
/** @type {string} */
this.name = row.name;
/** @type {string} */
this.mime = row.mime;
/** @type {boolean} */
this.isProtected = !!row.isProtected;
/** @type {string} */
this.contentCheckSum = row.contentCheckSum;
/** @type {string} */
this.utcDateModified = row.utcDateModified;
}
getNote() {
return becca.notes[this.noteId];
}
/** @returns {boolean} true if the note has string content (not binary) */
isStringNote() {
return utils.isStringNote(this.type, this.mime);
}
/** @returns {*} */
getContent(silentNotFoundError = false) {
const res = sql.getRow(`SELECT content FROM note_ancillary_contents WHERE noteAncillaryId = ?`, [this.noteAncillaryId]);
if (!res) {
if (silentNotFoundError) {
return undefined;
}
else {
throw new Error(`Cannot find note ancillary content for noteAncillaryId=${this.noteAncillaryId}`);
}
}
let content = res.content;
if (this.isProtected) {
if (protectedSessionService.isProtectedSessionAvailable()) {
content = protectedSessionService.decrypt(content);
}
else {
content = "";
}
}
if (this.isStringNote()) {
return content === null
? ""
: content.toString("UTF-8");
}
else {
return content;
}
}
setContent(content) {
sql.transactional(() => {
this.contentCheckSum = this.calculateCheckSum(content);
this.save(); // also explicitly save note_ancillary to update contentCheckSum
const pojo = {
noteAncillaryId: this.noteAncillaryId,
content: content,
utcDateModified: dateUtils.utcNowDateTime()
};
if (this.isProtected) {
if (protectedSessionService.isProtectedSessionAvailable()) {
pojo.content = protectedSessionService.encrypt(pojo.content);
} else {
throw new Error(`Cannot update content of noteAncillaryId=${this.noteAncillaryId} since we're out of protected session.`);
}
}
sql.upsert("note_ancillary_contents", "noteAncillaryId", pojo);
entityChangesService.addEntityChange({
entityName: 'note_ancillary_contents',
entityId: this.noteAncillaryId,
hash: this.contentCheckSum,
isErased: false,
utcDateChanged: pojo.utcDateModified,
isSynced: true
});
});
}
calculateCheckSum(content) {
return utils.hash(`${this.noteAncillaryId}|${content.toString()}`);
}
beforeSaving() {
if (!this.name.match(/^[a-z0-9]+$/i)) {
throw new Error(`Name must be alphanumerical, "${this.name}" given.`);
}
this.noteAncillaryId = `${this.noteId}_${this.name}`;
super.beforeSaving();
this.utcDateModified = dateUtils.utcNowDateTime();
}
getPojo() {
return {
noteAncillaryId: this.noteAncillaryId,
noteId: this.noteId,
name: this.name,
mime: this.mime,
isProtected: !!this.isProtected,
contentCheckSum: this.contentCheckSum,
isDeleted: false,
utcDateModified: this.utcDateModified
};
}
getPojoToSave() {
const pojo = this.getPojo();
delete pojo.content; // not getting persisted
return pojo;
}
}
module.exports = BNoteAncillary;

3
src/becca/entity_constructor.js
View File

@@ -1,6 +1,5 @@
const BNote = require('./entities/bnote');
const BNoteRevision = require('./entities/bnote_revision');
const BNoteAncillary = require("./entities/bnote_ancillary");
const BBranch = require('./entities/bbranch');
const BAttribute = require('./entities/battribute');
const BRecentNote = require('./entities/brecent_note');
@@ -14,8 +13,6 @@ const ENTITY_NAME_TO_ENTITY = {
"note_contents": BNote,
"note_revisions": BNoteRevision,
"note_revision_contents": BNoteRevision,
"note_ancillaries": BNoteAncillary,
"note_ancillary_contents": BNoteAncillary,
"recent_notes": BRecentNote,
"etapi_tokens": BEtapiToken,
"options": BOption

8
src/public/app/components/root_command_executor.js
View File

@@ -124,12 +124,4 @@ export default class RootCommandExecutor extends Component {
await appContext.tabManager.openContextWithNote(notePath, { activate: true, viewMode: 'source' });
}
}
async showNoteAncillariesCommand() {
const notePath = appContext.tabManager.getActiveContextNotePath();
if (notePath) {
await appContext.tabManager.openContextWithNote(notePath, { activate: true, viewMode: 'ancillaries' });
}
}
}

2
src/public/app/entities/fnote.js
View File

@@ -803,7 +803,7 @@ class FNote {
return labels.length > 0 ? labels[0].value : "";
}
/** @returns {boolean} true if this note is JavaScript (code or ancillary) */
/** @returns {boolean} true if this note is JavaScript (code or file) */
isJavaScript() {
return (this.type === "code" || this.type === "file" || this.type === 'launcher')
&& (this.mime.startsWith("application/javascript")

2
src/public/app/services/froca_updater.js
View File

@@ -36,7 +36,7 @@ async function processEntityChanges(entityChanges) {
loadResults.addOption(ec.entity.name);
}
else if (['etapi_tokens', 'note_ancillaries', 'note_ancillary_contents'].includes(ec.entityName)) {
else if (['etapi_tokens'].includes(ec.entityName)) {
// NOOP
}
else {

1
src/public/app/widgets/buttons/note_actions.js
View File

@@ -28,7 +28,6 @@ const TPL = `
<a data-trigger-command="renderActiveNote" class="dropdown-item render-note-button"><kbd data-command="renderActiveNote"></kbd> Re-render note</a>
<a data-trigger-command="findInText" class="dropdown-item find-in-text-button">Search in note <kbd data-command="findInText"></a>
<a data-trigger-command="showNoteSource" class="dropdown-item show-source-button"><kbd data-command="showNoteSource"></kbd> Note source</a>
<a data-trigger-command="showNoteAncillaries" class="dropdown-item"><kbd data-command="showNoteAncillaries"></kbd> Note ancillaries</a>
<a data-trigger-command="openNoteExternally" class="dropdown-item open-note-externally-button"><kbd data-command="openNoteExternally"></kbd> Open note externally</a>
<a class="dropdown-item import-files-button">Import files</a>
<a class="dropdown-item export-note-button">Export note</a>

8
src/public/app/widgets/mermaid.js
View File

@@ -78,14 +78,6 @@ export default class MermaidWidget extends NoteContextAwareWidget {
await this.renderSvg(async renderedSvg => {
this.$display.html(renderedSvg);
// not awaiting intentionally
// this is pretty hacky since we update ancillary on render
// but if nothing changed this should not trigger DB write and sync
server.put(`notes/${note.noteId}/ancillaries/mermaidSvg`, {
mime: 'image/svg+xml',
content: renderedSvg
});
await wheelZoomLoaded;
this.$display.attr("id", `mermaid-render-${idCounter}`);

6
src/public/app/widgets/note_detail.js
View File

@@ -27,7 +27,6 @@ import NoteMapTypeWidget from "./type_widgets/note_map.js";
import WebViewTypeWidget from "./type_widgets/web_view.js";
import DocTypeWidget from "./type_widgets/doc.js";
import ContentWidgetTypeWidget from "./type_widgets/content_widget.js";
import AncillariesTypeWidget from "./type_widgets/ancillaries.js";
const TPL = `
<div class="note-detail">
@@ -62,8 +61,7 @@ const typeWidgetClasses = {
'noteMap': NoteMapTypeWidget,
'webView': WebViewTypeWidget,
'doc': DocTypeWidget,
'contentWidget': ContentWidgetTypeWidget,
'ancillaries': AncillariesTypeWidget
'contentWidget': ContentWidgetTypeWidget
};
export default class NoteDetailWidget extends NoteContextAwareWidget {
@@ -191,8 +189,6 @@ export default class NoteDetailWidget extends NoteContextAwareWidget {
if (type === 'text' && this.noteContext.viewScope.viewMode === 'source') {
type = 'readOnlyCode';
} else if (this.noteContext.viewScope.viewMode === 'ancillaries') {
type = 'ancillaries';
} else if (type === 'text' && await this.noteContext.isReadOnly()) {
type = 'readOnlyText';
} else if ((type === 'code' || type === 'mermaid') && await this.noteContext.isReadOnly()) {

79
src/public/app/widgets/type_widgets/ancillaries.js
View File

@@ -1,79 +0,0 @@
import TypeWidget from "./type_widget.js";
import server from "../../services/server.js";
const TPL = `
<div class="note-ancillaries note-detail-printable">
<style>
.note-ancillaries {
padding: 15px;
}
.ancillary-content {
max-height: 400px;
background: var(--accented-background-color);
padding: 10px;
margin-top: 10px;
margin-bottom: 10px;
}
.ancillary-details th {
padding-left: 10px;
padding-right: 10px;
}
</style>
<div class="alert alert-info" style="margin: 10px 0 10px 0; padding: 20px;">
Note ancillaries are pieces of data attached to a given note, providing ancillary support.
This view is useful for diagnostics.
</div>
<div class="note-ancillary-list"></div>
</div>`;
export default class AncillariesTypeWidget extends TypeWidget {
static getType() { return "ancillaries"; }
doRender() {
this.$widget = $(TPL);
this.$list = this.$widget.find('.note-ancillary-list');
super.doRender();
}
async doRefresh(note) {
this.$list.empty();
const ancillaries = await server.get(`notes/${this.noteId}/ancillaries?includeContent=true`);
if (ancillaries.length === 0) {
this.$list.html("<strong>This note has no ancillaries.</strong>");
return;
}
for (const ancillary of ancillaries) {
this.$list.append(
$('<div class="note-ancillary-wrapper">')
.append(
$('<h4>').append($('<span class="ancillary-name">').text(ancillary.name))
)
.append(
$('<table class="ancillary-details">')
.append(
$('<tr>')
.append($('<th>').text('Length:'))
.append($('<td>').text(ancillary.contentLength))
.append($('<th>').text('MIME:'))
.append($('<td>').text(ancillary.mime))
.append($('<th>').text('Date modified:'))
.append($('<td>').text(ancillary.utcDateModified))
)
)
.append(
$('<pre class="ancillary-content">')
.text(ancillary.content)
)
);
}
}
}

17
src/public/app/widgets/type_widgets/canvas.js
View File

@@ -277,20 +277,15 @@ export default class ExcalidrawTypeWidget extends TypeWidget {
})
const content = {
elements,
appState,
files: activeFiles
_meta: "This note has type `canvas`. It uses excalidraw and stores an exported svg alongside.",
elements, // excalidraw
appState, // excalidraw
files: activeFiles, // excalidraw
svg: svgString, // not needed for excalidraw, used for note_short, content, and image api
};
return {
content: JSON.stringify(content),
ancillaries: [
{
name: 'canvasSvg',
mime: 'image/svg+xml',
content: svgString
}
]
content: JSON.stringify(content)
};
}

4
src/public/app/widgets/type_widgets/content_widget.js
View File

@@ -28,8 +28,6 @@ import ConsistencyChecksOptions from "./options/advanced/consistency_checks.js";
import VacuumDatabaseOptions from "./options/advanced/vacuum_database.js";
import DatabaseAnonymizationOptions from "./options/advanced/database_anonymization.js";
import BackendLogWidget from "./content/backend_log.js";
import OcrOptions from "./options/images/ocr.js";
import ExtractTextFromPdfOptions from "./options/images/extract_text_from_pdf.js";
const TPL = `<div class="note-detail-content-widget note-detail-printable">
<style>
@@ -70,7 +68,7 @@ const CONTENT_WIDGETS = {
CodeAutoReadOnlySizeOptions,
CodeMimeTypesOptions
],
_optionsImages: [ ImageOptions, OcrOptions, ExtractTextFromPdfOptions ],
_optionsImages: [ ImageOptions ],
_optionsSpellcheck: [ SpellcheckOptions ],
_optionsPassword: [ PasswordOptions ],
_optionsEtapi: [ EtapiOptions ],

28
src/public/app/widgets/type_widgets/options/images/extract_text_from_pdf.js
View File

@@ -1,28 +0,0 @@
import OptionsWidget from "../options_widget.js";
const TPL = `
<div class="options-section">
<h4>Extract text from PDF files</h4>
<label>
<input class="extract-text-from-pdf" type="checkbox">
Extract text from PDF
</label>
<p>Text extracted from PDFs will be considered when fulltext searching.</p>
</div>
`;
export default class ExtractTextFromPdfOptions extends OptionsWidget {
doRender() {
this.$widget = $(TPL);
this.$extractTextFromPdf = this.$widget.find(".extract-text-from-pdf");
this.$extractTextFromPdf.on("change", () =>
this.updateCheckboxOption('extractTextFromPdf', this.$extractTextFromPdf));
}
optionsLoaded(options) {
this.setCheckboxState(this.$extractTextFromPdf, options.extractTextFromPdf);
}
}

28
src/public/app/widgets/type_widgets/options/images/ocr.js
View File

@@ -1,28 +0,0 @@
import OptionsWidget from "../options_widget.js";
const TPL = `
<div class="options-section">
<h4>OCR</h4>
<label>
<input class="ocr-images" type="checkbox">
Extract text from images using OCR
</label>
<p>Text extracted from images will be considered when fulltext searching.</p>
</div>
`;
export default class OcrOptions extends OptionsWidget {
doRender() {
this.$widget = $(TPL);
this.$ocrImages = this.$widget.find(".ocr-images");
this.$ocrImages.on("change", () =>
this.updateCheckboxOption('ocrImages', this.$ocrImages));
}
optionsLoaded(options) {
this.setCheckboxState(this.$ocrImages, options.ocrImages);
}
}

1
src/public/app/widgets/type_widgets/relation_map.js
View File

@@ -587,7 +587,6 @@ export default class RelationMapTypeWidget extends TypeWidget {
}
getData() {
// TODO: save also image as ancillary
return {
content: JSON.stringify(this.mapData)
};

2
src/public/app/widgets/type_widgets/type_widget.js
View File

@@ -39,7 +39,7 @@ export default class TypeWidget extends NoteContextAwareWidget {
}
/**
* @returns {Promise<Object>|*} promise resolving note data. Note data is an object with content and ancillaries.
* @returns {Promise<Object>|*} promise resolving note data. Note data is an object with content.
*/
getData() {}

51
src/routes/api/notes.js
View File

@@ -54,10 +54,10 @@ function createNote(req) {
}
function updateNoteData(req) {
const {content, ancillaries} = req.body;
const {content} = req.body;
const {noteId} = req.params;
return noteService.updateNoteData(noteId, content, ancillaries);
return noteService.updateNoteData(noteId, content);
}
function deleteNote(req) {
@@ -127,49 +127,6 @@ function setNoteTypeMime(req) {
note.save();
}
function getNoteAncillaries(req) {
const includeContent = req.query.includeContent === 'true';
const {noteId} = req.params;
const note = becca.getNote(noteId);
if (!note) {
throw new NotFoundError(`Note '${noteId}' doesn't exist.`);
}
const noteAncillaries = note.getNoteAncillaries();
return noteAncillaries.map(ancillary => {
const pojo = ancillary.getPojo();
if (includeContent && utils.isStringNote(null, ancillary.mime)) {
pojo.content = ancillary.getContent()?.toString();
pojo.contentLength = pojo.content.length;
const MAX_ANCILLARY_LENGTH = 1_000_000;
if (pojo.content.length > MAX_ANCILLARY_LENGTH) {
pojo.content = pojo.content.substring(0, MAX_ANCILLARY_LENGTH);
}
}
return pojo;
});
}
function saveNoteAncillary(req) {
const {noteId, name} = req.params;
const {mime, content} = req.body;
const note = becca.getNote(noteId);
if (!note) {
throw new NotFoundError(`Note '${noteId}' doesn't exist.`);
}
note.saveNoteAncillary(name, mime, content);
}
function getRelationMap(req) {
const {relationMapNoteId, noteIds} = req.body;
@@ -383,7 +340,5 @@ module.exports = {
eraseDeletedNotesNow,
getDeleteNotesPreview,
uploadModifiedFile,
forceSaveNoteRevision,
getNoteAncillaries,
saveNoteAncillary
forceSaveNoteRevision
};

4
src/routes/api/options.js
View File

@@ -61,9 +61,7 @@ const ALLOWED_OPTIONS = new Set([
'downloadImagesAutomatically',
'minTocHeadings',
'checkForUpdates',
'disableTray',
'ocrImages',
'extractTextFromPdf'
'disableTray'
]);
function getOptions() {

8
src/routes/api/sync.js
View File

@@ -114,14 +114,6 @@ function forceNoteSync(req) {
entityChangesService.moveEntityChangeToTop('note_revision_contents', noteRevisionId);
}
for (const noteAncillaryId of sql.getColumn("SELECT noteAncillaryId FROM note_ancillaries WHERE noteId = ?", [noteId])) {
sql.execute(`UPDATE note_ancillaries SET utcDateModified = ? WHERE noteAncillaryId = ?`, [now, noteAncillaryId]);
entityChangesService.moveEntityChangeToTop('note_ancillaries', noteAncillaryId);
sql.execute(`UPDATE note_ancillary_contents SET utcDateModified = ? WHERE noteAncillaryId = ?`, [now, noteAncillaryId]);
entityChangesService.moveEntityChangeToTop('note_ancillary_contents', noteAncillaryId);
}
log.info(`Forcing note sync for ${noteId}`);
// not awaiting for the job to finish (will probably take a long time)

2
src/routes/routes.js
View File

@@ -126,8 +126,6 @@ function register(app) {
apiRoute(PUT, '/api/notes/:noteId/sort-children', notesApiRoute.sortChildNotes);
apiRoute(PUT, '/api/notes/:noteId/protect/:isProtected', notesApiRoute.protectNote);
apiRoute(PUT, '/api/notes/:noteId/type', notesApiRoute.setNoteTypeMime);
apiRoute(GET, '/api/notes/:noteId/ancillaries', notesApiRoute.getNoteAncillaries);
apiRoute(PUT, '/api/notes/:noteId/ancillaries/:name', notesApiRoute.saveNoteAncillary);
apiRoute(GET, '/api/notes/:noteId/revisions', noteRevisionsApiRoute.getNoteRevisions);
apiRoute(DELETE, '/api/notes/:noteId/revisions', noteRevisionsApiRoute.eraseAllNoteRevisions);
apiRoute(GET, '/api/notes/:noteId/revisions/:noteRevisionId', noteRevisionsApiRoute.getNoteRevision);

4
src/services/app_info.js
View File

@@ -4,8 +4,8 @@ const build = require('./build');
const packageJson = require('../../package');
const {TRILIUM_DATA_DIR} = require('./data_dir');
const APP_DB_VERSION = 214;
const SYNC_VERSION = 30;
const APP_DB_VERSION = 212;
const SYNC_VERSION = 29;
const CLIPPER_PROTOCOL_VERSION = "1.0";
module.exports = {

10
src/services/cls.js
View File

@@ -48,14 +48,6 @@ function isEntityEventsDisabled() {
return !!namespace.get('disableEntityEvents');
}
function isOcrDisabled() {
return !!namespace.get('disableOcr');
}
function disableOcr() {
namespace.set('disableOcr', true);
}
function getAndClearEntityChangeIds() {
const entityChangeIds = namespace.get('entityChangeIds') || [];
@@ -101,6 +93,4 @@ module.exports = {
getAndClearEntityChangeIds,
addEntityChange,
ignoreEntityChangeIds,
isOcrDisabled,
disableOcr
};

43
src/services/consistency_checks.js
View File

@@ -213,25 +213,6 @@ class ConsistencyChecks {
logError(`Relation '${attributeId}' references missing note '${noteId}'`)
}
});
this.findAndFixIssues(`
SELECT noteAncillaryId, note_ancillaries.noteId AS noteId
FROM note_ancillaries
LEFT JOIN notes USING (noteId)
WHERE notes.noteId IS NULL
AND note_ancillaries.isDeleted = 0`,
({noteAncillaryId, noteId}) => {
if (this.autoFix) {
const noteAncillary = becca.getNoteAncillary(noteAncillaryId);
noteAncillary.markAsDeleted();
this.reloadNeeded = false;
logFix(`Note ancillary '${noteAncillaryId}' has been deleted since it references missing note '${noteId}'`);
} else {
logError(`Note ancillary '${noteAncillaryId}' references missing note '${noteId}'`);
}
});
}
findExistencyIssues() {
@@ -339,26 +320,6 @@ class ConsistencyChecks {
logError(`Duplicate branches for note '${noteId}' and parent '${parentNoteId}'`);
}
});
this.findAndFixIssues(`
SELECT noteAncillaryId,
note_ancillaries.noteId AS noteId
FROM note_ancillaries
JOIN notes USING (noteId)
WHERE notes.isDeleted = 1
AND note_ancillaries.isDeleted = 0`,
({noteAncillaryId, noteId}) => {
if (this.autoFix) {
const noteAncillary = becca.getNoteAncillary(noteAncillaryId);
noteAncillary.markAsDeleted();
this.reloadNeeded = false;
logFix(`Note ancillary '${noteAncillaryId}' has been deleted since associated note '${noteId}' is deleted.`);
} else {
logError(`Note ancillary '${noteAncillaryId}' is not deleted even though associated note '${noteId}' is deleted.`)
}
});
}
findLogicIssues() {
@@ -659,8 +620,6 @@ class ConsistencyChecks {
this.runEntityChangeChecks("note_contents", "noteId");
this.runEntityChangeChecks("note_revisions", "noteRevisionId");
this.runEntityChangeChecks("note_revision_contents", "noteRevisionId");
this.runEntityChangeChecks("note_ancillaries", "noteAncillaryId");
this.runEntityChangeChecks("note_ancillary_contents", "noteAncillaryId");
this.runEntityChangeChecks("branches", "branchId");
this.runEntityChangeChecks("attributes", "attributeId");
this.runEntityChangeChecks("etapi_tokens", "etapiTokenId");
@@ -756,7 +715,7 @@ class ConsistencyChecks {
return `${tableName}: ${count}`;
}
const tables = [ "notes", "note_revisions", "note_ancillaries", "branches", "attributes", "etapi_tokens" ];
const tables = [ "notes", "note_revisions", "branches", "attributes", "etapi_tokens" ];
log.info(`Table counts: ${tables.map(tableName => getTableRowCount(tableName)).join(", ")}`);
}

2
src/services/entity_changes.js
View File

@@ -151,8 +151,6 @@ function fillAllEntityChanges() {
fillEntityChanges("branches", "branchId");
fillEntityChanges("note_revisions", "noteRevisionId");
fillEntityChanges("note_revision_contents", "noteRevisionId");
fillEntityChanges("note_ancillaries", "noteAncillaryId");
fillEntityChanges("note_ancillary_contents", "noteAncillaryId");
fillEntityChanges("attributes", "attributeId");
fillEntityChanges("etapi_tokens", "etapiTokenId");
fillEntityChanges("options", "name", 'isSynced = 1');

39
src/services/export/zip.js
View File

@@ -170,24 +170,6 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
meta.dataFileName = getDataFileName(note.type, note.mime, baseFileName, existingFileNames);
}
const ancillaries = note.getNoteAncillaries();
if (ancillaries.length > 0) {
meta.ancillaries = ancillaries
.filter(ancillary => ["canvasSvg", "mermaidSvg"].includes(ancillary.name))
.map(ancillary => ({
name: ancillary.name,
mime: ancillary.mime,
dataFileName: getDataFileName(
null,
ancillary.mime,
baseFileName + "_" + ancillary.name,
existingFileNames
)
}));
}
if (childBranches.length > 0) {
meta.dirFileName = getUniqueFilename(existingFileNames, baseFileName);
meta.children = [];
@@ -234,15 +216,8 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
const meta = noteIdToMeta[targetPath[targetPath.length - 1]];
// for some note types it's more user-friendly to see the ancillary (if exists) instead of source note
const preferredAncillary = (meta.ancillaries || []).find(ancillary => ['mermaidSvg', 'canvasSvg'].includes(ancillary.name));
if (preferredAncillary) {
url += encodeURIComponent(preferredAncillary.dataFileName);
} else {
// link can target note which is only "folder-note" and as such will not have a file in an export
url += encodeURIComponent(meta.dataFileName || meta.dirFileName);
}
// link can target note which is only "folder-note" and as such will not have a file in an export
url += encodeURIComponent(meta.dataFileName || meta.dirFileName);
return url;
}
@@ -344,16 +319,6 @@ ${markdownContent}`;
taskContext.increaseProgressCount();
for (const ancillaryMeta of noteMeta.ancillaries || []) {
const noteAncillary = note.getNoteAncillaryByName(ancillaryMeta.name);
const content = noteAncillary.getContent();
archive.append(content, {
name: filePathPrefix + ancillaryMeta.dataFileName,
date: dateUtils.parseDateTime(note.utcDateModified)
});
}
if (noteMeta.children && noteMeta.children.length > 0) {
const directoryPath = filePathPrefix + noteMeta.dirFileName;

5
src/services/image.js
View File

@@ -12,7 +12,6 @@ const sanitizeFilename = require('sanitize-filename');
const isSvg = require('is-svg');
const isAnimated = require('is-animated');
const htmlSanitizer = require("./html_sanitizer");
const textExtractingService = require("./text_extracting");
async function processImage(uploadBuffer, originalName, shrinkImageSwitch) {
const compressImages = optionService.getOptionBool("compressImages");
@@ -83,8 +82,6 @@ function updateImage(noteId, uploadBuffer, originalName) {
note.setContent(buffer);
});
runOcr(note, buffer);
});
}
@@ -126,8 +123,6 @@ function saveImage(parentNoteId, uploadBuffer, originalName, shrinkImageSwitch,
note.setContent(buffer);
});
textExtractingService.runOcr(note, buffer);
});
return {

34
src/services/import/zip.js
View File

@@ -14,7 +14,6 @@ const treeService = require("../tree");
const yauzl = require("yauzl");
const htmlSanitizer = require('../html_sanitizer');
const becca = require("../../becca/becca");
const BNoteAncillary = require("../../becca/entities/bnote_ancillary");
/**
* @param {TaskContext} taskContext
@@ -65,7 +64,6 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
};
let parent;
let ancillaryMeta = false;
for (const segment of pathSegments) {
if (!cursor || !cursor.children || cursor.children.length === 0) {
@@ -74,28 +72,11 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
parent = cursor;
cursor = parent.children.find(file => file.dataFileName === segment || file.dirFileName === segment);
if (!cursor) {
for (const file of parent.children) {
for (const ancillary of file.ancillaries || []) {
if (ancillary.dataFileName === segment) {
cursor = file;
ancillaryMeta = ancillary;
break;
}
}
if (cursor) {
break;
}
}
}
}
return {
parentNoteMeta: parent,
noteMeta: cursor,
ancillaryMeta
noteMeta: cursor
};
}
@@ -373,7 +354,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
}
function saveNote(filePath, content) {
const {parentNoteMeta, noteMeta, ancillaryMeta} = getMeta(filePath);
const {parentNoteMeta, noteMeta} = getMeta(filePath);
if (noteMeta?.noImport) {
return;
@@ -381,17 +362,6 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
const noteId = getNoteId(noteMeta, filePath);
if (ancillaryMeta) {
const noteAncillary = new BNoteAncillary({
noteId,
name: ancillaryMeta.name,
mime: ancillaryMeta.mime
});
noteAncillary.setContent(content);
return;
}
const parentNoteId = getParentNoteId(filePath, parentNoteMeta);
if (!parentNoteId) {

37
src/services/note_ancillaries.js
View File

@@ -1,37 +0,0 @@
const protectedSession = require("./protected_session");
const log = require("./log");
/**
* @param {BNote} note
*/
function protectNoteAncillaries(note) {
for (const noteAncillary of note.getNoteAncillaries()) {
if (note.isProtected !== noteAncillary.isProtected) {
if (!protectedSession.isProtectedSessionAvailable()) {
log.error("Protected session is not available to fix note ancillaries.");
return;
}
try {
const content = noteAncillary.getContent();
noteAncillary.isProtected = note.isProtected;
// this will force de/encryption
noteAncillary.setContent(content);
noteAncillary.save();
}
catch (e) {
log.error(`Could not un/protect note ancillary ID = ${noteAncillary.noteAncillaryId}`);
throw e;
}
}
}
}
module.exports = {
protectNoteAncillaries
}

53
src/services/notes.js
View File

@@ -9,7 +9,6 @@ const protectedSessionService = require('../services/protected_session');
const log = require('../services/log');
const utils = require('../services/utils');
const noteRevisionService = require('../services/note_revisions');
const noteAncillarieservice = require('../services/note_ancillaries');
const attributeService = require('../services/attributes');
const request = require('./request');
const path = require('path');
@@ -18,12 +17,10 @@ const becca = require('../becca/becca');
const BBranch = require('../becca/entities/bbranch');
const BNote = require('../becca/entities/bnote');
const BAttribute = require('../becca/entities/battribute');
const BNoteAncillary = require("../becca/entities/bnote_ancillary");
const dayjs = require("dayjs");
const htmlSanitizer = require("./html_sanitizer");
const ValidationError = require("../errors/validation_error");
const noteTypesService = require("./note_types");
const textExtractingService = require("./text_extracting");
function getNewNotePosition(parentNoteId) {
const note = becca.notes[parentNoteId];
@@ -302,7 +299,6 @@ function protectNote(note, protect) {
}
noteRevisionService.protectNoteRevisions(note);
noteAncillarieservice.protectNoteAncillaries(note);
}
catch (e) {
log.error(`Could not un/protect note ID = ${note.noteId}`);
@@ -593,7 +589,7 @@ function saveNoteRevisionIfNeeded(note) {
}
}
function updateNoteData(noteId, content, ancillaries = []) {
function updateNoteData(noteId, content) {
const note = becca.getNote(noteId);
if (!note.isContentAvailable()) {
@@ -605,10 +601,6 @@ function updateNoteData(noteId, content, ancillaries = []) {
content = saveLinks(note, content);
note.setContent(content);
for (const {name, mime, content} of ancillaries) {
note.saveNoteAncillary(name, mime, content);
}
}
/**
@@ -675,16 +667,6 @@ function undeleteBranch(branchId, deleteId, taskContext) {
new BAttribute(attribute).save({skipValidation: true});
}
const noteAncillaries = sql.getRows(`
SELECT * FROM note_ancillaries
WHERE isDeleted = 1
AND deleteId = ?
AND noteId = ?`, [deleteId, note.noteId]);
for (const noteAncillary of noteAncillaries) {
new BNoteAncillary(noteAncillary).save();
}
const childBranchIds = sql.getColumn(`
SELECT branches.branchId
FROM branches
@@ -734,8 +716,6 @@ function scanForLinks(note, content) {
*/
async function asyncPostProcessContent(note, content) {
scanForLinks(note, content);
await textExtractingService.runOcr(note, content);
await textExtractingService.extractTextFromPdf(note, content);
}
function eraseNotes(noteIdsToErase) {
@@ -765,11 +745,6 @@ function eraseNotes(noteIdsToErase) {
noteRevisionService.eraseNoteRevisions(noteRevisionIdsToErase);
const noteAncillaryIdsToErase = sql.getManyRows(`SELECT noteAncillaryId FROM note_ancillaries WHERE noteId IN (???)`, noteIdsToErase)
.map(row => row.noteAncillaryId);
eraseNoteAncillaries(noteAncillaryIdsToErase);
log.info(`Erased notes: ${JSON.stringify(noteIdsToErase)}`);
}
@@ -805,20 +780,6 @@ function eraseAttributes(attributeIdsToErase) {
log.info(`Erased attributes: ${JSON.stringify(attributeIdsToErase)}`);
}
function eraseNoteAncillaries(noteAncillaryIdsToErase) {
if (noteAncillaryIdsToErase.length === 0) {
return;
}
log.info(`Removing note ancillaries: ${JSON.stringify(noteAncillaryIdsToErase)}`);
sql.executeMany(`DELETE FROM note_ancillaries WHERE noteAncillaryId IN (???)`, noteAncillaryIdsToErase);
sql.executeMany(`UPDATE entity_changes SET isErased = 1 WHERE entityName = 'note_ancillaries' AND entityId IN (???)`, noteAncillaryIdsToErase);
sql.executeMany(`DELETE FROM note_ancillary_contents WHERE noteAncillaryId IN (???)`, noteAncillaryIdsToErase);
sql.executeMany(`UPDATE entity_changes SET isErased = 1 WHERE entityName = 'note_ancillary_contents' AND entityId IN (???)`, noteAncillaryIdsToErase);
}
function eraseDeletedEntities(eraseEntitiesAfterTimeInSeconds = null) {
// this is important also so that the erased entity changes are sent to the connected clients
sql.transactional(() => {
@@ -953,18 +914,6 @@ function duplicateSubtreeInner(origNote, origBranch, newParentNoteId, noteIdMapp
attr.save();
}
for (const noteAncillary of origNote.getNoteAncillaries()) {
const duplNoteAncillary = new BNoteAncillary({
...noteAncillary,
noteAncillaryId: undefined,
noteId: newNote.noteId
});
duplNoteAncillary.save();
duplNoteAncillary.setContent(noteAncillary.getContent());
}
for (const childBranch of origNote.getChildBranches()) {
duplicateSubtreeInner(childBranch.getNote(), childBranch, newNote.noteId, noteIdMapping);
}

2
src/services/options_init.js
View File

@@ -90,8 +90,6 @@ const defaultOptions = [
{ name: 'checkForUpdates', value: 'true', isSynced: true },
{ name: 'disableTray', value: 'false', isSynced: false },
{ name: 'userGuideSha256Hash', value: '', isSynced: true },
{ name: 'ocrImages', value: 'true', isSynced: true },
{ name: 'extractTextFromPdf', value: 'true', isSynced: true },
];
function initStartupOptions() {

3
src/services/scheduler.js
View File

@@ -7,7 +7,6 @@ const sql = require("./sql");
const becca = require("../becca/becca");
const protectedSessionService = require("../services/protected_session");
const hiddenSubtreeService = require("./hidden_subtree");
const helpImportService = require("./user_guide_import");
function getRunAtHours(note) {
try {
@@ -54,8 +53,6 @@ function runNotesWithLabel(runAttrValue) {
sqlInit.dbReady.then(() => {
cls.init(() => {
hiddenSubtreeService.checkHiddenSubtree();
helpImportService.importUserGuideIfNeeded();
});
if (!process.env.TRILIUM_SAFE_MODE) {

10
src/services/search/expressions/note_content_fulltext.js
View File

@@ -48,16 +48,6 @@ class NoteContentFulltextExp extends Expression {
this.findInText(row, inputNoteSet, resultNoteSet);
}
for (const row of sql.iterateRows(`
SELECT noteId, 'plainText' as type, mime, content, isProtected
FROM note_ancillaries JOIN note_ancillary_contents USING (noteAncillaryId)
WHERE name IN ('plainText') AND isDeleted = 0`)) {
if (!resultNoteSet.hasNoteId(row.noteId)) {
this.findInText(row, inputNoteSet, resultNoteSet);
}
}
return resultNoteSet;
}

2
src/services/sync.js
View File

@@ -321,7 +321,7 @@ function getEntityChangeRow(entityName, entityId) {
throw new Error(`Entity ${entityName} ${entityId} not found.`);
}
if (['note_contents', 'note_revision_contents', 'note_ancillary_contents'].includes(entityName) && entity.content !== null) {
if (['note_contents', 'note_revision_contents'].includes(entityName) && entity.content !== null) {
if (typeof entity.content === 'string') {
entity.content = Buffer.from(entity.content, 'UTF-8');
}

6
src/services/sync_update.js
View File

@@ -64,7 +64,7 @@ function updateNormalEntity(remoteEntityChange, remoteEntityRow, instanceId) {
|| localEntityChange.utcDateChanged < remoteEntityChange.utcDateChanged
|| localEntityChange.hash !== remoteEntityChange.hash // sync error, we should still update
) {
if (['note_contents', 'note_revision_contents', 'note_ancillary_contents'].includes(remoteEntityChange.entityName)) {
if (['note_contents', 'note_revision_contents'].includes(remoteEntityChange.entityName)) {
remoteEntityRow.content = handleContent(remoteEntityRow.content);
}
@@ -115,9 +115,7 @@ function eraseEntity(entityChange, instanceId) {
"branches",
"attributes",
"note_revisions",
"note_revision_contents",
"note_ancillaries",
"note_ancillary_contents"
"note_revision_contents"
];
if (!entityNames.includes(entityName)) {

150
src/services/text_extracting.js
View File

@@ -1,150 +0,0 @@
const Canvas = require("canvas");
const OCRAD = require("ocrad.js");
const log = require("./log");
const optionService = require("./options");
const cls = require("./cls");
function ocrFromByteArray(img) {
// byte array contains raw uncompressed pixel data
// kind: 1 - GRAYSCALE_1BPP (unsupported)
// kind: 2 - RGB_24BPP
// kind: 3 - RGBA_32BPP
if (!(img.data instanceof Uint8ClampedArray) || ![2, 3].includes(img.kind)) {
return null;
}
const start = Date.now();
const canvas = new Canvas.createCanvas(img.width, img.height);
const ctx = canvas.getContext('2d');
const imageData = ctx.createImageData(img.width, img.height);
const imageBytes = imageData.data;
for (let j = 0, k = 0, jj = img.width * img.height * 4; j < jj;) {
imageBytes[j++] = img.data[k++];
imageBytes[j++] = img.data[k++];
imageBytes[j++] = img.data[k++];
// in case of kind = 2, the alpha channel is missing in source pixels and we'll add it
imageBytes[j++] = img.kind === 2 ? 255 : img.data[k++];
}
ctx.putImageData(imageData, 0, 0);
const text = OCRAD(canvas);
log.info(`OCR of ${img.data.length} canvas into ${text.length} chars of text took ${Date.now() - start}ms`);
return text;
}
async function ocrTextFromPdfImages(pdfjsLib, page, strings) {
const ops = await page.getOperatorList();
const fns = ops.fnArray;
const args = ops.argsArray;
for (const arg of args) {
const i = args.indexOf(arg);
if (fns[i] !== pdfjsLib.OPS.paintXObject && fns[i] !== pdfjsLib.OPS.paintImageXObject) {
continue;
}
const imgKey = arg[0];
const img = await new Promise((res) => page.objs.get(imgKey, r => res(r)));
if (!img) {
continue;
}
const text = ocrFromByteArray(img);
if (text) {
strings.push(text);
}
}
}
async function extractTextFromPdf(note, buffer) {
if (note.mime !== 'application/pdf' || !optionService.getOptionBool('extractTextFromPdf')) {
return;
}
try {
const pdfjsLib = require("pdfjs-dist");
const doc = await pdfjsLib.getDocument({data: buffer}).promise;
let strings = [];
for (let p = 1; p <= doc.numPages; p++) {
const page = await doc.getPage(p);
const content = await page.getTextContent({
normalizeWhitespace: true,
disableCombineTextItems: false
});
content.items.forEach(({str}) => strings.push(str));
try {
if (optionService.getOptionBool('ocrImages') && !cls.isOcrDisabled()) {
await ocrTextFromPdfImages(pdfjsLib, page, strings);
}
}
catch (e) {
log.info(`Could not OCR images from PDF note '${note.noteId}': '${e.message}', stack '${e.stack}'`);
}
}
strings = strings.filter(str => str?.trim());
note.saveNoteAncillary('plainText', 'text/plain', strings.join(" "));
}
catch (e) {
log.info(`Extracting text from PDF on note '${note.noteId}' failed with error '${e.message}', stack ${e.stack}`);
}
}
async function ocrTextFromBuffer(buffer) {
// buffer is expected to contain an image in JPEG, PNG etc.
const start = Date.now();
const img = await new Promise((res, rej) => {
const img = new Canvas.Image();
img.onload = () => res(img);
img.onerror = err => rej(new Error("Can't load the image " + err));
img.src = buffer;
});
const canvas = new Canvas.createCanvas(img.width, img.height);
const ctx = canvas.getContext('2d');
ctx.drawImage(img, 0, 0, img.width, img.height);
const plainText = OCRAD(canvas);
log.info(`OCR of ${buffer.byteLength} image bytes into ${plainText.length} chars of text took ${Date.now() - start}ms`);
return plainText;
}
async function runOcr(note, buffer) {
if (!note.isImage()
|| !optionService.getOptionBool('ocrImages')
|| cls.isOcrDisabled()
|| buffer.length === 0
) {
return;
}
try {
const plainText = await ocrTextFromBuffer(buffer);
note.saveNoteAncillary('plainText', 'text/plain', plainText);
}
catch (e) {
log.error(`OCR on note '${note.noteId}' failed with error '${e.message}', stack ${e.stack}`);
}
}
module.exports = {
runOcr,
extractTextFromPdf
};

498
src/services/user_guide_import.js
View File

@@ -1,498 +0,0 @@
"use strict"
const becca = require("../becca/becca");
const fs = require("fs").promises;
const BAttribute = require('../becca/entities/battribute');
const utils = require('./utils');
const log = require('./log');
const noteService = require('./notes');
const attributeService = require('./attributes');
const BBranch = require('../becca/entities/bbranch');
const path = require('path');
const yauzl = require("yauzl");
const htmlSanitizer = require('./html_sanitizer');
const sql = require('./sql');
const options = require('./options');
const cls = require('./cls');
const {USER_GUIDE_ZIP_DIR} = require('./resource_dir');
async function importUserGuideIfNeeded() {
const userGuideSha256HashInDb = options.getOption('userGuideSha256Hash');
let userGuideSha256HashInFile = await fs.readFile(USER_GUIDE_ZIP_DIR + "/user-guide.zip.sha256");
if (!userGuideSha256HashInFile || userGuideSha256HashInFile.byteLength < 64) {
return;
}
userGuideSha256HashInFile = userGuideSha256HashInFile.toString().substr(0, 64);
if (userGuideSha256HashInDb === userGuideSha256HashInFile) {
// user guide ZIP file has been already imported and is up-to-date
return;
}
const hiddenRoot = becca.getNote("_hidden");
const data = await fs.readFile(USER_GUIDE_ZIP_DIR + "/user-guide.zip", "binary");
cls.disableOcr(); // no OCR needed for user guide images
await importZip(Buffer.from(data, 'binary'), hiddenRoot);
options.setOption('userGuideSha256Hash', userGuideSha256HashInFile);
}
async function importZip(fileBuffer, importRootNote) {
// maps from original noteId (in ZIP file) to newly generated noteId
const noteIdMap = {};
const attributes = [];
let metaFile = null;
function getNewNoteId(origNoteId) {
if (origNoteId === 'root' || origNoteId.startsWith("_")) {
// these "named" noteIds don't differ between Trilium instances
return origNoteId;
}
if (!noteIdMap[origNoteId]) {
noteIdMap[origNoteId] = utils.newEntityId();
}
return noteIdMap[origNoteId];
}
function getMeta(filePath) {
const pathSegments = filePath.split(/[\/\\]/g);
let cursor = {
isImportRoot: true,
children: metaFile.files
};
let parent;
for (const segment of pathSegments) {
if (!cursor || !cursor.children || cursor.children.length === 0) {
throw new Error(`Note meta for '${filePath}' not found.`);
}
parent = cursor;
cursor = cursor.children.find(file => file.dataFileName === segment || file.dirFileName === segment);
}
return {
parentNoteMeta: parent,
noteMeta: cursor
};
}
function getParentNoteId(filePath, parentNoteMeta) {
return parentNoteMeta.isImportRoot ? importRootNote.noteId : getNewNoteId(parentNoteMeta.noteId);
}
function getNoteId(noteMeta) {
let userGuideNoteId;// = noteMeta.attributes?.find(attr => attr.type === 'label' && attr.name === 'helpNoteId')?.value;
userGuideNoteId = '_userGuide' + noteMeta.title.replace(/[^a-z0-9]/ig, '');
if (noteMeta.title.trim() === 'User Guide') {
userGuideNoteId = '_userGuide';
}
const noteId = userGuideNoteId || noteMeta.noteId;
noteIdMap[noteMeta.noteId] = noteId;
return noteId;
}
function saveAttributes(note, noteMeta) {
if (!noteMeta) {
return;
}
for (const attr of noteMeta.attributes) {
attr.noteId = note.noteId;
if (attr.type === 'label-definition') {
attr.type = 'label';
attr.name = `label:${attr.name}`;
}
else if (attr.type === 'relation-definition') {
attr.type = 'label';
attr.name = `relation:${attr.name}`;
}
if (!attributeService.isAttributeType(attr.type)) {
log.error(`Unrecognized attribute type ${attr.type}`);
continue;
}
if (attr.type === 'relation' && ['internalLink', 'imageLink', 'relationMapLink', 'includeNoteLink'].includes(attr.name)) {
// these relations are created automatically and as such don't need to be duplicated in the import
continue;
}
if (attr.type === 'relation') {
attr.value = getNewNoteId(attr.value);
}
attributes.push(attr);
}
}
function saveDirectory(filePath) {
const { parentNoteMeta, noteMeta } = getMeta(filePath);
const noteId = getNoteId(noteMeta);
const parentNoteId = getParentNoteId(filePath, parentNoteMeta);
let note = becca.getNote(noteId);
if (note) {
return;
}
({note} = noteService.createNewNote({
parentNoteId: parentNoteId,
title: noteMeta.title,
content: '',
noteId: noteId,
type: noteMeta.type,
mime: noteMeta.mime,
prefix: noteMeta.prefix,
isExpanded: noteMeta.isExpanded,
notePosition: noteMeta.notePosition,
isProtected: false,
ignoreForbiddenParents: true
}));
saveAttributes(note, noteMeta);
return noteId;
}
function getNoteIdFromRelativeUrl(url, filePath) {
while (url.startsWith("./")) {
url = url.substr(2);
}
let absUrl = path.dirname(filePath);
while (url.startsWith("../")) {
absUrl = path.dirname(absUrl);
url = url.substr(3);
}
if (absUrl === '.') {
absUrl = '';
}
absUrl += `${absUrl.length > 0 ? '/' : ''}${url}`;
const {noteMeta} = getMeta(absUrl);
const targetNoteId = getNoteId(noteMeta);
return targetNoteId;
}
function processTextNoteContent(content, filePath, noteMeta) {
function isUrlAbsolute(url) {
return /^(?:[a-z]+:)?\/\//i.test(url);
}
content = content.replace(/<h1>([^<]*)<\/h1>/gi, (match, text) => {
if (noteMeta.title.trim() === text.trim()) {
return ""; // remove whole H1 tag
} else {
return `<h2>${text}</h2>`;
}
});
content = htmlSanitizer.sanitize(content);
content = content.replace(/<html.*<body[^>]*>/gis, "");
content = content.replace(/<\/body>.*<\/html>/gis, "");
content = content.replace(/src="([^"]*)"/g, (match, url) => {
try {
url = decodeURIComponent(url);
} catch (e) {
log.error(`Cannot parse image URL '${url}', keeping original (${e}).`);
return `src="${url}"`;
}
if (isUrlAbsolute(url) || url.startsWith("/")) {
return match;
}
const targetNoteId = getNoteIdFromRelativeUrl(url, filePath);
return `src="api/images/${targetNoteId}/${path.basename(url)}"`;
});
content = content.replace(/href="([^"]*)"/g, (match, url) => {
try {
url = decodeURIComponent(url);
} catch (e) {
log.error(`Cannot parse link URL '${url}', keeping original (${e}).`);
return `href="${url}"`;
}
if (url.startsWith('#') // already a note path (probably)
|| isUrlAbsolute(url)) {
return match;
}
const targetNoteId = getNoteIdFromRelativeUrl(url, filePath);
return `href="#root/${targetNoteId}"`;
});
content = content.replace(/data-note-path="([^"]*)"/g, (match, notePath) => {
const noteId = notePath.split("/").pop();
let targetNoteId;
if (noteId === 'root' || noteId.startsWith("_")) { // named noteIds stay identical across instances
targetNoteId = noteId;
} else {
targetNoteId = noteIdMap[noteId];
}
return `data-note-path="root/${targetNoteId}"`;
});
if (noteMeta) {
const includeNoteLinks = (noteMeta.attributes || [])
.filter(attr => attr.type === 'relation' && attr.name === 'includeNoteLink');
for (const link of includeNoteLinks) {
// no need to escape the regexp find string since it's a noteId which doesn't contain any special characters
content = content.replace(new RegExp(link.value, "g"), getNewNoteId(link.value));
}
}
return content;
}
function processNoteContent(noteMeta, type, mime, content, filePath) {
if (type === 'text') {
content = processTextNoteContent(content, filePath, noteMeta);
}
if (type === 'relationMap') {
const relationMapLinks = (noteMeta.attributes || [])
.filter(attr => attr.type === 'relation' && attr.name === 'relationMapLink');
// this will replace relation map links
for (const link of relationMapLinks) {
// no need to escape the regexp find string since it's a noteId which doesn't contain any special characters
content = content.replace(new RegExp(link.value, "g"), getNewNoteId(link.value));
}
}
return content;
}
function saveNote(filePath, content) {
const {parentNoteMeta, noteMeta} = getMeta(filePath);
if (noteMeta?.noImport) {
return;
}
const noteId = getNoteId(noteMeta);
const parentNoteId = getParentNoteId(filePath, parentNoteMeta);
if (!parentNoteId) {
throw new Error(`Cannot find parentNoteId for ${filePath}`);
}
if (noteMeta?.isClone) {
if (!becca.getBranchFromChildAndParent(noteId, parentNoteId)) {
new BBranch({
noteId,
parentNoteId,
isExpanded: noteMeta.isExpanded,
prefix: noteMeta.prefix,
notePosition: noteMeta.notePosition
}).save();
}
return;
}
let {type, mime} = noteMeta;
if (type !== 'file' && type !== 'image') {
content = content.toString("UTF-8");
}
content = processNoteContent(noteMeta, type, mime, content, filePath);
let note = becca.getNote(noteId);
if (note) {
// only skeleton was created because of altered order of cloned notes in ZIP, we need to update
// https://github.com/zadam/trilium/issues/2440
if (note.type === undefined) {
note.type = type;
note.mime = mime;
note.title = noteMeta.title;
note.isProtected = false;
note.save();
}
note.setContent(content);
if (!becca.getBranchFromChildAndParent(noteId, parentNoteId)) {
new BBranch({
noteId,
parentNoteId,
isExpanded: noteMeta.isExpanded,
prefix: noteMeta.prefix,
notePosition: noteMeta.notePosition
}).save();
}
}
else {
({note} = noteService.createNewNote({
parentNoteId: parentNoteId,
title: noteMeta.title,
content: content,
noteId,
type,
mime,
prefix: noteMeta.prefix,
isExpanded: noteMeta.isExpanded,
notePosition: noteMeta.notePosition,
isProtected: false,
ignoreForbiddenParents: true
}));
saveAttributes(note, noteMeta);
}
}
const entries = [];
await readZipFile(fileBuffer, async (zipfile, entry) => {
const filePath = normalizeFilePath(entry.fileName);
if (/\/$/.test(entry.fileName)) {
entries.push({
type: 'directory',
filePath
});
}
else {
entries.push({
type: 'file',
filePath,
content: await readContent(zipfile, entry)
});
}
zipfile.readEntry();
});
metaFile = JSON.parse(entries.find(entry => entry.type === 'file' && entry.filePath === '!!!meta.json').content);
sql.transactional(() => {
deleteUserGuideSubtree();
for (const {type, filePath, content} of entries) {
if (type === 'directory') {
saveDirectory(filePath);
} else if (type === 'file') {
if (filePath === '!!!meta.json') {
continue;
}
saveNote(filePath, content);
} else {
throw new Error(`Unknown type ${type}`)
}
}
});
// we're saving attributes and links only now so that all relation and link target notes
// are already in the database (we don't want to have "broken" relations, not even transitionally)
for (const attr of attributes) {
if (attr.type !== 'relation' || attr.value in becca.notes) {
new BAttribute(attr).save();
}
else {
log.info(`Relation not imported since the target note doesn't exist: ${JSON.stringify(attr)}`);
}
}
}
/**
* This is a special implementation of deleting the subtree, because we want to preserve the links to the user guide pages
* and clones.
*/
function deleteUserGuideSubtree() {
const DELETE_ID = 'user-guide';
function remove(branch) {
branch.markAsDeleted(DELETE_ID);
const note = becca.getNote(branch.noteId);
for (const branch of note.getChildBranches()) {
remove(branch);
}
note.getOwnedAttributes().forEach(attr => attr.markAsDeleted(DELETE_ID));
note.markAsDeleted(DELETE_ID)
}
remove(becca.getBranchFromChildAndParent('_userGuide', '_hidden'));
}
/** @returns {string} path without leading or trailing slash and backslashes converted to forward ones */
function normalizeFilePath(filePath) {
filePath = filePath.replace(/\\/g, "/");
if (filePath.startsWith("/")) {
filePath = filePath.substr(1);
}
if (filePath.endsWith("/")) {
filePath = filePath.substr(0, filePath.length - 1);
}
return filePath;
}
function streamToBuffer(stream) {
const chunks = [];
stream.on('data', chunk => chunks.push(chunk));
return new Promise((res, rej) => stream.on('end', () => res(Buffer.concat(chunks))));
}
function readContent(zipfile, entry) {
return new Promise((res, rej) => {
zipfile.openReadStream(entry, function(err, readStream) {
if (err) rej(err);
streamToBuffer(readStream).then(res);
});
});
}
function readZipFile(buffer, processEntryCallback) {
return new Promise((res, rej) => {
yauzl.fromBuffer(buffer, {lazyEntries: true, validateEntrySizes: false}, function(err, zipfile) {
if (err) throw err;
zipfile.readEntry();
zipfile.on("entry", entry => processEntryCallback(zipfile, entry));
zipfile.on("end", res);
});
});
}
module.exports = {
importUserGuideIfNeeded
};