Dockery/Trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-11-17 02:30:42 +01:00
Code Issues Packages Projects Releases Activity

chore(prettier): fix all files

Browse Source
This commit is contained in:
Elian Doran
2025-01-09 18:07:02 +02:00
parent 19ee861699
commit 4cbb529fd4
571 changed files with 23226 additions and 23940 deletions
Download Patch File Download Diff File Expand all files Collapse all files

155
src/services/import/enex.ts
View File

@@ -1,6 +1,6 @@
import sax from "sax";
import stream from "stream";
import { Throttle } from 'stream-throttle';
import { Throttle } from "stream-throttle";
import log from "../log.js";
import { md5, escapeHtml, fromBase64 } from "../utils.js";
import sql from "../sql.js";
@@ -23,8 +23,7 @@ function parseDate(text: string) {
text = text.replace(/[-:]/g, "");
// insert - and : to convert it to trilium format
text = text.substr(0, 4) + "-" + text.substr(4, 2) + "-" + text.substr(6, 2)
+ " " + text.substr(9, 2) + ":" + text.substr(11, 2) + ":" + text.substr(13, 2) + ".000Z";
text = text.substr(0, 4) + "-" + text.substr(4, 2) + "-" + text.substr(6, 2) + " " + text.substr(9, 2) + ":" + text.substr(11, 2) + ":" + text.substr(13, 2) + ".000Z";
return text;
}
@@ -50,7 +49,7 @@ interface Note {
noteId: string;
blobId: string;
content: string;
resources: Resource[]
resources: Resource[];
}
let note: Partial<Note> = {};
@@ -59,28 +58,26 @@ let resource: Resource;
function importEnex(taskContext: TaskContext, file: File, parentNote: BNote): Promise<BNote> {
const saxStream = sax.createStream(true);
const rootNoteTitle = file.originalname.toLowerCase().endsWith(".enex")
? file.originalname.substr(0, file.originalname.length - 5)
: file.originalname;
const rootNoteTitle = file.originalname.toLowerCase().endsWith(".enex") ? file.originalname.substr(0, file.originalname.length - 5) : file.originalname;
// root note is new note into all ENEX/notebook's notes will be imported
const rootNote = noteService.createNewNote({
parentNoteId: parentNote.noteId,
title: rootNoteTitle,
content: "",
type: 'text',
mime: 'text/html',
isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
type: "text",
mime: "text/html",
isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable()
}).note;
function extractContent(content: string) {
const openingNoteIndex = content.indexOf('<en-note>');
const openingNoteIndex = content.indexOf("<en-note>");
if (openingNoteIndex !== -1) {
content = content.substr(openingNoteIndex + 9);
}
const closingNoteIndex = content.lastIndexOf('</en-note>');
const closingNoteIndex = content.lastIndexOf("</en-note>");
if (closingNoteIndex !== -1) {
content = content.substr(0, closingNoteIndex);
@@ -109,15 +106,20 @@ function importEnex(taskContext: TaskContext, file: File, parentNote: BNote): Pr
// Replace OneNote converted checkboxes with unicode ballot box based
// on known hash of checkboxes for regular, p1, and p2 checkboxes
content = content.replace(/<en-media alt="To Do( priority [12])?" hash="(74de5d3d1286f01bac98d32a09f601d9|4a19d3041585e11643e808d68dd3e72f|8e17580123099ac6515c3634b1f6f9a1)"( type="[a-z\/]*"| width="\d+"| height="\d+")*\/>/g, "\u2610 ");
content = content.replace(/<en-media alt="To Do( priority [12])?" hash="(5069b775461e471a47ce04ace6e1c6ae|7912ee9cec35fc3dba49edb63a9ed158|3a05f4f006a6eaf2627dae5ed8b8013b)"( type="[a-z\/]*"| width="\d+"| height="\d+")*\/>/g, "\u2611 ");
content = content.replace(
/<en-media alt="To Do( priority [12])?" hash="(74de5d3d1286f01bac98d32a09f601d9|4a19d3041585e11643e808d68dd3e72f|8e17580123099ac6515c3634b1f6f9a1)"( type="[a-z\/]*"| width="\d+"| height="\d+")*\/>/g,
"\u2610 "
);
content = content.replace(
/<en-media alt="To Do( priority [12])?" hash="(5069b775461e471a47ce04ace6e1c6ae|7912ee9cec35fc3dba49edb63a9ed158|3a05f4f006a6eaf2627dae5ed8b8013b)"( type="[a-z\/]*"| width="\d+"| height="\d+")*\/>/g,
"\u2611 "
);
content = htmlSanitizer.sanitize(content);
return content;
}
const path: string[] = [];
function getCurrentTag() {
@@ -132,7 +134,7 @@ function importEnex(taskContext: TaskContext, file: File, parentNote: BNote): Pr
}
}
saxStream.on("error", e => {
saxStream.on("error", (e) => {
// unhandled errors will throw, since this is a proper node event emitter.
log.error(`error when parsing ENEX file: ${e}`);
// clear the error
@@ -140,92 +142,86 @@ function importEnex(taskContext: TaskContext, file: File, parentNote: BNote): Pr
saxStream._parser.resume();
});
saxStream.on("text", text => {
saxStream.on("text", (text) => {
const currentTag = getCurrentTag();
const previousTag = getPreviousTag();
if (previousTag === 'note-attributes') {
if (previousTag === "note-attributes") {
let labelName = currentTag;
if (labelName === 'source-url') {
labelName = 'pageUrl';
if (labelName === "source-url") {
labelName = "pageUrl";
}
labelName = sanitizeAttributeName(labelName || "");
if (note.attributes) {
note.attributes.push({
type: 'label',
type: "label",
name: labelName,
value: text
});
}
}
else if (previousTag === 'resource-attributes') {
if (currentTag === 'file-name') {
} else if (previousTag === "resource-attributes") {
if (currentTag === "file-name") {
resource.attributes.push({
type: 'label',
name: 'originalFileName',
type: "label",
name: "originalFileName",
value: text
});
resource.title = text;
}
else if (currentTag === 'source-url') {
} else if (currentTag === "source-url") {
resource.attributes.push({
type: 'label',
name: 'pageUrl',
type: "label",
name: "pageUrl",
value: text
});
}
}
else if (previousTag === 'resource') {
if (currentTag === 'data') {
text = text.replace(/\s/g, '');
} else if (previousTag === "resource") {
if (currentTag === "data") {
text = text.replace(/\s/g, "");
// resource can be chunked into multiple events: https://github.com/zadam/trilium/issues/3424
// it would probably make sense to do this in a more global way since it can in theory affect any field,
// not just data
resource.content = (resource.content || "") + text;
}
else if (currentTag === 'mime') {
} else if (currentTag === "mime") {
resource.mime = text.toLowerCase();
}
}
else if (previousTag === 'note') {
if (currentTag === 'title') {
} else if (previousTag === "note") {
if (currentTag === "title") {
note.title = text;
} else if (currentTag === 'created') {
} else if (currentTag === "created") {
note.utcDateCreated = parseDate(text);
} else if (currentTag === 'updated') {
} else if (currentTag === "updated") {
note.utcDateModified = parseDate(text);
} else if (currentTag === 'tag' && note.attributes) {
} else if (currentTag === "tag" && note.attributes) {
note.attributes.push({
type: 'label',
type: "label",
name: sanitizeAttributeName(text),
value: ''
})
value: ""
});
}
// unknown tags are just ignored
}
});
saxStream.on("attribute", attr => {
saxStream.on("attribute", (attr) => {
// an attribute. attr has "name" and "value"
});
saxStream.on("opentag", tag => {
saxStream.on("opentag", (tag) => {
path.push(tag.name);
if (tag.name === 'note') {
if (tag.name === "note") {
note = {
content: "",
// it's an array, not a key-value object because we don't know if attributes can be duplicated
attributes: [],
resources: []
};
}
else if (tag.name === 'resource') {
} else if (tag.name === "resource") {
resource = {
title: "resource",
attributes: []
@@ -239,25 +235,29 @@ function importEnex(taskContext: TaskContext, file: File, parentNote: BNote): Pr
function updateDates(note: BNote, utcDateCreated?: string, utcDateModified?: string) {
// it's difficult to force custom dateCreated and dateModified to Note entity, so we do it post-creation with SQL
sql.execute(`
sql.execute(
`
UPDATE notes
SET dateCreated = ?,
utcDateCreated = ?,
dateModified = ?,
utcDateModified = ?
WHERE noteId = ?`,
[utcDateCreated, utcDateCreated, utcDateModified, utcDateModified, note.noteId]);
[utcDateCreated, utcDateCreated, utcDateModified, utcDateModified, note.noteId]
);
sql.execute(`
sql.execute(
`
UPDATE blobs
SET utcDateModified = ?
WHERE blobId = ?`,
[utcDateModified, note.blobId]);
[utcDateModified, note.blobId]
);
}
function saveNote() {
// make a copy because stream continues with the next call and note gets overwritten
let {title, content, attributes, resources, utcDateCreated, utcDateModified} = note;
let { title, content, attributes, resources, utcDateCreated, utcDateModified } = note;
if (!title || !content) {
throw new Error("Missing title or content for note.");
@@ -270,9 +270,9 @@ function importEnex(taskContext: TaskContext, file: File, parentNote: BNote): Pr
title,
content,
utcDateCreated,
type: 'text',
mime: 'text/html',
isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
type: "text",
mime: "text/html",
isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable()
}).note;
for (const attr of attributes || []) {
@@ -297,16 +297,20 @@ function importEnex(taskContext: TaskContext, file: File, parentNote: BNote): Pr
const hash = md5(resource.content);
// skip all checked/unchecked checkboxes from OneNote
if (['74de5d3d1286f01bac98d32a09f601d9',
'4a19d3041585e11643e808d68dd3e72f',
'8e17580123099ac6515c3634b1f6f9a1',
'5069b775461e471a47ce04ace6e1c6ae',
'7912ee9cec35fc3dba49edb63a9ed158',
'3a05f4f006a6eaf2627dae5ed8b8013b'].includes(hash)) {
if (
[
"74de5d3d1286f01bac98d32a09f601d9",
"4a19d3041585e11643e808d68dd3e72f",
"8e17580123099ac6515c3634b1f6f9a1",
"5069b775461e471a47ce04ace6e1c6ae",
"7912ee9cec35fc3dba49edb63a9ed158",
"3a05f4f006a6eaf2627dae5ed8b8013b"
].includes(hash)
) {
continue;
}
const mediaRegex = new RegExp(`<en-media [^>]*hash="${hash}"[^>]*>`, 'g');
const mediaRegex = new RegExp(`<en-media [^>]*hash="${hash}"[^>]*>`, "g");
resource.mime = resource.mime || "application/octet-stream";
@@ -319,9 +323,9 @@ function importEnex(taskContext: TaskContext, file: File, parentNote: BNote): Pr
parentNoteId: noteEntity.noteId,
title: resource.title,
content: resource.content,
type: 'file',
type: "file",
mime: resource.mime,
isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable()
}).note;
for (const attr of resource.attributes) {
@@ -337,11 +341,9 @@ function importEnex(taskContext: TaskContext, file: File, parentNote: BNote): Pr
content = (content || "").replace(mediaRegex, resourceLink);
};
if (resource.mime && resource.mime.startsWith('image/')) {
if (resource.mime && resource.mime.startsWith("image/")) {
try {
const originalName = (resource.title && resource.title !== 'resource')
? resource.title
: `image.${resource.mime.substr(6)}`; // default if real name is not present
const originalName = resource.title && resource.title !== "resource" ? resource.title : `image.${resource.mime.substr(6)}`; // default if real name is not present
const attachment = imageService.saveImageToAttachment(noteEntity.noteId, resource.content, originalName, !!taskContext.data?.shrinkImages);
@@ -375,10 +377,10 @@ function importEnex(taskContext: TaskContext, file: File, parentNote: BNote): Pr
updateDates(noteEntity, utcDateCreated, utcDateModified);
}
saxStream.on("closetag", tag => {
saxStream.on("closetag", (tag) => {
path.pop();
if (tag === 'note') {
if (tag === "note") {
saveNote();
}
});
@@ -387,7 +389,7 @@ function importEnex(taskContext: TaskContext, file: File, parentNote: BNote): Pr
//console.log("opencdata");
});
saxStream.on("cdata", text => {
saxStream.on("cdata", (text) => {
note.content += text;
});
@@ -395,8 +397,7 @@ function importEnex(taskContext: TaskContext, file: File, parentNote: BNote): Pr
//console.log("closecdata");
});
return new Promise((resolve, reject) =>
{
return new Promise((resolve, reject) => {
// resolve only when we parse the whole document AND saving of all notes have been finished
saxStream.on("end", () => resolve(rootNote));
@@ -405,7 +406,7 @@ function importEnex(taskContext: TaskContext, file: File, parentNote: BNote): Pr
bufferStream
// rate limiting to improve responsiveness during / after import
.pipe(new Throttle({rate: 500000}))
.pipe(new Throttle({ rate: 500000 }))
.pipe(saxStream);
});
}

99
src/services/import/mime.ts
View File

@@ -2,46 +2,46 @@
import mimeTypes from "mime-types";
import path from "path";
import { TaskData } from '../task_context_interface.js';
import { TaskData } from "../task_context_interface.js";
const CODE_MIME_TYPES: Record<string, boolean | string> = {
'text/plain': true,
'text/x-csrc': true,
'text/x-c++src': true,
'text/x-csharp': true,
'text/x-clojure': true,
'text/css': true,
'text/x-dockerfile': true,
'text/x-erlang': true,
'text/x-feature': true,
'text/x-go': true,
'text/x-groovy': true,
'text/x-haskell': true,
'text/html': true,
'message/http': true,
'text/x-java': true,
'application/javascript': 'application/javascript;env=frontend',
'application/x-javascript': 'application/javascript;env=frontend',
'application/json': true,
'text/x-kotlin': true,
'text/x-stex': true,
'text/x-lua': true,
"text/plain": true,
"text/x-csrc": true,
"text/x-c++src": true,
"text/x-csharp": true,
"text/x-clojure": true,
"text/css": true,
"text/x-dockerfile": true,
"text/x-erlang": true,
"text/x-feature": true,
"text/x-go": true,
"text/x-groovy": true,
"text/x-haskell": true,
"text/html": true,
"message/http": true,
"text/x-java": true,
"application/javascript": "application/javascript;env=frontend",
"application/x-javascript": "application/javascript;env=frontend",
"application/json": true,
"text/x-kotlin": true,
"text/x-stex": true,
"text/x-lua": true,
// possibly later migrate to text/markdown as primary MIME
'text/markdown': 'text/x-markdown',
'text/x-markdown': true,
'text/x-objectivec': true,
'text/x-pascal': true,
'text/x-perl': true,
'text/x-php': true,
'text/x-python': true,
'text/x-ruby': true,
'text/x-rustsrc': true,
'text/x-scala': true,
'text/x-sh': true,
'text/x-sql': true,
'text/x-swift': true,
'text/xml': true,
'text/x-yaml': true
"text/markdown": "text/x-markdown",
"text/x-markdown": true,
"text/x-objectivec": true,
"text/x-pascal": true,
"text/x-perl": true,
"text/x-php": true,
"text/x-python": true,
"text/x-ruby": true,
"text/x-rustsrc": true,
"text/x-scala": true,
"text/x-sh": true,
"text/x-sql": true,
"text/x-swift": true,
"text/xml": true,
"text/x-yaml": true
};
// extensions missing in mime-db
@@ -67,7 +67,7 @@ const EXTENSION_TO_MIME: Record<string, string> = {
/** @returns false if MIME is not detected */
function getMime(fileName: string) {
if (fileName.toLowerCase() === 'dockerfile') {
if (fileName.toLowerCase() === "dockerfile") {
return "text/x-dockerfile";
}
@@ -81,24 +81,21 @@ function getMime(fileName: string) {
}
function getType(options: TaskData, mime: string) {
mime = mime ? mime.toLowerCase() : '';
mime = mime ? mime.toLowerCase() : "";
if (options.textImportedAsText && (mime === 'text/html' || ['text/markdown', 'text/x-markdown'].includes(mime))) {
return 'text';
}
else if (options.codeImportedAsCode && mime in CODE_MIME_TYPES) {
return 'code';
}
else if (mime.startsWith("image/")) {
return 'image';
}
else {
return 'file';
if (options.textImportedAsText && (mime === "text/html" || ["text/markdown", "text/x-markdown"].includes(mime))) {
return "text";
} else if (options.codeImportedAsCode && mime in CODE_MIME_TYPES) {
return "code";
} else if (mime.startsWith("image/")) {
return "image";
} else {
return "file";
}
}
function normalizeMimeType(mime: string) {
mime = mime ? mime.toLowerCase() : '';
mime = mime ? mime.toLowerCase() : "";
const mappedMime = CODE_MIME_TYPES[mime];
if (mappedMime === true) {

32
src/services/import/opml.ts
View File

@@ -14,9 +14,9 @@ interface OpmlXml {
interface OpmlBody {
$: {
version: string
}
body: OpmlOutline[]
version: string;
};
body: OpmlOutline[];
}
interface OpmlOutline {
@@ -29,19 +29,17 @@ interface OpmlOutline {
}
async function importOpml(taskContext: TaskContext, fileBuffer: string | Buffer, parentNote: BNote) {
const xml = await new Promise<OpmlXml>(function(resolve, reject)
{
const xml = await new Promise<OpmlXml>(function (resolve, reject) {
parseString(fileBuffer, function (err: any, result: OpmlXml) {
if (err) {
reject(err);
}
else {
} else {
resolve(result);
}
});
});
if (!['1.0', '1.1', '2.0'].includes(xml.opml.$.version)) {
if (!["1.0", "1.1", "2.0"].includes(xml.opml.$.version)) {
return [400, `Unsupported OPML version ${xml.opml.$.version}, 1.0, 1.1 or 2.0 expected instead.`];
}
@@ -57,30 +55,28 @@ async function importOpml(taskContext: TaskContext, fileBuffer: string | Buffer,
if (!title || !title.trim()) {
// https://github.com/zadam/trilium/issues/1862
title = outline.$.text;
content = '';
content = "";
}
}
else if (opmlVersion === 2) {
} else if (opmlVersion === 2) {
title = outline.$.text;
content = outline.$._note; // _note is already HTML
}
else {
} else {
throw new Error(`Unrecognized OPML version ${opmlVersion}`);
}
content = htmlSanitizer.sanitize(content || "");
const {note} = noteService.createNewNote({
const { note } = noteService.createNewNote({
parentNoteId,
title,
content,
type: 'text',
type: "text",
isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable()
});
taskContext.increaseProgressCount();
for (const childOutline of (outline.outline || [])) {
for (const childOutline of outline.outline || []) {
importOutline(childOutline, note.noteId);
}
@@ -102,10 +98,10 @@ async function importOpml(taskContext: TaskContext, fileBuffer: string | Buffer,
function toHtml(text: string) {
if (!text) {
return '';
return "";
}
return `<p>${text.replace(/(?:\r\n|\r|\n)/g, '</p><p>')}</p>`;
return `<p>${text.replace(/(?:\r\n|\r|\n)/g, "</p><p>")}</p>`;
}
export default {

47
src/services/import/single.ts
View File

@@ -17,16 +17,16 @@ function importSingleFile(taskContext: TaskContext, file: File, parentNote: BNot
const mime = mimeService.getMime(file.originalname) || file.mimetype;
if (taskContext?.data?.textImportedAsText) {
if (mime === 'text/html') {
if (mime === "text/html") {
return importHtml(taskContext, file, parentNote);
} else if (['text/markdown', 'text/x-markdown'].includes(mime)) {
} else if (["text/markdown", "text/x-markdown"].includes(mime)) {
return importMarkdown(taskContext, file, parentNote);
} else if (mime === 'text/plain') {
} else if (mime === "text/plain") {
return importPlainText(taskContext, file, parentNote);
}
}
if (taskContext?.data?.codeImportedAsCode && mimeService.getType(taskContext.data, mime) === 'code') {
if (taskContext?.data?.codeImportedAsCode && mimeService.getType(taskContext.data, mime) === "code") {
return importCodeNote(taskContext, file, parentNote);
}
@@ -41,7 +41,7 @@ function importImage(file: File, parentNote: BNote, taskContext: TaskContext) {
if (typeof file.buffer === "string") {
throw new Error("Invalid file content for image.");
}
const {note} = imageService.saveImage(parentNote.noteId, file.buffer, file.originalname, !!taskContext.data?.shrinkImages);
const { note } = imageService.saveImage(parentNote.noteId, file.buffer, file.originalname, !!taskContext.data?.shrinkImages);
taskContext.increaseProgressCount();
@@ -51,12 +51,12 @@ function importImage(file: File, parentNote: BNote, taskContext: TaskContext) {
function importFile(taskContext: TaskContext, file: File, parentNote: BNote) {
const originalName = file.originalname;
const {note} = noteService.createNewNote({
const { note } = noteService.createNewNote({
parentNoteId: parentNote.noteId,
title: originalName,
content: file.buffer,
isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
type: 'file',
type: "file",
mime: mimeService.getMime(originalName) || file.mimetype
});
@@ -77,7 +77,7 @@ function importCodeNote(taskContext: TaskContext, file: File, parentNote: BNote)
parentNoteId: parentNote.noteId,
title,
content,
type: 'code',
type: "code",
mime: mime,
isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable()
});
@@ -92,13 +92,13 @@ function importPlainText(taskContext: TaskContext, file: File, parentNote: BNote
const plainTextContent = file.buffer.toString("utf-8");
const htmlContent = convertTextToHtml(plainTextContent);
const {note} = noteService.createNewNote({
const { note } = noteService.createNewNote({
parentNoteId: parentNote.noteId,
title,
content: htmlContent,
type: 'text',
mime: 'text/html',
isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
type: "text",
mime: "text/html",
isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable()
});
taskContext.increaseProgressCount();
@@ -108,9 +108,7 @@ function importPlainText(taskContext: TaskContext, file: File, parentNote: BNote
function convertTextToHtml(text: string) {
// 1: Plain Text Search
text = text.replace(/&/g, "&amp;").
replace(/</g, "&lt;").
replace(/>/g, "&gt;");
text = text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
// 2: Line Breaks
text = text.replace(/\r\n?|\n/g, "<br>");
@@ -134,13 +132,13 @@ function importMarkdown(taskContext: TaskContext, file: File, parentNote: BNote)
htmlContent = htmlSanitizer.sanitize(htmlContent);
}
const {note} = noteService.createNewNote({
const { note } = noteService.createNewNote({
parentNoteId: parentNote.noteId,
title,
content: htmlContent,
type: 'text',
mime: 'text/html',
isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
type: "text",
mime: "text/html",
isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable()
});
taskContext.increaseProgressCount();
@@ -162,14 +160,13 @@ function importHtml(taskContext: TaskContext, file: File, parentNote: BNote) {
content = htmlSanitizer.sanitize(content);
}
const {note} = noteService.createNewNote({
const { note } = noteService.createNewNote({
parentNoteId: parentNote.noteId,
title,
content,
type: 'text',
mime: 'text/html',
isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
type: "text",
mime: "text/html",
isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable()
});
taskContext.increaseProgressCount();
@@ -188,7 +185,7 @@ function importAttachment(taskContext: TaskContext, file: File, parentNote: BNot
parentNote.saveAttachment({
title: file.originalname,
content: file.buffer,
role: 'file',
role: "file",
mime: mime
});

131
src/services/import/zip.ts
View File

@@ -19,11 +19,11 @@ import TaskContext from "../task_context.js";
import BNote from "../../becca/entities/bnote.js";
import NoteMeta from "../meta/note_meta.js";
import AttributeMeta from "../meta/attribute_meta.js";
import { Stream } from 'stream';
import { ALLOWED_NOTE_TYPES, NoteType } from '../../becca/entities/rows.js';
import { Stream } from "stream";
import { ALLOWED_NOTE_TYPES, NoteType } from "../../becca/entities/rows.js";
interface MetaFile {
files: NoteMeta[]
files: NoteMeta[];
}
async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRootNote: BNote): Promise<BNote> {
@@ -34,7 +34,7 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
const attributes: AttributeMeta[] = [];
// path => noteId, used only when meta file is not available
/** path => noteId | attachmentId */
const createdPaths: Record<string, string> = { '/': importRootNote.noteId, '\\': importRootNote.noteId };
const createdPaths: Record<string, string> = { "/": importRootNote.noteId, "\\": importRootNote.noteId };
let metaFile: MetaFile | null = null;
let firstNote: BNote | null = null;
const createdNoteIds = new Set<string>();
@@ -45,7 +45,7 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
return "empty_note_id";
}
if (origNoteId === 'root' || origNoteId.startsWith("_")) {
if (origNoteId === "root" || origNoteId.startsWith("_")) {
// these "named" noteIds don't differ between Trilium instances
return origNoteId;
}
@@ -108,7 +108,7 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
parent = cursor;
if (parent.children) {
cursor = parent.children.find(file => file.dataFileName === segment || file.dirFileName === segment);
cursor = parent.children.find((file) => file.dataFileName === segment || file.dirFileName === segment);
}
if (!cursor) {
@@ -128,11 +128,10 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
if (parentNoteMeta?.noteId) {
parentNoteId = parentNoteMeta.isImportRoot ? importRootNote.noteId : getNewNoteId(parentNoteMeta.noteId);
}
else {
} else {
const parentPath = path.dirname(filePath);
if (parentPath === '.') {
if (parentPath === ".") {
parentNoteId = importRootNote.noteId;
} else if (parentPath in createdPaths) {
parentNoteId = createdPaths[parentPath];
@@ -180,12 +179,11 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
for (const attr of noteMeta.attributes || []) {
attr.noteId = note.noteId;
if (attr.type === 'label-definition') {
attr.type = 'label';
if (attr.type === "label-definition") {
attr.type = "label";
attr.name = `label:${attr.name}`;
}
else if (attr.type === 'relation-definition') {
attr.type = 'label';
} else if (attr.type === "relation-definition") {
attr.type = "label";
attr.name = `relation:${attr.name}`;
}
@@ -194,12 +192,12 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
continue;
}
if (attr.type === 'relation' && ['internalLink', 'imageLink', 'relationMapLink', 'includeNoteLink'].includes(attr.name)) {
if (attr.type === "relation" && ["internalLink", "imageLink", "relationMapLink", "includeNoteLink"].includes(attr.name)) {
// these relations are created automatically and as such don't need to be duplicated in the import
continue;
}
if (attr.type === 'relation') {
if (attr.type === "relation") {
attr.value = getNewNoteId(attr.value);
}
@@ -232,17 +230,17 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
throw new Error("Missing parent note ID.");
}
const {note} = noteService.createNewNote({
const { note } = noteService.createNewNote({
parentNoteId: parentNoteId,
title: noteTitle || "",
content: '',
content: "",
noteId: noteId,
type: resolveNoteType(noteMeta?.type),
mime: noteMeta ? noteMeta.mime : 'text/html',
prefix: noteMeta?.prefix || '',
mime: noteMeta ? noteMeta.mime : "text/html",
prefix: noteMeta?.prefix || "",
isExpanded: !!noteMeta?.isExpanded,
notePosition: (noteMeta && firstNote) ? noteMeta.notePosition : undefined,
isProtected: importRootNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
notePosition: noteMeta && firstNote ? noteMeta.notePosition : undefined,
isProtected: importRootNote.isProtected && protectedSessionService.isProtectedSessionAvailable()
});
createdNoteIds.add(note.noteId);
@@ -267,11 +265,11 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
url = url.substr(3);
}
if (absUrl === '.') {
absUrl = '';
if (absUrl === ".") {
absUrl = "";
}
absUrl += `${absUrl.length > 0 ? '/' : ''}${url}`;
absUrl += `${absUrl.length > 0 ? "/" : ""}${url}`;
const { noteMeta, attachmentMeta } = getMeta(absUrl);
@@ -280,7 +278,8 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
attachmentId: getNewAttachmentId(attachmentMeta.attachmentId),
noteId: getNewNoteId(noteMeta.noteId)
};
} else { // don't check for noteMeta since it's not mandatory for notes
} else {
// don't check for noteMeta since it's not mandatory for notes
return {
noteId: getNoteId(noteMeta, absUrl)
};
@@ -345,8 +344,10 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
return `href="${url}"`;
}
if (url.startsWith('#') // already a note path (probably)
|| isUrlAbsolute(url)) {
if (
url.startsWith("#") || // already a note path (probably)
isUrlAbsolute(url)
) {
return match;
}
@@ -362,8 +363,7 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
});
if (noteMeta) {
const includeNoteLinks = (noteMeta.attributes || [])
.filter(attr => attr.type === 'relation' && attr.name === 'includeNoteLink');
const includeNoteLinks = (noteMeta.attributes || []).filter((attr) => attr.type === "relation" && attr.name === "includeNoteLink");
for (const link of includeNoteLinks) {
// no need to escape the regexp find string since it's a noteId which doesn't contain any special characters
@@ -377,31 +377,25 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
}
function removeTriliumTags(content: string) {
const tagsToRemove = [
'<h1 data-trilium-h1>([^<]*)<\/h1>',
'<title data-trilium-title>([^<]*)<\/title>'
]
const tagsToRemove = ["<h1 data-trilium-h1>([^<]*)<\/h1>", "<title data-trilium-title>([^<]*)<\/title>"];
for (const tag of tagsToRemove) {
let re = new RegExp(tag, "gi");
content = content.replace(re, '');
content = content.replace(re, "");
}
return content;
}
function processNoteContent(noteMeta: NoteMeta | undefined, type: string, mime: string, content: string | Buffer, noteTitle: string, filePath: string) {
if ((noteMeta?.format === 'markdown'
|| (!noteMeta && taskContext.data?.textImportedAsText && ['text/markdown', 'text/x-markdown'].includes(mime)))
&& typeof content === "string") {
if ((noteMeta?.format === "markdown" || (!noteMeta && taskContext.data?.textImportedAsText && ["text/markdown", "text/x-markdown"].includes(mime))) && typeof content === "string") {
content = markdownService.renderToHtml(content, noteTitle);
}
if (type === 'text' && typeof content === "string") {
if (type === "text" && typeof content === "string") {
content = processTextNoteContent(content, noteTitle, filePath, noteMeta);
}
if (type === 'relationMap' && noteMeta && typeof content === "string") {
const relationMapLinks = (noteMeta.attributes || [])
.filter(attr => attr.type === 'relation' && attr.name === 'relationMapLink');
if (type === "relationMap" && noteMeta && typeof content === "string") {
const relationMapLinks = (noteMeta.attributes || []).filter((attr) => attr.type === "relation" && attr.name === "relationMapLink");
// this will replace relation map links
for (const link of relationMapLinks) {
@@ -462,7 +456,7 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
throw new Error("Unable to resolve mime type.");
}
if (type !== 'file' && type !== 'image') {
if (type !== "file" && type !== "image") {
content = content.toString("utf-8");
}
@@ -496,21 +490,20 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
notePosition: noteMeta?.notePosition
}).save();
}
}
else {
({note} = noteService.createNewNote({
} else {
({ note } = noteService.createNewNote({
parentNoteId: parentNoteId,
title: noteTitle || "",
content: content,
noteId,
type,
mime,
prefix: noteMeta?.prefix || '',
prefix: noteMeta?.prefix || "",
isExpanded: !!noteMeta?.isExpanded,
// root notePosition should be ignored since it relates to the original document
// now import root should be placed after existing notes into new parent
notePosition: (noteMeta && firstNote) ? noteMeta.notePosition : undefined,
isProtected: isProtected,
notePosition: noteMeta && firstNote ? noteMeta.notePosition : undefined,
isProtected: isProtected
}));
createdNoteIds.add(note.noteId);
@@ -520,11 +513,11 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
firstNote = firstNote || note;
}
if (!noteMeta && (type === 'file' || type === 'image')) {
if (!noteMeta && (type === "file" || type === "image")) {
attributes.push({
noteId,
type: 'label',
name: 'originalFileName',
type: "label",
name: "originalFileName",
value: path.basename(filePath)
});
}
@@ -535,7 +528,7 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
await readZipFile(fileBuffer, async (zipfile: yauzl.ZipFile, entry: yauzl.Entry) => {
const filePath = normalizeFilePath(entry.fileName);
if (filePath === '!!!meta.json') {
if (filePath === "!!!meta.json") {
const content = await readContent(zipfile, entry);
metaFile = JSON.parse(content.toString("utf-8"));
@@ -549,8 +542,7 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
if (/\/$/.test(entry.fileName)) {
saveDirectory(filePath);
}
else if (filePath !== '!!!meta.json') {
} else if (filePath !== "!!!meta.json") {
const content = await readContent(zipfile, entry);
saveNote(filePath, content);
@@ -568,7 +560,7 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
if (!metaFile) {
// if there's no meta file, then the notes are created based on the order in that zip file but that
// is usually quite random, so we sort the notes in the way they would appear in the file manager
treeService.sortNotes(noteId, 'title', false, true);
treeService.sortNotes(noteId, "title", false, true);
}
taskContext.increaseProgressCount();
@@ -577,10 +569,9 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
// we're saving attributes and links only now so that all relation and link target notes
// are already in the database (we don't want to have "broken" relations, not even transitionally)
for (const attr of attributes) {
if (attr.type !== 'relation' || attr.value in becca.notes) {
if (attr.type !== "relation" || attr.value in becca.notes) {
new BAttribute(attr).save();
}
else {
} else {
log.info(`Relation not imported since the target note doesn't exist: ${JSON.stringify(attr)}`);
}
}
@@ -609,14 +600,14 @@ function normalizeFilePath(filePath: string): string {
function streamToBuffer(stream: Stream): Promise<Buffer> {
const chunks: Uint8Array[] = [];
stream.on('data', chunk => chunks.push(chunk));
stream.on("data", (chunk) => chunks.push(chunk));
return new Promise((res, rej) => stream.on('end', () => res(Buffer.concat(chunks))));
return new Promise((res, rej) => stream.on("end", () => res(Buffer.concat(chunks))));
}
function readContent(zipfile: yauzl.ZipFile, entry: yauzl.Entry): Promise<Buffer> {
return new Promise((res, rej) => {
zipfile.openReadStream(entry, function(err, readStream) {
zipfile.openReadStream(entry, function (err, readStream) {
if (err) rej(err);
if (!readStream) throw new Error("Unable to read content.");
@@ -627,12 +618,12 @@ function readContent(zipfile: yauzl.ZipFile, entry: yauzl.Entry): Promise<Buffer
function readZipFile(buffer: Buffer, processEntryCallback: (zipfile: yauzl.ZipFile, entry: yauzl.Entry) => void) {
return new Promise((res, rej) => {
yauzl.fromBuffer(buffer, {lazyEntries: true, validateEntrySizes: false}, function(err, zipfile) {
yauzl.fromBuffer(buffer, { lazyEntries: true, validateEntrySizes: false }, function (err, zipfile) {
if (err) rej(err);
if (!zipfile) throw new Error("Unable to read zip file.");
zipfile.readEntry();
zipfile.on("entry", async entry => {
zipfile.on("entry", async (entry) => {
try {
await processEntryCallback(zipfile, entry);
} catch (e) {
@@ -646,12 +637,12 @@ function readZipFile(buffer: Buffer, processEntryCallback: (zipfile: yauzl.ZipFi
function resolveNoteType(type: string | undefined): NoteType {
// BC for ZIPs created in Triliun 0.57 and older
if (type === 'relation-map') {
return 'relationMap';
} else if (type === 'note-map') {
return 'noteMap';
} else if (type === 'web-view') {
return 'webView';
if (type === "relation-map") {
return "relationMap";
} else if (type === "note-map") {
return "noteMap";
} else if (type === "web-view") {
return "webView";
}
if (type && (ALLOWED_NOTE_TYPES as readonly string[]).includes(type)) {