chore(import): fix a few type errors

This commit is contained in:
Elian Doran
2026-03-27 11:40:48 +02:00
parent 4ffaadd481
commit ecb467f2b7
14 changed files with 110 additions and 99 deletions

View File

@@ -68,7 +68,6 @@
"axios": "1.13.6",
"bindings": "1.5.0",
"bootstrap": "5.3.8",
"chardet": "2.1.1",
"cheerio": "1.2.0",
"chokidar": "5.0.0",
"cls-hooked": "4.2.2",
@@ -108,8 +107,7 @@
"safe-compare": "1.1.4",
"sax": "1.6.0",
"serve-favicon": "2.5.1",
"stream-throttle": "0.1.3",
"strip-bom": "5.0.0",
"stream-throttle": "0.1.3",
"striptags": "3.2.0",
"supertest": "7.2.2",
"swagger-jsdoc": "6.2.8",

View File

@@ -1,9 +1,7 @@
import { getCrypto,utils as coreUtils } from "@triliumnext/core";
import chardet from "chardet";
import { binary_utils,getCrypto, utils as coreUtils } from "@triliumnext/core";
import crypto from "crypto";
import { release as osRelease } from "os";
import path from "path";
import stripBom from "strip-bom";
const osVersion = osRelease().split('.').map(Number);
@@ -138,35 +136,6 @@ export function getResourceDir() {
return path.join(__dirname, "..");
}
/**
* For buffers, they are scanned for a supported encoding and decoded (UTF-8, UTF-16). In some cases, the BOM is also stripped.
*
* For strings, they are returned immediately without any transformation.
*
* For nullish values, an empty string is returned.
*
* @param data the string or buffer to process.
* @returns the string representation of the buffer, or the same string is it's a string.
*/
export function processStringOrBuffer(data: string | Buffer | null) {
if (!data) {
return "";
}
if (!Buffer.isBuffer(data)) {
return data;
}
const detectedEncoding = chardet.detect(data);
switch (detectedEncoding) {
case "UTF-16LE":
return stripBom(data.toString("utf-16le"));
case "UTF-8":
default:
return data.toString("utf-8");
}
}
/** @deprecated */
export const escapeHtml = coreUtils.escapeHtml;
/** @deprecated */
@@ -183,6 +152,7 @@ export const isEmptyOrWhitespace = coreUtils.isEmptyOrWhitespace;
export const normalizeUrl = coreUtils.normalizeUrl;
export const timeLimit = coreUtils.timeLimit;
export const sanitizeSqlIdentifier = coreUtils.sanitizeSqlIdentifier;
export const processStringOrBuffer = binary_utils.processStringOrBuffer;
export function waitForStreamToFinish(stream: any): Promise<void> {
return new Promise((resolve, reject) => {

View File

@@ -10,12 +10,14 @@
"@braintree/sanitize-url": "7.1.1",
"@triliumnext/commons": "workspace:*",
"async-mutex": "0.5.0",
"chardet": "2.1.1",
"escape-html": "1.0.3",
"i18next": "25.10.10",
"mime-types": "3.0.2",
"node-html-parser": "7.1.0",
"sanitize-filename": "1.6.4",
"sanitize-html": "2.17.2",
"strip-bom": "5.0.0",
"unescape": "1.0.1"
},
"devDependencies": {

View File

@@ -1,5 +1,5 @@
export default {
saveImageToAttachment(noteId: string, imageBuffer: Uint8Array, title: string, b1: boolean, b2: boolean) {
saveImageToAttachment(noteId: string, imageBuffer: Uint8Array, title: string, b1?: boolean, b2?: boolean) {
console.warn("Image save ignored", noteId, title);
return {
@@ -10,5 +10,13 @@ export default {
updateImage(noteId: string, imageBuffer: Uint8Array, title: string) {
console.warn("Image update ignored", noteId, title);
},
saveImage(noteId: string, imageBuffer: Uint8Array, title: string, b1?: boolean, b2?: boolean) {
console.warn("Image save ignored", noteId, title);
return {
note: null
};
}
}

View File

@@ -1,20 +1,22 @@
import type { AttributeType } from "@triliumnext/commons";
import { dayjs } from "@triliumnext/commons";
import { sanitize, utils } from "@triliumnext/core";
import sax from "sax";
import stream from "stream";
import { Throttle } from "stream-throttle";
import type BNote from "../../becca/entities/bnote.js";
import date_utils from "../date_utils.js";
import date_utils from "../utils/date.js";
import * as utils from "../utils/index.js";
import imageService from "../image.js";
import log from "../log.js";
import { getLog } from "../log.js";
import noteService from "../notes.js";
import protectedSessionService from "../protected_session.js";
import sql from "../sql.js";
import type TaskContext from "../task_context.js";
import { escapeHtml, fromBase64,md5 } from "../utils.js";
import { escapeHtml, md5 } from "../utils/index.js";
import { decodeBase64 } from "../utils/binary.js";
import type { File } from "./common.js";
import { sanitizeHtml } from "../sanitizer.js";
import { getSql } from "../sql/index.js";
/**
* date format is e.g. 20181121T193703Z or 2013-04-14T16:19:00.000Z (Mac evernote, see #3496)
@@ -38,7 +40,7 @@ interface Attribute {
interface Resource {
title: string;
content?: Buffer | string;
content?: Uint8Array | string;
mime?: string;
attributes: Attribute[];
}
@@ -117,7 +119,7 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN
"\u2611 "
);
content = sanitize.sanitizeHtml(content);
content = sanitizeHtml(content);
return content;
}
@@ -138,7 +140,7 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN
saxStream.on("error", (e) => {
// unhandled errors will throw, since this is a proper node event emitter.
log.error(`error when parsing ENEX file: ${e}`);
getLog().error(`error when parsing ENEX file: ${e}`);
// clear the error
(saxStream._parser as any).error = null;
saxStream._parser.resume();
@@ -235,6 +237,8 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN
}
});
const sql = getSql();
function updateDates(note: BNote, utcDateCreated?: string, utcDateModified?: string) {
// it's difficult to force custom dateCreated and dateModified to Note entity, so we do it post-creation with SQL
const dateCreated = formatDateTimeToLocalDbFormat(utcDateCreated, false);
@@ -295,7 +299,7 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN
}
if (typeof resource.content === "string") {
resource.content = fromBase64(resource.content);
resource.content = decodeBase64(resource.content);
}
const hash = md5(resource.content);
@@ -359,7 +363,7 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN
content += imageLink;
}
} catch (e: any) {
log.error(`error when saving image from ENEX file: ${e.message}`);
getLog().error(`error when saving image from ENEX file: ${e.message}`);
createFileNote();
}
} else {
@@ -367,7 +371,7 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN
}
}
content = sanitize.sanitizeHtml(content);
content = sanitizeHtml(content);
// save updated content with links to files/images
noteEntity.setContent(content);

View File

@@ -1,17 +1,15 @@
import { getMimeTypeFromMarkdownName, MIME_TYPE_AUTO } from "@triliumnext/commons";
import { normalizeMimeTypeForCKEditor } from "@triliumnext/commons";
import { sanitize } from "@triliumnext/core";
import { parse, Renderer, type Tokens,use } from "marked";
import { ADMONITION_TYPE_MAPPINGS } from "../export/markdown.js";
import utils from "../utils.js";
import wikiLinkInternalLink from "./markdown/wikilink_internal_link.js";
import wikiLinkTransclusion from "./markdown/wikilink_transclusion.js";
import importUtils from "./utils.js";
import { escapeHtml } from "../utils/index.js";
import { sanitizeHtml } from "../sanitizer.js";
const escape = utils.escapeHtml;
const escape = escapeHtml;
/**
* Keep renderer code up to date with https://github.com/markedjs/marked/blob/master/src/Renderer.ts.
@@ -151,7 +149,7 @@ function renderToHtml(content: string, title: string) {
// h1 handling needs to come before sanitization
html = importUtils.handleH1(html, title);
html = sanitize.sanitizeHtml(html);
html = sanitizeHtml(html);
// Add a trailing semicolon to CSS styles.
html = html.replaceAll(/(<(img|figure|col).*?style=".*?)"/g, "$1;\"");

View File

@@ -1,12 +1,10 @@
import { sanitize } from "@triliumnext/core";
import xml2js from "xml2js";
import type BNote from "../../becca/entities/bnote.js";
import noteService from "../../services/notes.js";
import protectedSessionService from "../protected_session.js";
import type TaskContext from "../task_context.js";
import { sanitizeHtml } from "../sanitizer.js";
const parseString = xml2js.parseString;
interface OpmlXml {
@@ -65,7 +63,7 @@ async function importOpml(taskContext: TaskContext<"importNotes">, fileBuffer: s
throw new Error(`Unrecognized OPML version ${opmlVersion}`);
}
content = sanitize.sanitizeHtml(content || "");
content = sanitizeHtml(content || "");
const { note } = noteService.createNewNote({
parentNoteId,

View File

@@ -6,10 +6,10 @@ import { dirname } from "path";
import becca from "../../becca/becca.js";
import BNote from "../../becca/entities/bnote.js";
import TaskContext from "../task_context.js";
import cls from "../cls.js";
import sql_init from "../sql_init.js";
import single from "./single.js";
import stripBom from "strip-bom";
import { getContext } from "../context.js";
const scriptDir = dirname(fileURLToPath(import.meta.url));
async function testImport(fileName: string, mimetype: string) {
@@ -20,7 +20,7 @@ async function testImport(fileName: string, mimetype: string) {
});
return new Promise<{ buffer: Buffer; importedNote: BNote }>((resolve, reject) => {
cls.init(async () => {
getContext().init(async () => {
const rootNote = becca.getNote("root");
if (!rootNote) {
reject("Missing root note.");
@@ -36,6 +36,10 @@ async function testImport(fileName: string, mimetype: string) {
},
rootNote as BNote
);
if (importedNote === null) {
reject("Import failed.");
return;
}
resolve({
buffer,
importedNote

View File

@@ -1,16 +1,17 @@
import type { NoteType } from "@triliumnext/commons";
import { sanitize, utils } from "@triliumnext/core";
import type BNote from "../../becca/entities/bnote.js";
import imageService from "../../services/image.js";
import noteService from "../../services/notes.js";
import { processStringOrBuffer } from "../../services/utils.js";
import protectedSessionService from "../protected_session.js";
import type TaskContext from "../task_context.js";
import type { File } from "./common.js";
import markdownService from "./markdown.js";
import mimeService from "./mime.js";
import importUtils from "./utils.js";
import { getNoteTitle } from "../utils/index.js";
import { sanitizeHtml } from "../sanitizer.js";
import { processStringOrBuffer } from "../utils/binary.js";
function importSingleFile(taskContext: TaskContext<"importNotes">, file: File, parentNote: BNote) {
const mime = mimeService.getMime(file.originalname) || file.mimetype;
@@ -57,7 +58,7 @@ function importFile(taskContext: TaskContext<"importNotes">, file: File, parentN
const mime = mimeService.getMime(originalName) || file.mimetype;
const { note } = noteService.createNewNote({
parentNoteId: parentNote.noteId,
title: utils.getNoteTitle(originalName, mime === "application/pdf", { mime }),
title: getNoteTitle(originalName, mime === "application/pdf", { mime }),
content: file.buffer,
isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
type: "file",
@@ -72,7 +73,7 @@ function importFile(taskContext: TaskContext<"importNotes">, file: File, parentN
}
function importCodeNote(taskContext: TaskContext<"importNotes">, file: File, parentNote: BNote) {
const title = utils.getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces);
const title = getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces);
const content = processStringOrBuffer(file.buffer);
const detectedMime = mimeService.getMime(file.originalname) || file.mimetype;
const mime = mimeService.normalizeMimeType(detectedMime);
@@ -97,7 +98,7 @@ function importCodeNote(taskContext: TaskContext<"importNotes">, file: File, par
}
function importCustomType(taskContext: TaskContext<"importNotes">, file: File, parentNote: BNote, type: NoteType, mime: string) {
const title = utils.getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces);
const title = getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces);
const content = processStringOrBuffer(file.buffer);
const { note } = noteService.createNewNote({
@@ -115,7 +116,7 @@ function importCustomType(taskContext: TaskContext<"importNotes">, file: File, p
}
function importPlainText(taskContext: TaskContext<"importNotes">, file: File, parentNote: BNote) {
const title = utils.getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces);
const title = getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces);
const plainTextContent = processStringOrBuffer(file.buffer);
const htmlContent = convertTextToHtml(plainTextContent);
@@ -150,13 +151,13 @@ function convertTextToHtml(text: string) {
}
function importMarkdown(taskContext: TaskContext<"importNotes">, file: File, parentNote: BNote) {
const title = utils.getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces);
const title = getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces);
const markdownContent = processStringOrBuffer(file.buffer);
let htmlContent = markdownService.renderToHtml(markdownContent, title);
if (taskContext.data?.safeImport) {
htmlContent = sanitize.sanitizeHtml(htmlContent);
htmlContent = sanitizeHtml(htmlContent);
}
const { note } = noteService.createNewNote({
@@ -179,12 +180,12 @@ function importHtml(taskContext: TaskContext<"importNotes">, file: File, parentN
// Try to get title from HTML first, fall back to filename
// We do this before sanitization since that turns all <h1>s into <h2>
const htmlTitle = importUtils.extractHtmlTitle(content);
const title = htmlTitle || utils.getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces);
const title = htmlTitle || getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces);
content = importUtils.handleH1(content, title);
if (taskContext?.data?.safeImport) {
content = sanitize.sanitizeHtml(content);
content = sanitizeHtml(content);
}
const { note } = noteService.createNewNote({

View File

@@ -1,6 +1,4 @@
"use strict";
import { unescapeHtml } from "../utils.js";
import { unescapeHtml } from "../utils";
function handleH1(content: string, title: string) {
let isFirstH1Handled = false;

View File

@@ -7,9 +7,9 @@ import zip, { removeTriliumTags } from "./zip.js";
import becca from "../../becca/becca.js";
import BNote from "../../becca/entities/bnote.js";
import TaskContext from "../task_context.js";
import cls from "../cls.js";
import sql_init from "../sql_init.js";
import { trimIndentation } from "@triliumnext/commons";
import { getContext } from "../context.js";
const scriptDir = dirname(fileURLToPath(import.meta.url));
async function testImport(fileName: string) {
@@ -19,7 +19,7 @@ async function testImport(fileName: string) {
});
return new Promise<{ importedNote: BNote; rootNote: BNote }>((resolve, reject) => {
cls.init(async () => {
getContext().init(async () => {
const rootNote = becca.getNote("root");
if (!rootNote) {
expect(rootNote).toBeTruthy();

View File

@@ -1,7 +1,4 @@
import { ALLOWED_NOTE_TYPES, type NoteType } from "@triliumnext/commons";
import { sanitize, utils } from "@triliumnext/core";
import path from "path";
import type { Stream } from "stream";
import yauzl from "yauzl";
@@ -12,16 +9,17 @@ import BAttribute from "../../becca/entities/battribute.js";
import BBranch from "../../becca/entities/bbranch.js";
import type BNote from "../../becca/entities/bnote.js";
import attributeService from "../../services/attributes.js";
import log from "../../services/log.js";
import { getLog } from "../../services/log.js";
import noteService from "../../services/notes.js";
import { newEntityId, processStringOrBuffer, unescapeHtml } from "../../services/utils.js";
import type AttributeMeta from "../meta/attribute_meta.js";
import type NoteMeta from "../meta/note_meta.js";
import { getNoteTitle, newEntityId, removeFileExtension, unescapeHtml } from "../../services/utils/index.js";
import { processStringOrBuffer } from "../../services/utils/binary.js";
import protectedSessionService from "../protected_session.js";
import type TaskContext from "../task_context.js";
import treeService from "../tree.js";
import markdownService from "./markdown.js";
import mimeService from "./mime.js";
import { AttributeMeta, NoteMeta } from "../../meta.js";
import { sanitizeHtml } from "../sanitizer.js";
interface MetaFile {
files: NoteMeta[];
@@ -162,7 +160,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu
// in case we lack metadata, we treat e.g. "Programming.html" and "Programming" as the same note
// (one data file, the other directory for children)
const filePathNoExt = utils.removeFileExtension(filePath);
const filePathNoExt = removeFileExtension(filePath);
if (filePathNoExt in createdPaths) {
return createdPaths[filePathNoExt];
@@ -199,7 +197,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu
}
if (!attributeService.isAttributeType(attr.type)) {
log.error(`Unrecognized attribute type ${attr.type}`);
getLog().error(`Unrecognized attribute type ${attr.type}`);
continue;
}
@@ -217,8 +215,8 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu
}
if (taskContext.data?.safeImport) {
attr.name = sanitize.sanitizeHtml(attr.name);
attr.value = sanitize.sanitizeHtml(attr.value);
attr.name = sanitizeHtml(attr.name);
attr.value = sanitizeHtml(attr.value);
}
attributes.push(attr);
@@ -234,7 +232,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu
return;
}
const noteTitle = utils.getNoteTitle(filePath, !!taskContext.data?.replaceUnderscoresWithSpaces, noteMeta);
const noteTitle = getNoteTitle(filePath, !!taskContext.data?.replaceUnderscoresWithSpaces, noteMeta);
const parentNoteId = getParentNoteId(filePath, parentNoteMeta);
if (!parentNoteId) {
@@ -318,7 +316,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu
});
if (taskContext.data?.safeImport) {
content = sanitize.sanitizeHtml(content);
content = sanitizeHtml(content);
}
content = content.replace(/<html.*<body[^>]*>/gis, "");
@@ -333,7 +331,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu
try {
url = decodeURIComponent(url).trim();
} catch (e: any) {
log.error(`Cannot parse image URL '${url}', keeping original. Error: ${e.message}.`);
getLog().error(`Cannot parse image URL '${url}', keeping original. Error: ${e.message}.`);
return `src="${url}"`;
}
@@ -356,7 +354,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu
try {
url = decodeURIComponent(url).trim();
} catch (e: any) {
log.error(`Cannot parse link URL '${url}', keeping original. Error: ${e.message}.`);
getLog().error(`Cannot parse link URL '${url}', keeping original. Error: ${e.message}.`);
return `href="${url}"`;
}
@@ -467,7 +465,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu
content = processStringOrBuffer(content);
}
const noteTitle = utils.getNoteTitle(filePath, taskContext.data?.replaceUnderscoresWithSpaces || false, noteMeta);
const noteTitle = getNoteTitle(filePath, taskContext.data?.replaceUnderscoresWithSpaces || false, noteMeta);
content = processNoteContent(noteMeta, type, mime, content, noteTitle || "", filePath);
@@ -613,7 +611,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu
if (attr.type !== "relation" || attr.value in becca.notes) {
new BAttribute(attr).save();
} else {
log.info(`Relation not imported since the target note doesn't exist: ${JSON.stringify(attr)}`);
getLog().info(`Relation not imported since the target note doesn't exist: ${JSON.stringify(attr)}`);
}
}

View File

@@ -1,3 +1,6 @@
import chardet from "chardet";
import stripBom from "strip-bom";
const utf8Decoder = new TextDecoder("utf-8");
const utf8Encoder = new TextEncoder();
@@ -59,3 +62,32 @@ export function wrapStringOrBuffer(stringOrBuffer: string | Uint8Array) {
return stringOrBuffer;
}
}
/**
* For buffers, they are scanned for a supported encoding and decoded (UTF-8, UTF-16). In some cases, the BOM is also stripped.
*
* For strings, they are returned immediately without any transformation.
*
* For nullish values, an empty string is returned.
*
* @param data the string or buffer to process.
* @returns the string representation of the buffer, or the same string is it's a string.
*/
export function processStringOrBuffer(data: string | Buffer | null) {
if (!data) {
return "";
}
if (!Buffer.isBuffer(data)) {
return data;
}
const detectedEncoding = chardet.detect(data);
switch (detectedEncoding) {
case "UTF-16LE":
return stripBom(data.toString("utf-16le"));
case "UTF-8":
default:
return data.toString("utf-8");
}
}

16
pnpm-lock.yaml generated
View File

@@ -878,9 +878,6 @@ importers:
bootstrap:
specifier: 5.3.8
version: 5.3.8(@popperjs/core@2.11.8)
chardet:
specifier: 2.1.1
version: 2.1.1
cheerio:
specifier: 1.2.0
version: 1.2.0
@@ -1001,9 +998,6 @@ importers:
stream-throttle:
specifier: 0.1.3
version: 0.1.3
strip-bom:
specifier: 5.0.0
version: 5.0.0
striptags:
specifier: 3.2.0
version: 3.2.0
@@ -1707,6 +1701,9 @@ importers:
async-mutex:
specifier: 0.5.0
version: 0.5.0
chardet:
specifier: 2.1.1
version: 2.1.1
escape-html:
specifier: 1.0.3
version: 1.0.3
@@ -1725,6 +1722,9 @@ importers:
sanitize-html:
specifier: 2.17.2
version: 2.17.2
strip-bom:
specifier: 5.0.0
version: 5.0.0
unescape:
specifier: 1.0.1
version: 1.0.1
@@ -17625,6 +17625,8 @@ snapshots:
'@ckeditor/ckeditor5-widget': 47.6.1
ckeditor5: 47.6.1
es-toolkit: 1.39.5
transitivePeerDependencies:
- supports-color
'@ckeditor/ckeditor5-import-word@47.6.1':
dependencies:
@@ -17752,8 +17754,6 @@ snapshots:
'@ckeditor/ckeditor5-utils': 47.6.1
ckeditor5: 47.6.1
es-toolkit: 1.39.5
transitivePeerDependencies:
- supports-color
'@ckeditor/ckeditor5-merge-fields@47.6.1':
dependencies: