From ecb467f2b7e47f929baee398e8ebe4543c2bc365 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 11:40:48 +0200 Subject: [PATCH] chore(import): fix a few type errors --- apps/server/package.json | 4 +-- apps/server/src/services/utils.ts | 34 ++----------------- packages/trilium-core/package.json | 2 ++ packages/trilium-core/src/services/image.ts | 10 +++++- .../trilium-core/src/services/import/enex.ts | 26 ++++++++------ .../src/services/import/markdown.ts | 10 +++--- .../trilium-core/src/services/import/opml.ts | 6 ++-- .../src/services/import/single.spec.ts | 8 +++-- .../src/services/import/single.ts | 21 ++++++------ .../trilium-core/src/services/import/utils.ts | 4 +-- .../src/services/import/zip.spec.ts | 4 +-- .../trilium-core/src/services/import/zip.ts | 32 ++++++++--------- .../trilium-core/src/services/utils/binary.ts | 32 +++++++++++++++++ pnpm-lock.yaml | 16 ++++----- 14 files changed, 110 insertions(+), 99 deletions(-) diff --git a/apps/server/package.json b/apps/server/package.json index 243a987b41..f6fac1a4d8 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -68,7 +68,6 @@ "axios": "1.13.6", "bindings": "1.5.0", "bootstrap": "5.3.8", - "chardet": "2.1.1", "cheerio": "1.2.0", "chokidar": "5.0.0", "cls-hooked": "4.2.2", @@ -108,8 +107,7 @@ "safe-compare": "1.1.4", "sax": "1.6.0", "serve-favicon": "2.5.1", - "stream-throttle": "0.1.3", - "strip-bom": "5.0.0", + "stream-throttle": "0.1.3", "striptags": "3.2.0", "supertest": "7.2.2", "swagger-jsdoc": "6.2.8", diff --git a/apps/server/src/services/utils.ts b/apps/server/src/services/utils.ts index 7b4d6cd1d3..db7b4d38f9 100644 --- a/apps/server/src/services/utils.ts +++ b/apps/server/src/services/utils.ts @@ -1,9 +1,7 @@ -import { getCrypto,utils as coreUtils } from "@triliumnext/core"; -import chardet from "chardet"; +import { binary_utils,getCrypto, utils as coreUtils } from "@triliumnext/core"; import crypto from "crypto"; import { release as osRelease } from "os"; import path from "path"; -import stripBom from "strip-bom"; const osVersion = osRelease().split('.').map(Number); @@ -138,35 +136,6 @@ export function getResourceDir() { return path.join(__dirname, ".."); } -/** - * For buffers, they are scanned for a supported encoding and decoded (UTF-8, UTF-16). In some cases, the BOM is also stripped. - * - * For strings, they are returned immediately without any transformation. - * - * For nullish values, an empty string is returned. - * - * @param data the string or buffer to process. - * @returns the string representation of the buffer, or the same string is it's a string. - */ -export function processStringOrBuffer(data: string | Buffer | null) { - if (!data) { - return ""; - } - - if (!Buffer.isBuffer(data)) { - return data; - } - - const detectedEncoding = chardet.detect(data); - switch (detectedEncoding) { - case "UTF-16LE": - return stripBom(data.toString("utf-16le")); - case "UTF-8": - default: - return data.toString("utf-8"); - } -} - /** @deprecated */ export const escapeHtml = coreUtils.escapeHtml; /** @deprecated */ @@ -183,6 +152,7 @@ export const isEmptyOrWhitespace = coreUtils.isEmptyOrWhitespace; export const normalizeUrl = coreUtils.normalizeUrl; export const timeLimit = coreUtils.timeLimit; export const sanitizeSqlIdentifier = coreUtils.sanitizeSqlIdentifier; +export const processStringOrBuffer = binary_utils.processStringOrBuffer; export function waitForStreamToFinish(stream: any): Promise { return new Promise((resolve, reject) => { diff --git a/packages/trilium-core/package.json b/packages/trilium-core/package.json index 2a1133b32e..3a79abdcb0 100644 --- a/packages/trilium-core/package.json +++ b/packages/trilium-core/package.json @@ -10,12 +10,14 @@ "@braintree/sanitize-url": "7.1.1", "@triliumnext/commons": "workspace:*", "async-mutex": "0.5.0", + "chardet": "2.1.1", "escape-html": "1.0.3", "i18next": "25.10.10", "mime-types": "3.0.2", "node-html-parser": "7.1.0", "sanitize-filename": "1.6.4", "sanitize-html": "2.17.2", + "strip-bom": "5.0.0", "unescape": "1.0.1" }, "devDependencies": { diff --git a/packages/trilium-core/src/services/image.ts b/packages/trilium-core/src/services/image.ts index 605c3cec21..7ccd1ff5ad 100644 --- a/packages/trilium-core/src/services/image.ts +++ b/packages/trilium-core/src/services/image.ts @@ -1,5 +1,5 @@ export default { - saveImageToAttachment(noteId: string, imageBuffer: Uint8Array, title: string, b1: boolean, b2: boolean) { + saveImageToAttachment(noteId: string, imageBuffer: Uint8Array, title: string, b1?: boolean, b2?: boolean) { console.warn("Image save ignored", noteId, title); return { @@ -10,5 +10,13 @@ export default { updateImage(noteId: string, imageBuffer: Uint8Array, title: string) { console.warn("Image update ignored", noteId, title); + }, + + saveImage(noteId: string, imageBuffer: Uint8Array, title: string, b1?: boolean, b2?: boolean) { + console.warn("Image save ignored", noteId, title); + + return { + note: null + }; } } diff --git a/packages/trilium-core/src/services/import/enex.ts b/packages/trilium-core/src/services/import/enex.ts index d50f6e4174..18ab4e888d 100644 --- a/packages/trilium-core/src/services/import/enex.ts +++ b/packages/trilium-core/src/services/import/enex.ts @@ -1,20 +1,22 @@ import type { AttributeType } from "@triliumnext/commons"; import { dayjs } from "@triliumnext/commons"; -import { sanitize, utils } from "@triliumnext/core"; import sax from "sax"; import stream from "stream"; import { Throttle } from "stream-throttle"; import type BNote from "../../becca/entities/bnote.js"; -import date_utils from "../date_utils.js"; +import date_utils from "../utils/date.js"; +import * as utils from "../utils/index.js"; import imageService from "../image.js"; -import log from "../log.js"; +import { getLog } from "../log.js"; import noteService from "../notes.js"; import protectedSessionService from "../protected_session.js"; -import sql from "../sql.js"; import type TaskContext from "../task_context.js"; -import { escapeHtml, fromBase64,md5 } from "../utils.js"; +import { escapeHtml, md5 } from "../utils/index.js"; +import { decodeBase64 } from "../utils/binary.js"; import type { File } from "./common.js"; +import { sanitizeHtml } from "../sanitizer.js"; +import { getSql } from "../sql/index.js"; /** * date format is e.g. 20181121T193703Z or 2013-04-14T16:19:00.000Z (Mac evernote, see #3496) @@ -38,7 +40,7 @@ interface Attribute { interface Resource { title: string; - content?: Buffer | string; + content?: Uint8Array | string; mime?: string; attributes: Attribute[]; } @@ -117,7 +119,7 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN "\u2611 " ); - content = sanitize.sanitizeHtml(content); + content = sanitizeHtml(content); return content; } @@ -138,7 +140,7 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN saxStream.on("error", (e) => { // unhandled errors will throw, since this is a proper node event emitter. - log.error(`error when parsing ENEX file: ${e}`); + getLog().error(`error when parsing ENEX file: ${e}`); // clear the error (saxStream._parser as any).error = null; saxStream._parser.resume(); @@ -235,6 +237,8 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN } }); + const sql = getSql(); + function updateDates(note: BNote, utcDateCreated?: string, utcDateModified?: string) { // it's difficult to force custom dateCreated and dateModified to Note entity, so we do it post-creation with SQL const dateCreated = formatDateTimeToLocalDbFormat(utcDateCreated, false); @@ -295,7 +299,7 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN } if (typeof resource.content === "string") { - resource.content = fromBase64(resource.content); + resource.content = decodeBase64(resource.content); } const hash = md5(resource.content); @@ -359,7 +363,7 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN content += imageLink; } } catch (e: any) { - log.error(`error when saving image from ENEX file: ${e.message}`); + getLog().error(`error when saving image from ENEX file: ${e.message}`); createFileNote(); } } else { @@ -367,7 +371,7 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN } } - content = sanitize.sanitizeHtml(content); + content = sanitizeHtml(content); // save updated content with links to files/images noteEntity.setContent(content); diff --git a/packages/trilium-core/src/services/import/markdown.ts b/packages/trilium-core/src/services/import/markdown.ts index dec4a68e7f..35bc7cf8d4 100644 --- a/packages/trilium-core/src/services/import/markdown.ts +++ b/packages/trilium-core/src/services/import/markdown.ts @@ -1,17 +1,15 @@ - - import { getMimeTypeFromMarkdownName, MIME_TYPE_AUTO } from "@triliumnext/commons"; import { normalizeMimeTypeForCKEditor } from "@triliumnext/commons"; -import { sanitize } from "@triliumnext/core"; import { parse, Renderer, type Tokens,use } from "marked"; import { ADMONITION_TYPE_MAPPINGS } from "../export/markdown.js"; -import utils from "../utils.js"; import wikiLinkInternalLink from "./markdown/wikilink_internal_link.js"; import wikiLinkTransclusion from "./markdown/wikilink_transclusion.js"; import importUtils from "./utils.js"; +import { escapeHtml } from "../utils/index.js"; +import { sanitizeHtml } from "../sanitizer.js"; -const escape = utils.escapeHtml; +const escape = escapeHtml; /** * Keep renderer code up to date with https://github.com/markedjs/marked/blob/master/src/Renderer.ts. @@ -151,7 +149,7 @@ function renderToHtml(content: string, title: string) { // h1 handling needs to come before sanitization html = importUtils.handleH1(html, title); - html = sanitize.sanitizeHtml(html); + html = sanitizeHtml(html); // Add a trailing semicolon to CSS styles. html = html.replaceAll(/(<(img|figure|col).*?style=".*?)"/g, "$1;\""); diff --git a/packages/trilium-core/src/services/import/opml.ts b/packages/trilium-core/src/services/import/opml.ts index d9a2ed8fbe..f44371e95e 100644 --- a/packages/trilium-core/src/services/import/opml.ts +++ b/packages/trilium-core/src/services/import/opml.ts @@ -1,12 +1,10 @@ - - -import { sanitize } from "@triliumnext/core"; import xml2js from "xml2js"; import type BNote from "../../becca/entities/bnote.js"; import noteService from "../../services/notes.js"; import protectedSessionService from "../protected_session.js"; import type TaskContext from "../task_context.js"; +import { sanitizeHtml } from "../sanitizer.js"; const parseString = xml2js.parseString; interface OpmlXml { @@ -65,7 +63,7 @@ async function importOpml(taskContext: TaskContext<"importNotes">, fileBuffer: s throw new Error(`Unrecognized OPML version ${opmlVersion}`); } - content = sanitize.sanitizeHtml(content || ""); + content = sanitizeHtml(content || ""); const { note } = noteService.createNewNote({ parentNoteId, diff --git a/packages/trilium-core/src/services/import/single.spec.ts b/packages/trilium-core/src/services/import/single.spec.ts index 866a0874d6..4720bff43e 100644 --- a/packages/trilium-core/src/services/import/single.spec.ts +++ b/packages/trilium-core/src/services/import/single.spec.ts @@ -6,10 +6,10 @@ import { dirname } from "path"; import becca from "../../becca/becca.js"; import BNote from "../../becca/entities/bnote.js"; import TaskContext from "../task_context.js"; -import cls from "../cls.js"; import sql_init from "../sql_init.js"; import single from "./single.js"; import stripBom from "strip-bom"; +import { getContext } from "../context.js"; const scriptDir = dirname(fileURLToPath(import.meta.url)); async function testImport(fileName: string, mimetype: string) { @@ -20,7 +20,7 @@ async function testImport(fileName: string, mimetype: string) { }); return new Promise<{ buffer: Buffer; importedNote: BNote }>((resolve, reject) => { - cls.init(async () => { + getContext().init(async () => { const rootNote = becca.getNote("root"); if (!rootNote) { reject("Missing root note."); @@ -36,6 +36,10 @@ async function testImport(fileName: string, mimetype: string) { }, rootNote as BNote ); + if (importedNote === null) { + reject("Import failed."); + return; + } resolve({ buffer, importedNote diff --git a/packages/trilium-core/src/services/import/single.ts b/packages/trilium-core/src/services/import/single.ts index 7fba5024a5..87cb07da64 100644 --- a/packages/trilium-core/src/services/import/single.ts +++ b/packages/trilium-core/src/services/import/single.ts @@ -1,16 +1,17 @@ import type { NoteType } from "@triliumnext/commons"; -import { sanitize, utils } from "@triliumnext/core"; import type BNote from "../../becca/entities/bnote.js"; import imageService from "../../services/image.js"; import noteService from "../../services/notes.js"; -import { processStringOrBuffer } from "../../services/utils.js"; import protectedSessionService from "../protected_session.js"; import type TaskContext from "../task_context.js"; import type { File } from "./common.js"; import markdownService from "./markdown.js"; import mimeService from "./mime.js"; import importUtils from "./utils.js"; +import { getNoteTitle } from "../utils/index.js"; +import { sanitizeHtml } from "../sanitizer.js"; +import { processStringOrBuffer } from "../utils/binary.js"; function importSingleFile(taskContext: TaskContext<"importNotes">, file: File, parentNote: BNote) { const mime = mimeService.getMime(file.originalname) || file.mimetype; @@ -57,7 +58,7 @@ function importFile(taskContext: TaskContext<"importNotes">, file: File, parentN const mime = mimeService.getMime(originalName) || file.mimetype; const { note } = noteService.createNewNote({ parentNoteId: parentNote.noteId, - title: utils.getNoteTitle(originalName, mime === "application/pdf", { mime }), + title: getNoteTitle(originalName, mime === "application/pdf", { mime }), content: file.buffer, isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(), type: "file", @@ -72,7 +73,7 @@ function importFile(taskContext: TaskContext<"importNotes">, file: File, parentN } function importCodeNote(taskContext: TaskContext<"importNotes">, file: File, parentNote: BNote) { - const title = utils.getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); + const title = getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); const content = processStringOrBuffer(file.buffer); const detectedMime = mimeService.getMime(file.originalname) || file.mimetype; const mime = mimeService.normalizeMimeType(detectedMime); @@ -97,7 +98,7 @@ function importCodeNote(taskContext: TaskContext<"importNotes">, file: File, par } function importCustomType(taskContext: TaskContext<"importNotes">, file: File, parentNote: BNote, type: NoteType, mime: string) { - const title = utils.getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); + const title = getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); const content = processStringOrBuffer(file.buffer); const { note } = noteService.createNewNote({ @@ -115,7 +116,7 @@ function importCustomType(taskContext: TaskContext<"importNotes">, file: File, p } function importPlainText(taskContext: TaskContext<"importNotes">, file: File, parentNote: BNote) { - const title = utils.getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); + const title = getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); const plainTextContent = processStringOrBuffer(file.buffer); const htmlContent = convertTextToHtml(plainTextContent); @@ -150,13 +151,13 @@ function convertTextToHtml(text: string) { } function importMarkdown(taskContext: TaskContext<"importNotes">, file: File, parentNote: BNote) { - const title = utils.getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); + const title = getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); const markdownContent = processStringOrBuffer(file.buffer); let htmlContent = markdownService.renderToHtml(markdownContent, title); if (taskContext.data?.safeImport) { - htmlContent = sanitize.sanitizeHtml(htmlContent); + htmlContent = sanitizeHtml(htmlContent); } const { note } = noteService.createNewNote({ @@ -179,12 +180,12 @@ function importHtml(taskContext: TaskContext<"importNotes">, file: File, parentN // Try to get title from HTML first, fall back to filename // We do this before sanitization since that turns all

s into

const htmlTitle = importUtils.extractHtmlTitle(content); - const title = htmlTitle || utils.getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); + const title = htmlTitle || getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); content = importUtils.handleH1(content, title); if (taskContext?.data?.safeImport) { - content = sanitize.sanitizeHtml(content); + content = sanitizeHtml(content); } const { note } = noteService.createNewNote({ diff --git a/packages/trilium-core/src/services/import/utils.ts b/packages/trilium-core/src/services/import/utils.ts index 110e521cc5..3c5a73d9a0 100644 --- a/packages/trilium-core/src/services/import/utils.ts +++ b/packages/trilium-core/src/services/import/utils.ts @@ -1,6 +1,4 @@ -"use strict"; - -import { unescapeHtml } from "../utils.js"; +import { unescapeHtml } from "../utils"; function handleH1(content: string, title: string) { let isFirstH1Handled = false; diff --git a/packages/trilium-core/src/services/import/zip.spec.ts b/packages/trilium-core/src/services/import/zip.spec.ts index ecc070c6a9..397bce0db1 100644 --- a/packages/trilium-core/src/services/import/zip.spec.ts +++ b/packages/trilium-core/src/services/import/zip.spec.ts @@ -7,9 +7,9 @@ import zip, { removeTriliumTags } from "./zip.js"; import becca from "../../becca/becca.js"; import BNote from "../../becca/entities/bnote.js"; import TaskContext from "../task_context.js"; -import cls from "../cls.js"; import sql_init from "../sql_init.js"; import { trimIndentation } from "@triliumnext/commons"; +import { getContext } from "../context.js"; const scriptDir = dirname(fileURLToPath(import.meta.url)); async function testImport(fileName: string) { @@ -19,7 +19,7 @@ async function testImport(fileName: string) { }); return new Promise<{ importedNote: BNote; rootNote: BNote }>((resolve, reject) => { - cls.init(async () => { + getContext().init(async () => { const rootNote = becca.getNote("root"); if (!rootNote) { expect(rootNote).toBeTruthy(); diff --git a/packages/trilium-core/src/services/import/zip.ts b/packages/trilium-core/src/services/import/zip.ts index 29fce3430a..30aec07ada 100644 --- a/packages/trilium-core/src/services/import/zip.ts +++ b/packages/trilium-core/src/services/import/zip.ts @@ -1,7 +1,4 @@ - - import { ALLOWED_NOTE_TYPES, type NoteType } from "@triliumnext/commons"; -import { sanitize, utils } from "@triliumnext/core"; import path from "path"; import type { Stream } from "stream"; import yauzl from "yauzl"; @@ -12,16 +9,17 @@ import BAttribute from "../../becca/entities/battribute.js"; import BBranch from "../../becca/entities/bbranch.js"; import type BNote from "../../becca/entities/bnote.js"; import attributeService from "../../services/attributes.js"; -import log from "../../services/log.js"; +import { getLog } from "../../services/log.js"; import noteService from "../../services/notes.js"; -import { newEntityId, processStringOrBuffer, unescapeHtml } from "../../services/utils.js"; -import type AttributeMeta from "../meta/attribute_meta.js"; -import type NoteMeta from "../meta/note_meta.js"; +import { getNoteTitle, newEntityId, removeFileExtension, unescapeHtml } from "../../services/utils/index.js"; +import { processStringOrBuffer } from "../../services/utils/binary.js"; import protectedSessionService from "../protected_session.js"; import type TaskContext from "../task_context.js"; import treeService from "../tree.js"; import markdownService from "./markdown.js"; import mimeService from "./mime.js"; +import { AttributeMeta, NoteMeta } from "../../meta.js"; +import { sanitizeHtml } from "../sanitizer.js"; interface MetaFile { files: NoteMeta[]; @@ -162,7 +160,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu // in case we lack metadata, we treat e.g. "Programming.html" and "Programming" as the same note // (one data file, the other directory for children) - const filePathNoExt = utils.removeFileExtension(filePath); + const filePathNoExt = removeFileExtension(filePath); if (filePathNoExt in createdPaths) { return createdPaths[filePathNoExt]; @@ -199,7 +197,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu } if (!attributeService.isAttributeType(attr.type)) { - log.error(`Unrecognized attribute type ${attr.type}`); + getLog().error(`Unrecognized attribute type ${attr.type}`); continue; } @@ -217,8 +215,8 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu } if (taskContext.data?.safeImport) { - attr.name = sanitize.sanitizeHtml(attr.name); - attr.value = sanitize.sanitizeHtml(attr.value); + attr.name = sanitizeHtml(attr.name); + attr.value = sanitizeHtml(attr.value); } attributes.push(attr); @@ -234,7 +232,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu return; } - const noteTitle = utils.getNoteTitle(filePath, !!taskContext.data?.replaceUnderscoresWithSpaces, noteMeta); + const noteTitle = getNoteTitle(filePath, !!taskContext.data?.replaceUnderscoresWithSpaces, noteMeta); const parentNoteId = getParentNoteId(filePath, parentNoteMeta); if (!parentNoteId) { @@ -318,7 +316,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu }); if (taskContext.data?.safeImport) { - content = sanitize.sanitizeHtml(content); + content = sanitizeHtml(content); } content = content.replace(/]*>/gis, ""); @@ -333,7 +331,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu try { url = decodeURIComponent(url).trim(); } catch (e: any) { - log.error(`Cannot parse image URL '${url}', keeping original. Error: ${e.message}.`); + getLog().error(`Cannot parse image URL '${url}', keeping original. Error: ${e.message}.`); return `src="${url}"`; } @@ -356,7 +354,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu try { url = decodeURIComponent(url).trim(); } catch (e: any) { - log.error(`Cannot parse link URL '${url}', keeping original. Error: ${e.message}.`); + getLog().error(`Cannot parse link URL '${url}', keeping original. Error: ${e.message}.`); return `href="${url}"`; } @@ -467,7 +465,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu content = processStringOrBuffer(content); } - const noteTitle = utils.getNoteTitle(filePath, taskContext.data?.replaceUnderscoresWithSpaces || false, noteMeta); + const noteTitle = getNoteTitle(filePath, taskContext.data?.replaceUnderscoresWithSpaces || false, noteMeta); content = processNoteContent(noteMeta, type, mime, content, noteTitle || "", filePath); @@ -613,7 +611,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu if (attr.type !== "relation" || attr.value in becca.notes) { new BAttribute(attr).save(); } else { - log.info(`Relation not imported since the target note doesn't exist: ${JSON.stringify(attr)}`); + getLog().info(`Relation not imported since the target note doesn't exist: ${JSON.stringify(attr)}`); } } diff --git a/packages/trilium-core/src/services/utils/binary.ts b/packages/trilium-core/src/services/utils/binary.ts index 34b2909b9a..dc31b46af0 100644 --- a/packages/trilium-core/src/services/utils/binary.ts +++ b/packages/trilium-core/src/services/utils/binary.ts @@ -1,3 +1,6 @@ +import chardet from "chardet"; +import stripBom from "strip-bom"; + const utf8Decoder = new TextDecoder("utf-8"); const utf8Encoder = new TextEncoder(); @@ -59,3 +62,32 @@ export function wrapStringOrBuffer(stringOrBuffer: string | Uint8Array) { return stringOrBuffer; } } + +/** + * For buffers, they are scanned for a supported encoding and decoded (UTF-8, UTF-16). In some cases, the BOM is also stripped. + * + * For strings, they are returned immediately without any transformation. + * + * For nullish values, an empty string is returned. + * + * @param data the string or buffer to process. + * @returns the string representation of the buffer, or the same string is it's a string. + */ +export function processStringOrBuffer(data: string | Buffer | null) { + if (!data) { + return ""; + } + + if (!Buffer.isBuffer(data)) { + return data; + } + + const detectedEncoding = chardet.detect(data); + switch (detectedEncoding) { + case "UTF-16LE": + return stripBom(data.toString("utf-16le")); + case "UTF-8": + default: + return data.toString("utf-8"); + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8aa762e626..123ff9f72b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -878,9 +878,6 @@ importers: bootstrap: specifier: 5.3.8 version: 5.3.8(@popperjs/core@2.11.8) - chardet: - specifier: 2.1.1 - version: 2.1.1 cheerio: specifier: 1.2.0 version: 1.2.0 @@ -1001,9 +998,6 @@ importers: stream-throttle: specifier: 0.1.3 version: 0.1.3 - strip-bom: - specifier: 5.0.0 - version: 5.0.0 striptags: specifier: 3.2.0 version: 3.2.0 @@ -1707,6 +1701,9 @@ importers: async-mutex: specifier: 0.5.0 version: 0.5.0 + chardet: + specifier: 2.1.1 + version: 2.1.1 escape-html: specifier: 1.0.3 version: 1.0.3 @@ -1725,6 +1722,9 @@ importers: sanitize-html: specifier: 2.17.2 version: 2.17.2 + strip-bom: + specifier: 5.0.0 + version: 5.0.0 unescape: specifier: 1.0.1 version: 1.0.1 @@ -17625,6 +17625,8 @@ snapshots: '@ckeditor/ckeditor5-widget': 47.6.1 ckeditor5: 47.6.1 es-toolkit: 1.39.5 + transitivePeerDependencies: + - supports-color '@ckeditor/ckeditor5-import-word@47.6.1': dependencies: @@ -17752,8 +17754,6 @@ snapshots: '@ckeditor/ckeditor5-utils': 47.6.1 ckeditor5: 47.6.1 es-toolkit: 1.39.5 - transitivePeerDependencies: - - supports-color '@ckeditor/ckeditor5-merge-fields@47.6.1': dependencies: