From 22c86cf3b575d0dedab9d42b9bdc3db4d4d5d2ad Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 18:11:59 +0200 Subject: [PATCH] feat(standalone): basic ZIP support --- apps/client-standalone/package.json | 1 + .../src/lightweight/zip_provider.ts | 27 +++++++ .../src/local-server-worker.ts | 5 ++ apps/desktop/src/main.ts | 2 + apps/server/spec/setup.ts | 2 + apps/server/src/main.ts | 2 + apps/server/src/zip_provider.ts | 46 +++++++++++ packages/trilium-core/src/index.ts | 5 +- .../trilium-core/src/services/import/zip.ts | 76 +++++-------------- .../src/services/import/zip_provider.ts | 25 ++++++ .../trilium-core/src/services/utils/path.ts | 9 +++ pnpm-lock.yaml | 5 ++ 12 files changed, 145 insertions(+), 60 deletions(-) create mode 100644 apps/client-standalone/src/lightweight/zip_provider.ts create mode 100644 apps/server/src/zip_provider.ts create mode 100644 packages/trilium-core/src/services/import/zip_provider.ts diff --git a/apps/client-standalone/package.json b/apps/client-standalone/package.json index dc06104c1f..40c5730612 100644 --- a/apps/client-standalone/package.json +++ b/apps/client-standalone/package.json @@ -40,6 +40,7 @@ "color": "5.0.3", "debounce": "3.0.0", "draggabilly": "3.0.0", + "fflate": "0.8.2", "force-graph": "1.51.2", "globals": "17.4.0", "i18next": "25.10.10", diff --git a/apps/client-standalone/src/lightweight/zip_provider.ts b/apps/client-standalone/src/lightweight/zip_provider.ts new file mode 100644 index 0000000000..47a1c20f0c --- /dev/null +++ b/apps/client-standalone/src/lightweight/zip_provider.ts @@ -0,0 +1,27 @@ +import type { ZipEntry, ZipProvider } from "@triliumnext/core/src/services/import/zip_provider.js"; +import { unzip } from "fflate"; + +export default class BrowserZipProvider implements ZipProvider { + readZipFile( + buffer: Uint8Array, + processEntry: (entry: ZipEntry, readContent: () => Promise) => Promise + ): Promise { + return new Promise((res, rej) => { + unzip(buffer, async (err, files) => { + if (err) { rej(err); return; } + + try { + for (const [fileName, data] of Object.entries(files)) { + await processEntry( + { fileName }, + () => Promise.resolve(data) + ); + } + res(); + } catch (e) { + rej(e); + } + }); + }); + } +} diff --git a/apps/client-standalone/src/local-server-worker.ts b/apps/client-standalone/src/local-server-worker.ts index be86cf8391..6a31561117 100644 --- a/apps/client-standalone/src/local-server-worker.ts +++ b/apps/client-standalone/src/local-server-worker.ts @@ -55,6 +55,7 @@ let BrowserSqlProvider: typeof import('./lightweight/sql_provider').default; let WorkerMessagingProvider: typeof import('./lightweight/messaging_provider').default; let BrowserExecutionContext: typeof import('./lightweight/cls_provider').default; let BrowserCryptoProvider: typeof import('./lightweight/crypto_provider').default; +let BrowserZipProvider: typeof import('./lightweight/zip_provider').default; let FetchRequestProvider: typeof import('./lightweight/request_provider').default; let StandalonePlatformProvider: typeof import('./lightweight/platform_provider').default; let translationProvider: typeof import('./lightweight/translation_provider').default; @@ -82,6 +83,7 @@ async function loadModules(): Promise { messagingModule, clsModule, cryptoModule, + zipModule, requestModule, platformModule, translationModule, @@ -91,6 +93,7 @@ async function loadModules(): Promise { import('./lightweight/messaging_provider.js'), import('./lightweight/cls_provider.js'), import('./lightweight/crypto_provider.js'), + import('./lightweight/zip_provider.js'), import('./lightweight/request_provider.js'), import('./lightweight/platform_provider.js'), import('./lightweight/translation_provider.js'), @@ -101,6 +104,7 @@ async function loadModules(): Promise { WorkerMessagingProvider = messagingModule.default; BrowserExecutionContext = clsModule.default; BrowserCryptoProvider = cryptoModule.default; + BrowserZipProvider = zipModule.default; FetchRequestProvider = requestModule.default; StandalonePlatformProvider = platformModule.default; translationProvider = translationModule.default; @@ -152,6 +156,7 @@ async function initialize(): Promise { await coreModule.initializeCore({ executionContext: new BrowserExecutionContext(), crypto: new BrowserCryptoProvider(), + zip: new BrowserZipProvider(), messaging: messagingProvider!, request: new FetchRequestProvider(), platform: new StandalonePlatformProvider(queryString), diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index b8d7d33d19..22c20aa513 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -1,6 +1,7 @@ import { getLog, initializeCore, sql_init } from "@triliumnext/core"; import ClsHookedExecutionContext from "@triliumnext/server/src/cls_provider.js"; import NodejsCryptoProvider from "@triliumnext/server/src/crypto_provider.js"; +import NodejsZipProvider from "@triliumnext/server/src/zip_provider.js"; import dataDirs from "@triliumnext/server/src/services/data_dir.js"; import options from "@triliumnext/server/src/services/options.js"; import port from "@triliumnext/server/src/services/port.js"; @@ -133,6 +134,7 @@ async function main() { } }, crypto: new NodejsCryptoProvider(), + zip: new NodejsZipProvider(), request: new NodeRequestProvider(), executionContext: new ClsHookedExecutionContext(), messaging: new WebSocketMessagingProvider(), diff --git a/apps/server/spec/setup.ts b/apps/server/spec/setup.ts index 437ff8a2f8..cd0ef855cc 100644 --- a/apps/server/spec/setup.ts +++ b/apps/server/spec/setup.ts @@ -4,6 +4,7 @@ import { join } from "path"; import { initializeCore } from "@triliumnext/core"; import ClsHookedExecutionContext from "../src/cls_provider.js"; import NodejsCryptoProvider from "../src/crypto_provider.js"; +import NodejsZipProvider from "../src/zip_provider.js"; import ServerPlatformProvider from "../src/platform_provider.js"; import BetterSqlite3Provider from "../src/sql_provider.js"; import { initializeTranslations } from "../src/services/i18n.js"; @@ -27,6 +28,7 @@ beforeAll(async () => { onTransactionRollback() {} }, crypto: new NodejsCryptoProvider(), + zip: new NodejsZipProvider(), executionContext: new ClsHookedExecutionContext(), schema: readFileSync(require.resolve("@triliumnext/core/src/assets/schema.sql"), "utf-8"), platform: new ServerPlatformProvider(), diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index c7bb8d3798..d09ef82ebd 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -10,6 +10,7 @@ import path from "path"; import ClsHookedExecutionContext from "./cls_provider.js"; import NodejsCryptoProvider from "./crypto_provider.js"; +import NodejsZipProvider from "./zip_provider.js"; import ServerPlatformProvider from "./platform_provider.js"; import dataDirs from "./services/data_dir.js"; import port from "./services/port.js"; @@ -51,6 +52,7 @@ async function startApplication() { } }, crypto: new NodejsCryptoProvider(), + zip: new NodejsZipProvider(), request: new NodeRequestProvider(), executionContext: new ClsHookedExecutionContext(), messaging: new WebSocketMessagingProvider(), diff --git a/apps/server/src/zip_provider.ts b/apps/server/src/zip_provider.ts new file mode 100644 index 0000000000..1f858e8de4 --- /dev/null +++ b/apps/server/src/zip_provider.ts @@ -0,0 +1,46 @@ +import type { ZipEntry, ZipProvider } from "@triliumnext/core/src/services/import/zip_provider.js"; +import type { Stream } from "stream"; +import yauzl from "yauzl"; + +function streamToBuffer(stream: Stream): Promise { + const chunks: Uint8Array[] = []; + stream.on("data", (chunk: Uint8Array) => chunks.push(chunk)); + return new Promise((res, rej) => { + stream.on("end", () => res(Buffer.concat(chunks))); + stream.on("error", rej); + }); +} + +export default class NodejsZipProvider implements ZipProvider { + readZipFile( + buffer: Uint8Array, + processEntry: (entry: ZipEntry, readContent: () => Promise) => Promise + ): Promise { + return new Promise((res, rej) => { + yauzl.fromBuffer(Buffer.from(buffer), { lazyEntries: true, validateEntrySizes: false }, (err, zipfile) => { + if (err) { rej(err); return; } + if (!zipfile) { rej(new Error("Unable to read zip file.")); return; } + + zipfile.readEntry(); + zipfile.on("entry", async (entry: yauzl.Entry) => { + try { + const readContent = () => new Promise((res, rej) => { + zipfile.openReadStream(entry, (err, readStream) => { + if (err) { rej(err); return; } + if (!readStream) { rej(new Error("Unable to read content.")); return; } + streamToBuffer(readStream).then(res, rej); + }); + }); + + await processEntry({ fileName: entry.fileName }, readContent); + } catch (e) { + rej(e); + } + zipfile.readEntry(); + }); + zipfile.on("end", res); + zipfile.on("error", rej); + }); + }); + } +} diff --git a/packages/trilium-core/src/index.ts b/packages/trilium-core/src/index.ts index df899145e8..c4fbcb0920 100644 --- a/packages/trilium-core/src/index.ts +++ b/packages/trilium-core/src/index.ts @@ -9,6 +9,7 @@ import { initTranslations, TranslationProvider } from "./services/i18n"; import { initSchema } from "./services/sql_init"; import appInfo from "./services/app_info"; import { type PlatformProvider, initPlatform } from "./services/platform"; +import { type ZipProvider, initZipProvider } from "./services/import/zip_provider"; export { getLog } from "./services/log"; export type * from "./services/sql/types"; @@ -102,10 +103,11 @@ export * as routeHelpers from "./routes/helpers"; export * as becca_easy_mocking from "./test/becca_easy_mocking"; export * as becca_mocking from "./test/becca_mocking"; -export async function initializeCore({ dbConfig, executionContext, crypto, translations, messaging, request, schema, extraAppInfo, platform }: { +export async function initializeCore({ dbConfig, executionContext, crypto, zip, translations, messaging, request, schema, extraAppInfo, platform }: { dbConfig: SqlServiceParams, executionContext: ExecutionContext, crypto: CryptoProvider, + zip: ZipProvider, translations: TranslationProvider, platform: PlatformProvider, schema: string, @@ -120,6 +122,7 @@ export async function initializeCore({ dbConfig, executionContext, crypto, trans initLog(); await initTranslations(translations); initCrypto(crypto); + initZipProvider(zip); initContext(executionContext); initSql(new SqlService(dbConfig, getLog())); initSchema(schema); diff --git a/packages/trilium-core/src/services/import/zip.ts b/packages/trilium-core/src/services/import/zip.ts index 0aa29ab568..94df44ce32 100644 --- a/packages/trilium-core/src/services/import/zip.ts +++ b/packages/trilium-core/src/services/import/zip.ts @@ -1,7 +1,6 @@ import { ALLOWED_NOTE_TYPES, type NoteType } from "@triliumnext/commons"; -import path from "path"; -import type { Stream } from "stream"; -import yauzl from "yauzl"; +import { basename, dirname } from "../utils/path.js"; +import { getZipProvider } from "./zip_provider.js"; import becca from "../../becca/becca.js"; import BAttachment from "../../becca/entities/battachment.js"; @@ -138,7 +137,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui if (parentNoteMeta?.noteId) { parentNoteId = parentNoteMeta.isImportRoot ? importRootNote.noteId : getNewNoteId(parentNoteMeta.noteId); } else { - const parentPath = path.dirname(filePath); + const parentPath = dirname(filePath); if (parentPath === ".") { parentNoteId = importRootNote.noteId; @@ -267,10 +266,10 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui url = url.substr(2); } - absUrl = path.dirname(filePath); + absUrl = dirname(filePath); while (url.startsWith("../")) { - absUrl = path.dirname(absUrl); + absUrl = dirname(absUrl); url = url.substr(3); } @@ -342,9 +341,9 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui const target = getEntityIdFromRelativeUrl(url, filePath); if (target.attachmentId) { - return `src="api/attachments/${target.attachmentId}/image/${path.basename(url)}"`; + return `src="api/attachments/${target.attachmentId}/image/${basename(url)}"`; } else if (target.noteId) { - return `src="api/images/${target.noteId}/${path.basename(url)}"`; + return `src="api/images/${target.noteId}/${basename(url)}"`; } return match; @@ -390,7 +389,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui return content; } - function processNoteContent(noteMeta: NoteMeta | undefined, type: string, mime: string, content: string | Buffer, noteTitle: string, filePath: string) { + function processNoteContent(noteMeta: NoteMeta | undefined, type: string, mime: string, content: string | Uint8Array, noteTitle: string, filePath: string) { if ((noteMeta?.format === "markdown" || (!noteMeta && taskContext.data?.textImportedAsText && ["text/markdown", "text/x-markdown", "text/mdx"].includes(mime))) && typeof content === "string") { content = markdownService.renderToHtml(content, noteTitle); } @@ -412,7 +411,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui return content; } - function saveNote(filePath: string, content: string | Buffer) { + function saveNote(filePath: string, content: string | Uint8Array) { const { parentNoteMeta, noteMeta, attachmentMeta } = getMeta(filePath); if (noteMeta?.noImport) { @@ -549,46 +548,42 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui noteId, type: "label", name: "originalFileName", - value: path.basename(filePath) + value: basename(filePath) }); } } // we're running two passes in order to obtain critical information first (meta file and root) const topLevelItems = new Set(); - await readZipFile(fileBuffer, async (zipfile: yauzl.ZipFile, entry: yauzl.Entry) => { + const zipProvider = getZipProvider(); + + await zipProvider.readZipFile(fileBuffer, async (entry, readContent) => { const filePath = normalizeFilePath(entry.fileName); // make sure that the meta file is loaded before the rest of the files is processed. if (filePath === "!!!meta.json") { - const content = await readContent(zipfile, entry); - - metaFile = JSON.parse(content.toString("utf-8")); + const content = await readContent(); + metaFile = JSON.parse(new TextDecoder("utf-8").decode(content)); } // determine the root of the .zip (i.e. if it has only one top-level folder then the root is that folder, or the root of the archive if there are multiple top-level folders). const firstSlash = filePath.indexOf("/"); const topLevelPath = (firstSlash !== -1 ? filePath.substring(0, firstSlash) : filePath); topLevelItems.add(topLevelPath); - - zipfile.readEntry(); }); topLevelPath = (topLevelItems.size > 1 ? "" : topLevelItems.values().next().value ?? ""); - await readZipFile(fileBuffer, async (zipfile: yauzl.ZipFile, entry: yauzl.Entry) => { + await zipProvider.readZipFile(fileBuffer, async (entry, readContent) => { const filePath = normalizeFilePath(entry.fileName); if (/\/$/.test(entry.fileName)) { saveDirectory(filePath); } else if (filePath !== "!!!meta.json") { - const content = await readContent(zipfile, entry); - - saveNote(filePath, content); + saveNote(filePath, await readContent()); } taskContext.increaseProgressCount(); - zipfile.readEntry(); }); for (const noteId of createdNoteIds) { @@ -637,43 +632,6 @@ function normalizeFilePath(filePath: string): string { return filePath; } -function streamToBuffer(stream: Stream): Promise { - const chunks: Uint8Array[] = []; - stream.on("data", (chunk) => chunks.push(chunk)); - - return new Promise((res, rej) => stream.on("end", () => res(Buffer.concat(chunks)))); -} - -export function readContent(zipfile: yauzl.ZipFile, entry: yauzl.Entry): Promise { - return new Promise((res, rej) => { - zipfile.openReadStream(entry, (err, readStream) => { - if (err) rej(err); - if (!readStream) throw new Error("Unable to read content."); - - streamToBuffer(readStream).then(res); - }); - }); -} - -export function readZipFile(buffer: Uint8Array, processEntryCallback: (zipfile: yauzl.ZipFile, entry: yauzl.Entry) => Promise) { - return new Promise((res, rej) => { - yauzl.fromBuffer(Buffer.from(buffer), { lazyEntries: true, validateEntrySizes: false }, (err, zipfile) => { - if (err) rej(err); - if (!zipfile) throw new Error("Unable to read zip file."); - - zipfile.readEntry(); - zipfile.on("entry", async (entry) => { - try { - await processEntryCallback(zipfile, entry); - } catch (e) { - rej(e); - } - }); - zipfile.on("end", res); - }); - }); -} - function resolveNoteType(type: string | undefined): NoteType { // BC for ZIPs created in Trilium 0.57 and older switch (type) { diff --git a/packages/trilium-core/src/services/import/zip_provider.ts b/packages/trilium-core/src/services/import/zip_provider.ts new file mode 100644 index 0000000000..7b6cc9d04e --- /dev/null +++ b/packages/trilium-core/src/services/import/zip_provider.ts @@ -0,0 +1,25 @@ +export interface ZipEntry { + fileName: string; +} + +export interface ZipProvider { + /** + * Iterates over every entry in a ZIP buffer, calling `processEntry` for each one. + * `readContent()` inside the callback reads the raw bytes of that entry on demand. + */ + readZipFile( + buffer: Uint8Array, + processEntry: (entry: ZipEntry, readContent: () => Promise) => Promise + ): Promise; +} + +let zipProvider: ZipProvider | null = null; + +export function initZipProvider(provider: ZipProvider) { + zipProvider = provider; +} + +export function getZipProvider(): ZipProvider { + if (!zipProvider) throw new Error("ZipProvider not initialized."); + return zipProvider; +} diff --git a/packages/trilium-core/src/services/utils/path.ts b/packages/trilium-core/src/services/utils/path.ts index d470b5dc16..da2497cdcd 100644 --- a/packages/trilium-core/src/services/utils/path.ts +++ b/packages/trilium-core/src/services/utils/path.ts @@ -16,3 +16,12 @@ export function basename(filePath: string): string { const lastSlash = Math.max(filePath.lastIndexOf("/"), filePath.lastIndexOf("\\")); return filePath.substring(lastSlash + 1); } + +/** Returns the directory part of a file path, or "." if there is none. */ +export function dirname(filePath: string): string { + const normalized = filePath.replace(/\\/g, "/"); + const lastSlash = normalized.lastIndexOf("/"); + if (lastSlash === -1) return "."; + if (lastSlash === 0) return "/"; + return normalized.substring(0, lastSlash); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8b9940114e..63bc0cd559 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -489,6 +489,9 @@ importers: draggabilly: specifier: 3.0.0 version: 3.0.0 + fflate: + specifier: 0.8.2 + version: 0.8.2 force-graph: specifier: 1.51.2 version: 1.51.2 @@ -17694,6 +17697,8 @@ snapshots: '@ckeditor/ckeditor5-widget': 47.6.1 ckeditor5: 47.6.1 es-toolkit: 1.39.5 + transitivePeerDependencies: + - supports-color '@ckeditor/ckeditor5-list-multi-level@47.6.1': dependencies: