Dockery/Trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2026-05-06 15:26:59 +02:00
Code Issues Packages Projects Releases Activity

feat(standalone): basic ZIP support

Browse Source
This commit is contained in:
Elian Doran
2026-03-27 18:11:59 +02:00
parent a0573c439b
commit 22c86cf3b5
12 changed files with 145 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
apps/client-standalone/package.json
View File

@@ -40,6 +40,7 @@
"color": "5.0.3",
"debounce": "3.0.0",
"draggabilly": "3.0.0",
"fflate": "0.8.2",
"force-graph": "1.51.2",
"globals": "17.4.0",
"i18next": "25.10.10",

27
apps/client-standalone/src/lightweight/zip_provider.ts Normal file
View File

@@ -0,0 +1,27 @@
import type { ZipEntry, ZipProvider } from "@triliumnext/core/src/services/import/zip_provider.js";
import { unzip } from "fflate";
export default class BrowserZipProvider implements ZipProvider {
readZipFile(
buffer: Uint8Array,
processEntry: (entry: ZipEntry, readContent: () => Promise<Uint8Array>) => Promise<void>
): Promise<void> {
return new Promise<void>((res, rej) => {
unzip(buffer, async (err, files) => {
if (err) { rej(err); return; }
try {
for (const [fileName, data] of Object.entries(files)) {
await processEntry(
{ fileName },
() => Promise.resolve(data)
);
}
res();
} catch (e) {
rej(e);
}
});
});
}
}

5
apps/client-standalone/src/local-server-worker.ts
View File

@@ -55,6 +55,7 @@ let BrowserSqlProvider: typeof import('./lightweight/sql_provider').default;
let WorkerMessagingProvider: typeof import('./lightweight/messaging_provider').default;
let BrowserExecutionContext: typeof import('./lightweight/cls_provider').default;
let BrowserCryptoProvider: typeof import('./lightweight/crypto_provider').default;
let BrowserZipProvider: typeof import('./lightweight/zip_provider').default;
let FetchRequestProvider: typeof import('./lightweight/request_provider').default;
let StandalonePlatformProvider: typeof import('./lightweight/platform_provider').default;
let translationProvider: typeof import('./lightweight/translation_provider').default;
@@ -82,6 +83,7 @@ async function loadModules(): Promise<void> {
messagingModule,
clsModule,
cryptoModule,
zipModule,
requestModule,
platformModule,
translationModule,
@@ -91,6 +93,7 @@ async function loadModules(): Promise<void> {
import('./lightweight/messaging_provider.js'),
import('./lightweight/cls_provider.js'),
import('./lightweight/crypto_provider.js'),
import('./lightweight/zip_provider.js'),
import('./lightweight/request_provider.js'),
import('./lightweight/platform_provider.js'),
import('./lightweight/translation_provider.js'),
@@ -101,6 +104,7 @@ async function loadModules(): Promise<void> {
WorkerMessagingProvider = messagingModule.default;
BrowserExecutionContext = clsModule.default;
BrowserCryptoProvider = cryptoModule.default;
BrowserZipProvider = zipModule.default;
FetchRequestProvider = requestModule.default;
StandalonePlatformProvider = platformModule.default;
translationProvider = translationModule.default;
@@ -152,6 +156,7 @@ async function initialize(): Promise<void> {
await coreModule.initializeCore({
executionContext: new BrowserExecutionContext(),
crypto: new BrowserCryptoProvider(),
zip: new BrowserZipProvider(),
messaging: messagingProvider!,
request: new FetchRequestProvider(),
platform: new StandalonePlatformProvider(queryString),

2
apps/desktop/src/main.ts
View File

@@ -1,6 +1,7 @@
import { getLog, initializeCore, sql_init } from "@triliumnext/core";
import ClsHookedExecutionContext from "@triliumnext/server/src/cls_provider.js";
import NodejsCryptoProvider from "@triliumnext/server/src/crypto_provider.js";
import NodejsZipProvider from "@triliumnext/server/src/zip_provider.js";
import dataDirs from "@triliumnext/server/src/services/data_dir.js";
import options from "@triliumnext/server/src/services/options.js";
import port from "@triliumnext/server/src/services/port.js";
@@ -133,6 +134,7 @@ async function main() {
}
},
crypto: new NodejsCryptoProvider(),
zip: new NodejsZipProvider(),
request: new NodeRequestProvider(),
executionContext: new ClsHookedExecutionContext(),
messaging: new WebSocketMessagingProvider(),

2
apps/server/spec/setup.ts
View File

@@ -4,6 +4,7 @@ import { join } from "path";
import { initializeCore } from "@triliumnext/core";
import ClsHookedExecutionContext from "../src/cls_provider.js";
import NodejsCryptoProvider from "../src/crypto_provider.js";
import NodejsZipProvider from "../src/zip_provider.js";
import ServerPlatformProvider from "../src/platform_provider.js";
import BetterSqlite3Provider from "../src/sql_provider.js";
import { initializeTranslations } from "../src/services/i18n.js";
@@ -27,6 +28,7 @@ beforeAll(async () => {
onTransactionRollback() {}
},
crypto: new NodejsCryptoProvider(),
zip: new NodejsZipProvider(),
executionContext: new ClsHookedExecutionContext(),
schema: readFileSync(require.resolve("@triliumnext/core/src/assets/schema.sql"), "utf-8"),
platform: new ServerPlatformProvider(),

2
apps/server/src/main.ts
View File

@@ -10,6 +10,7 @@ import path from "path";
import ClsHookedExecutionContext from "./cls_provider.js";
import NodejsCryptoProvider from "./crypto_provider.js";
import NodejsZipProvider from "./zip_provider.js";
import ServerPlatformProvider from "./platform_provider.js";
import dataDirs from "./services/data_dir.js";
import port from "./services/port.js";
@@ -51,6 +52,7 @@ async function startApplication() {
}
},
crypto: new NodejsCryptoProvider(),
zip: new NodejsZipProvider(),
request: new NodeRequestProvider(),
executionContext: new ClsHookedExecutionContext(),
messaging: new WebSocketMessagingProvider(),

46
apps/server/src/zip_provider.ts Normal file
View File

@@ -0,0 +1,46 @@
import type { ZipEntry, ZipProvider } from "@triliumnext/core/src/services/import/zip_provider.js";
import type { Stream } from "stream";
import yauzl from "yauzl";
function streamToBuffer(stream: Stream): Promise<Buffer> {
const chunks: Uint8Array[] = [];
stream.on("data", (chunk: Uint8Array) => chunks.push(chunk));
return new Promise((res, rej) => {
stream.on("end", () => res(Buffer.concat(chunks)));
stream.on("error", rej);
});
}
export default class NodejsZipProvider implements ZipProvider {
readZipFile(
buffer: Uint8Array,
processEntry: (entry: ZipEntry, readContent: () => Promise<Uint8Array>) => Promise<void>
): Promise<void> {
return new Promise<void>((res, rej) => {
yauzl.fromBuffer(Buffer.from(buffer), { lazyEntries: true, validateEntrySizes: false }, (err, zipfile) => {
if (err) { rej(err); return; }
if (!zipfile) { rej(new Error("Unable to read zip file.")); return; }
zipfile.readEntry();
zipfile.on("entry", async (entry: yauzl.Entry) => {
try {
const readContent = () => new Promise<Uint8Array>((res, rej) => {
zipfile.openReadStream(entry, (err, readStream) => {
if (err) { rej(err); return; }
if (!readStream) { rej(new Error("Unable to read content.")); return; }
streamToBuffer(readStream).then(res, rej);
});
});
await processEntry({ fileName: entry.fileName }, readContent);
} catch (e) {
rej(e);
}
zipfile.readEntry();
});
zipfile.on("end", res);
zipfile.on("error", rej);
});
});
}
}

5
packages/trilium-core/src/index.ts
View File

@@ -9,6 +9,7 @@ import { initTranslations, TranslationProvider } from "./services/i18n";
import { initSchema } from "./services/sql_init";
import appInfo from "./services/app_info";
import { type PlatformProvider, initPlatform } from "./services/platform";
import { type ZipProvider, initZipProvider } from "./services/import/zip_provider";
export { getLog } from "./services/log";
export type * from "./services/sql/types";
@@ -102,10 +103,11 @@ export * as routeHelpers from "./routes/helpers";
export * as becca_easy_mocking from "./test/becca_easy_mocking";
export * as becca_mocking from "./test/becca_mocking";
export async function initializeCore({ dbConfig, executionContext, crypto, translations, messaging, request, schema, extraAppInfo, platform }: {
export async function initializeCore({ dbConfig, executionContext, crypto, zip, translations, messaging, request, schema, extraAppInfo, platform }: {
dbConfig: SqlServiceParams,
executionContext: ExecutionContext,
crypto: CryptoProvider,
zip: ZipProvider,
translations: TranslationProvider,
platform: PlatformProvider,
schema: string,
@@ -120,6 +122,7 @@ export async function initializeCore({ dbConfig, executionContext, crypto, trans
initLog();
await initTranslations(translations);
initCrypto(crypto);
initZipProvider(zip);
initContext(executionContext);
initSql(new SqlService(dbConfig, getLog()));
initSchema(schema);

76
packages/trilium-core/src/services/import/zip.ts
View File

@@ -1,7 +1,6 @@
import { ALLOWED_NOTE_TYPES, type NoteType } from "@triliumnext/commons";
import path from "path";
import type { Stream } from "stream";
import yauzl from "yauzl";
import { basename, dirname } from "../utils/path.js";
import { getZipProvider } from "./zip_provider.js";
import becca from "../../becca/becca.js";
import BAttachment from "../../becca/entities/battachment.js";
@@ -138,7 +137,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui
if (parentNoteMeta?.noteId) {
parentNoteId = parentNoteMeta.isImportRoot ? importRootNote.noteId : getNewNoteId(parentNoteMeta.noteId);
} else {
const parentPath = path.dirname(filePath);
const parentPath = dirname(filePath);
if (parentPath === ".") {
parentNoteId = importRootNote.noteId;
@@ -267,10 +266,10 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui
url = url.substr(2);
}
absUrl = path.dirname(filePath);
absUrl = dirname(filePath);
while (url.startsWith("../")) {
absUrl = path.dirname(absUrl);
absUrl = dirname(absUrl);
url = url.substr(3);
}
@@ -342,9 +341,9 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui
const target = getEntityIdFromRelativeUrl(url, filePath);
if (target.attachmentId) {
return `src="api/attachments/${target.attachmentId}/image/${path.basename(url)}"`;
return `src="api/attachments/${target.attachmentId}/image/${basename(url)}"`;
} else if (target.noteId) {
return `src="api/images/${target.noteId}/${path.basename(url)}"`;
return `src="api/images/${target.noteId}/${basename(url)}"`;
}
return match;
@@ -390,7 +389,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui
return content;
}
function processNoteContent(noteMeta: NoteMeta | undefined, type: string, mime: string, content: string | Buffer, noteTitle: string, filePath: string) {
function processNoteContent(noteMeta: NoteMeta | undefined, type: string, mime: string, content: string | Uint8Array, noteTitle: string, filePath: string) {
if ((noteMeta?.format === "markdown" || (!noteMeta && taskContext.data?.textImportedAsText && ["text/markdown", "text/x-markdown", "text/mdx"].includes(mime))) && typeof content === "string") {
content = markdownService.renderToHtml(content, noteTitle);
}
@@ -412,7 +411,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui
return content;
}
function saveNote(filePath: string, content: string | Buffer) {
function saveNote(filePath: string, content: string | Uint8Array) {
const { parentNoteMeta, noteMeta, attachmentMeta } = getMeta(filePath);
if (noteMeta?.noImport) {
@@ -549,46 +548,42 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui
noteId,
type: "label",
name: "originalFileName",
value: path.basename(filePath)
value: basename(filePath)
});
}
}
// we're running two passes in order to obtain critical information first (meta file and root)
const topLevelItems = new Set<string>();
await readZipFile(fileBuffer, async (zipfile: yauzl.ZipFile, entry: yauzl.Entry) => {
const zipProvider = getZipProvider();
await zipProvider.readZipFile(fileBuffer, async (entry, readContent) => {
const filePath = normalizeFilePath(entry.fileName);
// make sure that the meta file is loaded before the rest of the files is processed.
if (filePath === "!!!meta.json") {
const content = await readContent(zipfile, entry);
metaFile = JSON.parse(content.toString("utf-8"));
const content = await readContent();
metaFile = JSON.parse(new TextDecoder("utf-8").decode(content));
}
// determine the root of the .zip (i.e. if it has only one top-level folder then the root is that folder, or the root of the archive if there are multiple top-level folders).
const firstSlash = filePath.indexOf("/");
const topLevelPath = (firstSlash !== -1 ? filePath.substring(0, firstSlash) : filePath);
topLevelItems.add(topLevelPath);
zipfile.readEntry();
});
topLevelPath = (topLevelItems.size > 1 ? "" : topLevelItems.values().next().value ?? "");
await readZipFile(fileBuffer, async (zipfile: yauzl.ZipFile, entry: yauzl.Entry) => {
await zipProvider.readZipFile(fileBuffer, async (entry, readContent) => {
const filePath = normalizeFilePath(entry.fileName);
if (/\/$/.test(entry.fileName)) {
saveDirectory(filePath);
} else if (filePath !== "!!!meta.json") {
const content = await readContent(zipfile, entry);
saveNote(filePath, content);
saveNote(filePath, await readContent());
}
taskContext.increaseProgressCount();
zipfile.readEntry();
});
for (const noteId of createdNoteIds) {
@@ -637,43 +632,6 @@ function normalizeFilePath(filePath: string): string {
return filePath;
}
function streamToBuffer(stream: Stream): Promise<Buffer> {
const chunks: Uint8Array[] = [];
stream.on("data", (chunk) => chunks.push(chunk));
return new Promise((res, rej) => stream.on("end", () => res(Buffer.concat(chunks))));
}
export function readContent(zipfile: yauzl.ZipFile, entry: yauzl.Entry): Promise<Buffer> {
return new Promise((res, rej) => {
zipfile.openReadStream(entry, (err, readStream) => {
if (err) rej(err);
if (!readStream) throw new Error("Unable to read content.");
streamToBuffer(readStream).then(res);
});
});
}
export function readZipFile(buffer: Uint8Array, processEntryCallback: (zipfile: yauzl.ZipFile, entry: yauzl.Entry) => Promise<void>) {
return new Promise<void>((res, rej) => {
yauzl.fromBuffer(Buffer.from(buffer), { lazyEntries: true, validateEntrySizes: false }, (err, zipfile) => {
if (err) rej(err);
if (!zipfile) throw new Error("Unable to read zip file.");
zipfile.readEntry();
zipfile.on("entry", async (entry) => {
try {
await processEntryCallback(zipfile, entry);
} catch (e) {
rej(e);
}
});
zipfile.on("end", res);
});
});
}
function resolveNoteType(type: string | undefined): NoteType {
// BC for ZIPs created in Trilium 0.57 and older
switch (type) {

25
packages/trilium-core/src/services/import/zip_provider.ts Normal file
View File

@@ -0,0 +1,25 @@
export interface ZipEntry {
fileName: string;
}
export interface ZipProvider {
/**
* Iterates over every entry in a ZIP buffer, calling `processEntry` for each one.
* `readContent()` inside the callback reads the raw bytes of that entry on demand.
*/
readZipFile(
buffer: Uint8Array,
processEntry: (entry: ZipEntry, readContent: () => Promise<Uint8Array>) => Promise<void>
): Promise<void>;
}
let zipProvider: ZipProvider | null = null;
export function initZipProvider(provider: ZipProvider) {
zipProvider = provider;
}
export function getZipProvider(): ZipProvider {
if (!zipProvider) throw new Error("ZipProvider not initialized.");
return zipProvider;
}

9
packages/trilium-core/src/services/utils/path.ts
View File

@@ -16,3 +16,12 @@ export function basename(filePath: string): string {
const lastSlash = Math.max(filePath.lastIndexOf("/"), filePath.lastIndexOf("\\"));
return filePath.substring(lastSlash + 1);
}
/** Returns the directory part of a file path, or "." if there is none. */
export function dirname(filePath: string): string {
const normalized = filePath.replace(/\\/g, "/");
const lastSlash = normalized.lastIndexOf("/");
if (lastSlash === -1) return ".";
if (lastSlash === 0) return "/";
return normalized.substring(0, lastSlash);
}

5
pnpm-lock.yaml generated
View File

@@ -489,6 +489,9 @@ importers:
draggabilly:
specifier: 3.0.0
version: 3.0.0
fflate:
specifier: 0.8.2
version: 0.8.2
force-graph:
specifier: 1.51.2
version: 1.51.2
@@ -17694,6 +17697,8 @@ snapshots:
'@ckeditor/ckeditor5-widget': 47.6.1
ckeditor5: 47.6.1
es-toolkit: 1.39.5
transitivePeerDependencies:
- supports-color
'@ckeditor/ckeditor5-list-multi-level@47.6.1':
dependencies: