diff --git a/apps/client-standalone/src/lightweight/browser_routes.ts b/apps/client-standalone/src/lightweight/browser_routes.ts index 739fae625e..556eaf9ce7 100644 --- a/apps/client-standalone/src/lightweight/browser_routes.ts +++ b/apps/client-standalone/src/lightweight/browser_routes.ts @@ -166,6 +166,7 @@ function createAsyncRoute(router: BrowserRouter) { * Used for route handlers (like image routes) that write directly to the response. */ function createMockExpressResponse() { + const chunks: string[] = []; const res = { _used: false, _status: 200, @@ -179,6 +180,10 @@ function createMockExpressResponse() { res._headers[name] = value; return res; }, + removeHeader(name: string) { + delete res._headers[name]; + return res; + }, status(code: number) { res._status = code; return res; @@ -192,6 +197,15 @@ function createMockExpressResponse() { res._used = true; res._status = code; return res; + }, + write(chunk: string) { + chunks.push(chunk); + return true; + }, + end() { + res._used = true; + res._body = chunks.join(""); + return res; } }; return res; diff --git a/apps/client-standalone/src/lightweight/zip_export_provider_factory.ts b/apps/client-standalone/src/lightweight/zip_export_provider_factory.ts new file mode 100644 index 0000000000..12e1f75e2b --- /dev/null +++ b/apps/client-standalone/src/lightweight/zip_export_provider_factory.ts @@ -0,0 +1,18 @@ +import { type ExportFormat, type ZipExportProviderData, ZipExportProvider } from "@triliumnext/core"; + +import contentCss from "@triliumnext/ckeditor5/src/theme/ck-content.css?raw"; + +export async function standaloneZipExportProviderFactory(format: ExportFormat, data: ZipExportProviderData): Promise { + switch (format) { + case "html": { + const { default: HtmlExportProvider } = await import("@triliumnext/core/src/services/export/zip/html.js"); + return new HtmlExportProvider(data, { contentCss }); + } + case "markdown": { + const { default: MarkdownExportProvider } = await import("@triliumnext/core/src/services/export/zip/markdown.js"); + return new MarkdownExportProvider(data); + } + default: + throw new Error(`Unsupported export format: '${format}'`); + } +} diff --git a/apps/client-standalone/src/lightweight/zip_provider.ts b/apps/client-standalone/src/lightweight/zip_provider.ts index 47a1c20f0c..5827ebcdae 100644 --- a/apps/client-standalone/src/lightweight/zip_provider.ts +++ b/apps/client-standalone/src/lightweight/zip_provider.ts @@ -1,7 +1,59 @@ -import type { ZipEntry, ZipProvider } from "@triliumnext/core/src/services/import/zip_provider.js"; -import { unzip } from "fflate"; +import type { FileStream, ZipArchive, ZipEntry, ZipProvider } from "@triliumnext/core/src/services/zip_provider.js"; +import { strToU8, unzip, zipSync } from "fflate"; + +type ZipOutput = { + send?: (body: unknown) => unknown; + write?: (chunk: Uint8Array | string) => unknown; + end?: (chunk?: Uint8Array | string) => unknown; +}; + +class BrowserZipArchive implements ZipArchive { + readonly #entries: Record = {}; + #destination: ZipOutput | null = null; + + append(content: string | Uint8Array, options: { name: string }) { + this.#entries[options.name] = typeof content === "string" ? strToU8(content) : content; + } + + pipe(destination: unknown) { + this.#destination = destination as ZipOutput; + } + + async finalize(): Promise { + if (!this.#destination) { + throw new Error("ZIP output destination not set."); + } + + const content = zipSync(this.#entries, { level: 9 }); + + if (typeof this.#destination.send === "function") { + this.#destination.send(content); + return; + } + + if (typeof this.#destination.end === "function") { + if (typeof this.#destination.write === "function") { + this.#destination.write(content); + this.#destination.end(); + } else { + this.#destination.end(content); + } + return; + } + + throw new Error("Unsupported ZIP output destination."); + } +} export default class BrowserZipProvider implements ZipProvider { + createZipArchive(): ZipArchive { + return new BrowserZipArchive(); + } + + createFileStream(_filePath: string): FileStream { + throw new Error("File stream creation is not supported in the browser."); + } + readZipFile( buffer: Uint8Array, processEntry: (entry: ZipEntry, readContent: () => Promise) => Promise diff --git a/apps/client-standalone/src/local-server-worker.ts b/apps/client-standalone/src/local-server-worker.ts index 6b54a947a5..7accb40430 100644 --- a/apps/client-standalone/src/local-server-worker.ts +++ b/apps/client-standalone/src/local-server-worker.ts @@ -157,6 +157,7 @@ async function initialize(): Promise { executionContext: new BrowserExecutionContext(), crypto: new BrowserCryptoProvider(), zip: new BrowserZipProvider(), + zipExportProviderFactory: (await import("./lightweight/zip_export_provider_factory.js")).standaloneZipExportProviderFactory, messaging: messagingProvider!, request: new FetchRequestProvider(), platform: new StandalonePlatformProvider(queryString), diff --git a/apps/client-standalone/src/sw.ts b/apps/client-standalone/src/sw.ts index 7a5edf73a0..630d09acd9 100644 --- a/apps/client-standalone/src/sw.ts +++ b/apps/client-standalone/src/sw.ts @@ -162,6 +162,13 @@ self.addEventListener("fetch", (event) => { // Only handle same-origin if (url.origin !== self.location.origin) return; + // API-ish: local-first via bridge (must be checked before navigate handling, + // because export triggers a navigation to an /api/ URL) + if (isLocalFirst(url)) { + event.respondWith(forwardToClientLocalServer(event.request, event.clientId)); + return; + } + // HTML files: network-first to ensure updates are reflected immediately if (event.request.mode === "navigate" || url.pathname.endsWith(".html")) { event.respondWith(networkFirst(event.request)); @@ -169,17 +176,11 @@ self.addEventListener("fetch", (event) => { } // Static assets: cache-first for performance - if (event.request.method === "GET" && !isLocalFirst(url)) { + if (event.request.method === "GET") { event.respondWith(cacheFirst(event.request)); return; } - // API-ish: local-first via bridge - if (isLocalFirst(url)) { - event.respondWith(forwardToClientLocalServer(event.request, event.clientId)); - return; - } - // Default event.respondWith(fetch(event.request)); }); diff --git a/apps/client/src/widgets/dialogs/export.tsx b/apps/client/src/widgets/dialogs/export.tsx index 01caa7659f..d31bf6a126 100644 --- a/apps/client/src/widgets/dialogs/export.tsx +++ b/apps/client/src/widgets/dialogs/export.tsx @@ -1,16 +1,18 @@ +import "./export.css"; + import { useState } from "preact/hooks"; + +import froca from "../../services/froca"; import { t } from "../../services/i18n"; +import open from "../../services/open"; +import toastService, { type ToastOptionsWithRequiredId } from "../../services/toast"; import tree from "../../services/tree"; +import utils, { isStandalone } from "../../services/utils"; +import ws from "../../services/ws"; import Button from "../react/Button"; import FormRadioGroup from "../react/FormRadioGroup"; -import Modal from "../react/Modal"; -import "./export.css"; -import ws from "../../services/ws"; -import toastService, { type ToastOptionsWithRequiredId } from "../../services/toast"; -import utils from "../../services/utils"; -import open from "../../services/open"; -import froca from "../../services/froca"; import { useTriliumEvent } from "../react/hooks"; +import Modal from "../react/Modal"; interface ExportDialogProps { branchId?: string | null; @@ -79,7 +81,7 @@ export default function ExportDialog() { values={[ { value: "html", label: t("export.format_html_zip") }, { value: "markdown", label: t("export.format_markdown") }, - { value: "share", label: t("export.share-format") }, + !isStandalone && { value: "share", label: t("export.share-format") }, { value: "opml", label: t("export.format_opml") } ]} /> diff --git a/apps/client/src/widgets/react/FormRadioGroup.tsx b/apps/client/src/widgets/react/FormRadioGroup.tsx index bf81b17d83..54eaa07fe2 100644 --- a/apps/client/src/widgets/react/FormRadioGroup.tsx +++ b/apps/client/src/widgets/react/FormRadioGroup.tsx @@ -1,30 +1,35 @@ import type { ComponentChildren } from "preact"; + import { useUniqueName } from "./hooks"; interface FormRadioProps { name: string; currentValue?: string; - values: { + values: ({ value: string; label: string | ComponentChildren; inlineDescription?: string | ComponentChildren; - }[]; + } | false)[]; onChange(newValue: string): void; } export default function FormRadioGroup({ values, ...restProps }: FormRadioProps) { return (
- {(values || []).map(({ value, label, inlineDescription }) => ( -
- -
- ))} + {(values || []).map((el) => { + if (!el) return null; + const { value, label, inlineDescription } = el; + return ( +
+ +
+ ); + })}
); } @@ -32,9 +37,13 @@ export default function FormRadioGroup({ values, ...restProps }: FormRadioProps) export function FormInlineRadioGroup({ values, ...restProps }: FormRadioProps) { return (
- {values.map(({ value, label }) => ())} + {values.map((el) => { + if (!el) return null; + const { value, label, inlineDescription } = el; + return ; + })}
- ) + ); } function FormRadio({ name, value, label, currentValue, onChange, labelClassName, inlineDescription }: Omit & { value: string, label: ComponentChildren, inlineDescription?: ComponentChildren, labelClassName?: string }) { @@ -50,7 +59,7 @@ function FormRadio({ name, value, label, currentValue, onChange, labelClassName, /> {inlineDescription ? <>{label} - {inlineDescription} - : label} + : label} - ) -} \ No newline at end of file + ); +} diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index 8aef99bae6..61698721f9 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -135,6 +135,7 @@ async function main() { }, crypto: new NodejsCryptoProvider(), zip: new NodejsZipProvider(), + zipExportProviderFactory: (await import("@triliumnext/server/src/services/export/zip/factory.js")).serverZipExportProviderFactory, request: new NodeRequestProvider(), executionContext: new ClsHookedExecutionContext(), messaging: new WebSocketMessagingProvider(), diff --git a/apps/edit-docs/src/edit-demo.ts b/apps/edit-docs/src/edit-demo.ts index 4f70a380e7..915d7c857e 100644 --- a/apps/edit-docs/src/edit-demo.ts +++ b/apps/edit-docs/src/edit-demo.ts @@ -54,8 +54,8 @@ async function registerHandlers() { } async function exportData() { - const { exportToZipFile } = (await import("@triliumnext/server/src/services/export/zip.js")).default; - await exportToZipFile("root", "html", DEMO_ZIP_PATH); + const { zipExportService } = (await import("@triliumnext/core")); + await zipExportService.exportToZipFile("root", "html", DEMO_ZIP_PATH); } main(); diff --git a/apps/edit-docs/src/edit-docs.ts b/apps/edit-docs/src/edit-docs.ts index 67a9ac06bb..239b52a0bb 100644 --- a/apps/edit-docs/src/edit-docs.ts +++ b/apps/edit-docs/src/edit-docs.ts @@ -1,7 +1,6 @@ import debounce from "@triliumnext/client/src/services/debounce.js"; +import type { AdvancedExportOptions, ExportFormat } from "@triliumnext/core"; import cls from "@triliumnext/server/src/services/cls.js"; -import type { AdvancedExportOptions, ExportFormat } from "@triliumnext/server/src/services/export/zip/abstract_provider.js"; -import { initializeTranslations } from "@triliumnext/server/src/services/i18n.js"; import { parseNoteMetaFile } from "@triliumnext/server/src/services/in_app_help.js"; import type { NoteMetaFile } from "@triliumnext/server/src/services/meta/note_meta.js"; import type NoteMeta from "@triliumnext/server/src/services/meta/note_meta.js"; @@ -11,7 +10,7 @@ import yaml from "js-yaml"; import path from "path"; import packageJson from "../package.json" with { type: "json" }; -import { extractZip, importData, initializeDatabase, startElectron } from "./utils.js"; +import { extractZip, importData, startElectron } from "./utils.js"; interface NoteMapping { rootNoteId: string; @@ -153,7 +152,7 @@ async function exportData(noteId: string, format: ExportFormat, outputPath: stri await fsExtra.mkdir(outputPath); // First export as zip. - const { exportToZipFile } = (await import("@triliumnext/server/src/services/export/zip.js")).default; + const { zipExportService } = (await import("@triliumnext/core")); const exportOpts: AdvancedExportOptions = {}; if (format === "html") { @@ -205,7 +204,7 @@ async function exportData(noteId: string, format: ExportFormat, outputPath: stri }; } - await exportToZipFile(noteId, format, zipFilePath, exportOpts); + await zipExportService.exportToZipFile(noteId, format, zipFilePath, exportOpts); await extractZip(zipFilePath, outputPath, ignoredFiles); } finally { if (await fsExtra.exists(zipFilePath)) { diff --git a/apps/server/package.json b/apps/server/package.json index f6fac1a4d8..2ddc4d6670 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -5,7 +5,7 @@ "private": true, "main": "./src/main.ts", "scripts": { - "dev": "cross-env NODE_ENV=development TRILIUM_ENV=dev TRILIUM_DATA_DIR=data TRILIUM_RESOURCE_DIR=src tsx watch --ignore '../client/node_modules/.vite-temp' ./src/main.ts", + "dev": "cross-env NODE_ENV=development TRILIUM_PORT=8081 TRILIUM_ENV=dev TRILIUM_DATA_DIR=data TRILIUM_RESOURCE_DIR=src tsx watch --ignore '../client/node_modules/.vite-temp' ./src/main.ts", "dev-alt": "cross-env NODE_ENV=development TRILIUM_ENV=dev TRILIUM_DATA_DIR=data2 TRILIUM_RESOURCE_DIR=src tsx watch --ignore '../client/node_modules/.vite-temp' ./src/main.ts", "start-no-dir": "cross-env NODE_ENV=development TRILIUM_ENV=dev TRILIUM_RESOURCE_DIR=src tsx watch --ignore '../client/node_modules/.vite-temp' ./src/main.ts", "edit-integration-db": "cross-env NODE_ENV=development TRILIUM_PORT=8086 TRILIUM_ENV=dev TRILIUM_DATA_DIR=spec/db TRILIUM_INTEGRATION_TEST=edit TRILIUM_RESOURCE_DIR=src tsx watch --ignore '../client/node_modules/.vite-temp' ./src/main.ts", @@ -41,7 +41,6 @@ "@triliumnext/core": "workspace:*", "@triliumnext/express-partial-content": "workspace:*", "@triliumnext/highlightjs": "workspace:*", - "@triliumnext/turndown-plugin-gfm": "workspace:*", "@types/archiver": "7.0.0", "@types/better-sqlite3": "7.6.13", "@types/cls-hooked": "4.3.9", diff --git a/apps/server/spec/setup.ts b/apps/server/spec/setup.ts index cd0ef855cc..484d582465 100644 --- a/apps/server/spec/setup.ts +++ b/apps/server/spec/setup.ts @@ -2,6 +2,7 @@ import { beforeAll } from "vitest"; import { readFileSync } from "fs"; import { join } from "path"; import { initializeCore } from "@triliumnext/core"; +import { serverZipExportProviderFactory } from "../src/services/export/zip/factory.js"; import ClsHookedExecutionContext from "../src/cls_provider.js"; import NodejsCryptoProvider from "../src/crypto_provider.js"; import NodejsZipProvider from "../src/zip_provider.js"; @@ -29,6 +30,7 @@ beforeAll(async () => { }, crypto: new NodejsCryptoProvider(), zip: new NodejsZipProvider(), + zipExportProviderFactory: serverZipExportProviderFactory, executionContext: new ClsHookedExecutionContext(), schema: readFileSync(require.resolve("@triliumnext/core/src/assets/schema.sql"), "utf-8"), platform: new ServerPlatformProvider(), diff --git a/apps/server/src/etapi/notes.ts b/apps/server/src/etapi/notes.ts index 8c5e7d35b8..3dd81c3612 100644 --- a/apps/server/src/etapi/notes.ts +++ b/apps/server/src/etapi/notes.ts @@ -1,10 +1,8 @@ -import { NoteParams, SearchParams, zipImportService } from "@triliumnext/core"; +import { type ExportFormat, NoteParams, SearchParams, zipExportService, zipImportService } from "@triliumnext/core"; import type { Request, Router } from "express"; import type { ParsedQs } from "qs"; import becca from "../becca/becca.js"; -import zipExportService from "../services/export/zip.js"; -import type { ExportFormat } from "../services/export/zip/abstract_provider.js"; import noteService from "../services/notes.js"; import SearchContext from "../services/search/search_context.js"; import searchService from "../services/search/services/search.js"; diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index 5a5c58c150..66572d1c8a 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -3,7 +3,7 @@ * are loaded later and will result in an empty string. */ -import { getLog,initializeCore, sql_init } from "@triliumnext/core"; +import { getLog, initializeCore, sql_init } from "@triliumnext/core"; import fs from "fs"; import { t } from "i18next"; import path from "path"; @@ -53,6 +53,7 @@ async function startApplication() { }, crypto: new NodejsCryptoProvider(), zip: new NodejsZipProvider(), + zipExportProviderFactory: (await import("./services/export/zip/factory.js")).serverZipExportProviderFactory, request: new NodeRequestProvider(), executionContext: new ClsHookedExecutionContext(), messaging: new WebSocketMessagingProvider(), diff --git a/apps/server/src/platform_provider.ts b/apps/server/src/platform_provider.ts index ae3433f47c..f99121c20c 100644 --- a/apps/server/src/platform_provider.ts +++ b/apps/server/src/platform_provider.ts @@ -6,7 +6,7 @@ export default class ServerPlatformProvider implements PlatformProvider { readonly isWindows = process.platform === "win32"; crash(message: string): void { - getLog().error(message); + getLog().banner(message); process.exit(1); } diff --git a/apps/server/src/routes/api/other.ts b/apps/server/src/routes/api/other.ts deleted file mode 100644 index 3481c1647b..0000000000 --- a/apps/server/src/routes/api/other.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { RenderMarkdownResponse, ToMarkdownResponse } from "@triliumnext/commons"; -import { markdownImportService } from "@triliumnext/core"; -import type { Request } from "express"; - -import markdown from "../../services/export/markdown.js"; - -function renderMarkdown(req: Request) { - const { markdownContent } = req.body; - if (!markdownContent || typeof markdownContent !== 'string') { - throw new Error('markdownContent parameter is required and must be a string'); - } - return { - htmlContent: markdownImportService.renderToHtml(markdownContent, "") - } satisfies RenderMarkdownResponse; -} - -function toMarkdown(req: Request) { - const { htmlContent } = req.body; - if (!htmlContent || typeof htmlContent !== 'string') { - throw new Error('htmlContent parameter is required and must be a string'); - } - return { - markdownContent: markdown.toMarkdown(htmlContent) - } satisfies ToMarkdownResponse; -} - -export default { - renderMarkdown, - toMarkdown -}; diff --git a/apps/server/src/routes/routes.ts b/apps/server/src/routes/routes.ts index b34a610957..5a1a2bdb5e 100644 --- a/apps/server/src/routes/routes.ts +++ b/apps/server/src/routes/routes.ts @@ -22,12 +22,10 @@ import backendLogRoute from "./api/backend_log.js"; import clipperRoute from "./api/clipper.js"; import databaseRoute from "./api/database.js"; import etapiTokensApiRoutes from "./api/etapi_tokens.js"; -import exportRoute from "./api/export.js"; import filesRoute from "./api/files.js"; import fontsRoute from "./api/fonts.js"; import loginApiRoute from "./api/login.js"; import metricsRoute from "./api/metrics.js"; -import otherRoute from "./api/other.js"; import passwordApiRoute from "./api/password.js"; import recoveryCodes from './api/recovery_codes.js'; import scriptRoute from "./api/script.js"; @@ -130,10 +128,6 @@ function register(app: express.Application) { // TODO: Re-enable once we support route() // route(GET, "/api/revisions/:revisionId/download", [auth.checkApiAuthOrElectron], revisionsApiRoute.downloadRevision); - route(GET, "/api/branches/:branchId/export/:type/:format/:version/:taskId", [auth.checkApiAuthOrElectron], exportRoute.exportBranch); - - // :filename is not used by trilium, but instead used for "save as" to assign a human-readable filename - apiRoute(PST, "/api/password/change", passwordApiRoute.changePassword); apiRoute(PST, "/api/password/reset", passwordApiRoute.resetPassword); @@ -198,8 +192,6 @@ function register(app: express.Application) { asyncApiRoute(GET, "/api/backend-log", backendLogRoute.getBackendLog); route(GET, "/api/fonts", [auth.checkApiAuthOrElectron], fontsRoute.getFontCss); - apiRoute(PST, "/api/other/render-markdown", otherRoute.renderMarkdown); - apiRoute(PST, "/api/other/to-markdown", otherRoute.toMarkdown); shareRoutes.register(router); diff --git a/apps/server/src/services/backend_script_api.ts b/apps/server/src/services/backend_script_api.ts index 2be378caa4..98b2e76c30 100644 --- a/apps/server/src/services/backend_script_api.ts +++ b/apps/server/src/services/backend_script_api.ts @@ -1,5 +1,5 @@ import { type AttributeRow, dayjs, formatLogMessage } from "@triliumnext/commons"; -import { type AbstractBeccaEntity, Becca, branches as branchService, NoteParams, SearchContext, sync_mutex as syncMutex } from "@triliumnext/core"; +import { type AbstractBeccaEntity, Becca, branches as branchService, NoteParams, SearchContext, sync_mutex as syncMutex, zipExportService } from "@triliumnext/core"; import axios from "axios"; import * as cheerio from "cheerio"; import xml2js from "xml2js"; @@ -19,7 +19,6 @@ import backupService from "./backup.js"; import cloningService from "./cloning.js"; import config from "./config.js"; import dateNoteService from "./date_notes.js"; -import exportService from "./export/zip.js"; import log from "./log.js"; import noteService from "./notes.js"; import optionsService from "./options.js"; @@ -662,7 +661,7 @@ function BackendScriptApi(this: Api, currentNote: BNote, apiParams: ApiParams) { return { note: launcherNote }; }; - this.exportSubtreeToZipFile = async (noteId, format, zipFilePath) => await exportService.exportToZipFile(noteId, format, zipFilePath); + this.exportSubtreeToZipFile = async (noteId, format, zipFilePath) => await zipExportService.exportToZipFile(noteId, format, zipFilePath); this.runOnFrontend = async (_script, params = []) => { let script: string; diff --git a/apps/server/src/services/export/markdown.ts b/apps/server/src/services/export/markdown.ts deleted file mode 100644 index 35e08ee11f..0000000000 --- a/apps/server/src/services/export/markdown.ts +++ /dev/null @@ -1,284 +0,0 @@ -import { gfm } from "@triliumnext/turndown-plugin-gfm"; -import Turnish, { type Rule } from "turnish"; - -let instance: Turnish | null = null; - -// TODO: Move this to a dedicated file someday. -export const ADMONITION_TYPE_MAPPINGS: Record = { - note: "NOTE", - tip: "TIP", - important: "IMPORTANT", - caution: "CAUTION", - warning: "WARNING" -}; - -export const DEFAULT_ADMONITION_TYPE = ADMONITION_TYPE_MAPPINGS.note; - -const fencedCodeBlockFilter: Rule = { - filter (node, options) { - return options.codeBlockStyle === "fenced" && node.nodeName === "PRE" && node.firstChild !== null && node.firstChild.nodeName === "CODE"; - }, - - replacement (content, node, options) { - if (!node.firstChild || !("getAttribute" in node.firstChild) || typeof node.firstChild.getAttribute !== "function") { - return content; - } - - const className = node.firstChild.getAttribute("class") || ""; - const language = rewriteLanguageTag((className.match(/language-(\S+)/) || [null, ""])[1]); - - return `\n\n${options.fence}${language}\n${node.firstChild.textContent}\n${options.fence}\n\n`; - } -}; - -function toMarkdown(content: string) { - if (instance === null) { - instance = new Turnish({ - headingStyle: "atx", - bulletListMarker: "*", - emDelimiter: "_", - codeBlockStyle: "fenced", - blankReplacement(_content, node) { - if (node.nodeName === "SECTION" && node.classList.contains("include-note")) { - return node.outerHTML; - } - - // Original implementation as per https://github.com/mixmark-io/turndown/blob/master/src/turndown.js. - return ("isBlock" in node && node.isBlock) ? '\n\n' : ''; - }, - }); - // Filter is heavily based on: https://github.com/mixmark-io/turndown/issues/274#issuecomment-458730974 - instance.addRule("fencedCodeBlock", fencedCodeBlockFilter); - instance.addRule("img", buildImageFilter()); - instance.addRule("admonition", buildAdmonitionFilter()); - instance.addRule("inlineLink", buildInlineLinkFilter()); - instance.addRule("figure", buildFigureFilter()); - instance.addRule("math", buildMathFilter()); - instance.addRule("li", buildListItemFilter()); - instance.use(gfm); - instance.keep([ "kbd", "sup", "sub" ]); - } - - return instance.render(content); -} - -function rewriteLanguageTag(source: string) { - if (!source) { - return source; - } - - switch (source) { - case "text-x-trilium-auto": - return ""; - case "application-javascript-env-frontend": - case "application-javascript-env-backend": - return "javascript"; - case "text-x-nginx-conf": - return "nginx"; - default: - return source.split("-").at(-1); - } -} - -// TODO: Remove once upstream delivers a fix for https://github.com/mixmark-io/turndown/issues/467. -function buildImageFilter() { - const ESCAPE_PATTERNS = { - before: /([\\*`[\]_]|(?:^[-+>])|(?:^~~~)|(?:^#{1-6}))/g, - after: /((?:^\d+(?=\.)))/ - }; - - const escapePattern = new RegExp(`(?:${ESCAPE_PATTERNS.before.source}|${ESCAPE_PATTERNS.after.source})`, 'g'); - - function escapeMarkdown (content: string) { - return content.replace(escapePattern, (match, before, after) => { - return before ? `\\${before}` : `${after}\\`; - }); - } - - function escapeLinkDestination(destination: string) { - return destination - .replace(/([()])/g, '\\$1') - .replace(/ /g, "%20"); - } - - function escapeLinkTitle (title: string) { - return title.replace(/"/g, '\\"'); - } - - const imageFilter: Rule = { - filter: "img", - replacement(content, _node) { - const node = _node as HTMLElement; - - // Preserve image verbatim if it has a width or height attribute. - if (node.hasAttribute("width") || node.hasAttribute("height")) { - return node.outerHTML; - } - - // TODO: Deduplicate with upstream. - const untypedNode = (node as any); - const alt = escapeMarkdown(cleanAttribute(untypedNode.getAttribute('alt'))); - const src = escapeLinkDestination(untypedNode.getAttribute('src') || ''); - const title = cleanAttribute(untypedNode.getAttribute('title')); - const titlePart = title ? ` "${escapeLinkTitle(title)}"` : ''; - - return src ? `![${alt}](${src}${titlePart})` : ''; - } - }; - return imageFilter; -} - -function buildAdmonitionFilter() { - function parseAdmonitionType(_node: Node) { - if (!("getAttribute" in _node)) { - return DEFAULT_ADMONITION_TYPE; - } - - const node = _node as Element; - const classList = node.getAttribute("class")?.split(" ") ?? []; - - for (const className of classList) { - if (className === "admonition") { - continue; - } - - const mappedType = ADMONITION_TYPE_MAPPINGS[className]; - if (mappedType) { - return mappedType; - } - } - - return DEFAULT_ADMONITION_TYPE; - } - - const admonitionFilter: Rule = { - filter(node, options) { - return node.nodeName === "ASIDE" && node.classList.contains("admonition"); - }, - replacement(content, node) { - // Parse the admonition type. - const admonitionType = parseAdmonitionType(node); - - content = content.replace(/^\n+|\n+$/g, ''); - content = content.replace(/^/gm, '> '); - content = `> [!${admonitionType}]\n${content}`; - - return `\n\n${content}\n\n`; - } - }; - return admonitionFilter; -} - -/** - * Variation of the original ruleset: https://github.com/mixmark-io/turndown/blob/master/src/commonmark-rules.js. - * - * Detects if the URL is a Trilium reference link and returns it verbatim if that's the case. - * - * @returns - */ -function buildInlineLinkFilter(): Rule { - return { - filter (node, options) { - return ( - options.linkStyle === 'inlined' && - node.nodeName === 'A' && - !!node.getAttribute('href') - ); - }, - - replacement (content, _node) { - const node = _node as HTMLElement; - - // Return reference links verbatim. - if (node.classList.contains("reference-link")) { - return node.outerHTML; - } - - // Otherwise treat as normal. - // TODO: Call super() somehow instead of duplicating the implementation. - let href = node.getAttribute('href'); - if (href) href = href.replace(/([()])/g, '\\$1'); - let title = cleanAttribute(node.getAttribute('title')); - if (title) title = ` "${title.replace(/"/g, '\\"')}"`; - return `[${content}](${href}${title})`; - } - }; -} - -function buildFigureFilter(): Rule { - return { - filter(node, options) { - return node.nodeName === 'FIGURE' - && node.classList.contains("image"); - }, - replacement(content, node) { - return (node as HTMLElement).outerHTML; - } - }; -} - -// Keep in line with https://github.com/mixmark-io/turndown/blob/master/src/commonmark-rules.js. -function buildListItemFilter(): Rule { - return { - filter: "li", - replacement(content, node, options) { - content = content - .trim() - .replace(/\n/gm, '\n '); // indent - let prefix = `${options.bulletListMarker} `; - const parent = node.parentNode as HTMLElement; - if (parent.nodeName === 'OL') { - const start = parent.getAttribute('start'); - const index = Array.prototype.indexOf.call(parent.children, node); - prefix = `${start ? Number(start) + index : index + 1}. `; - } else if (parent.classList.contains("todo-list")) { - const isChecked = node.querySelector("input[type=checkbox]:checked"); - prefix = (isChecked ? "- [x] " : "- [ ] "); - } - - const result = prefix + content + (node.nextSibling && !/\n$/.test(content) ? '\n' : ''); - return result; - } - }; -} - -function buildMathFilter(): Rule { - const MATH_INLINE_PREFIX = "\\("; - const MATH_INLINE_SUFFIX = "\\)"; - - const MATH_DISPLAY_PREFIX = "\\["; - const MATH_DISPLAY_SUFFIX = "\\]"; - - return { - filter(node) { - return node.nodeName === "SPAN" && node.classList.contains("math-tex"); - }, - replacement(_, node) { - // We have to use the raw HTML text, otherwise the content is escaped too much. - const content = (node as HTMLElement).innerText; - - // Inline math - if (content.startsWith(MATH_INLINE_PREFIX) && content.endsWith(MATH_INLINE_SUFFIX)) { - return `$${content.substring(MATH_INLINE_PREFIX.length, content.length - MATH_INLINE_SUFFIX.length)}$`; - } - - // Display math - if (content.startsWith(MATH_DISPLAY_PREFIX) && content.endsWith(MATH_DISPLAY_SUFFIX)) { - return `$$${content.substring(MATH_DISPLAY_PREFIX.length, content.length - MATH_DISPLAY_SUFFIX.length)}$$`; - } - - // Unknown. - return content; - } - }; -} - -// Taken from upstream since it's not exposed. -// https://github.com/mixmark-io/turndown/blob/master/src/commonmark-rules.js -function cleanAttribute(attribute: string | null | undefined) { - return attribute ? attribute.replace(/(\n+\s*)+/g, '\n') : ''; -} - -export default { - toMarkdown -}; diff --git a/apps/server/src/services/export/zip/factory.ts b/apps/server/src/services/export/zip/factory.ts new file mode 100644 index 0000000000..975cd78650 --- /dev/null +++ b/apps/server/src/services/export/zip/factory.ts @@ -0,0 +1,31 @@ +import { type ExportFormat, ZipExportProvider, type ZipExportProviderData } from "@triliumnext/core"; +import fs from "fs"; +import path from "path"; + +import { getResourceDir, isDev } from "../../utils.js"; + +function readContentCss(): string { + const cssFile = isDev + ? path.join(require.resolve("ckeditor5/ckeditor5-content.css")) + : path.join(getResourceDir(), "ckeditor5-content.css"); + return fs.readFileSync(cssFile, "utf-8"); +} + +export async function serverZipExportProviderFactory(format: ExportFormat, data: ZipExportProviderData): Promise { + switch (format) { + case "html": { + const { default: HtmlExportProvider } = await import("@triliumnext/core/src/services/export/zip/html.js"); + return new HtmlExportProvider(data, { contentCss: readContentCss() }); + } + case "markdown": { + const { default: MarkdownExportProvider } = await import("@triliumnext/core/src/services/export/zip/markdown.js"); + return new MarkdownExportProvider(data); + } + case "share": { + const { default: ShareThemeExportProvider } = await import("./share_theme.js"); + return new ShareThemeExportProvider(data); + } + default: + throw new Error(`Unsupported export format: '${format}'`); + } +} diff --git a/apps/server/src/services/export/zip/share_theme.ts b/apps/server/src/services/export/zip/share_theme.ts index a8b3496c03..70464a5751 100644 --- a/apps/server/src/services/export/zip/share_theme.ts +++ b/apps/server/src/services/export/zip/share_theme.ts @@ -1,3 +1,4 @@ +import { ExportFormat, icon_packs as iconPackService, ZipExportProvider } from "@triliumnext/core"; import ejs from "ejs"; import fs, { readdirSync, readFileSync } from "fs"; import { convert as convertToText } from "html-to-text"; @@ -9,12 +10,10 @@ import type BBranch from "../../../becca/entities/bbranch.js"; import type BNote from "../../../becca/entities/bnote.js"; import { getClientDir, getShareThemeAssetDir } from "../../../routes/assets"; import { getDefaultTemplatePath, readTemplate, renderNoteForExport } from "../../../share/content_renderer"; -import { icon_packs as iconPackService } from "@triliumnext/core"; import log from "../../log"; import NoteMeta, { NoteMetaFile } from "../../meta/note_meta"; import { RESOURCE_DIR } from "../../resource_dir"; import { getResourceDir, isDev } from "../../utils"; -import { ExportFormat, ZipExportProvider } from "./abstract_provider.js"; const shareThemeAssetDir = getShareThemeAssetDir(); diff --git a/apps/server/src/services/utils.ts b/apps/server/src/services/utils.ts index 789f8af3d3..6d5211e545 100644 --- a/apps/server/src/services/utils.ts +++ b/apps/server/src/services/utils.ts @@ -13,6 +13,7 @@ export const isWindows11 = isWindows && osVersion[0] === 10 && osVersion[2] >= 2 export const isElectron = !!process.versions["electron"]; +/** @deprecated Use `isDev()` from `@triliumnext/core` instead. */ export const isDev = !!(process.env.TRILIUM_ENV && process.env.TRILIUM_ENV === "dev"); /** @deprecated */ diff --git a/apps/server/src/share/content_renderer.ts b/apps/server/src/share/content_renderer.ts index 6f3a2c6083..c4d81dde8a 100644 --- a/apps/server/src/share/content_renderer.ts +++ b/apps/server/src/share/content_renderer.ts @@ -1,6 +1,5 @@ import { renderSpreadsheetToHtml } from "@triliumnext/commons"; -import { sanitize } from "@triliumnext/core"; -import { icon_packs as iconPackService } from "@triliumnext/core"; +import { icon_packs as iconPackService, sanitize, utils } from "@triliumnext/core"; import { highlightAuto } from "@triliumnext/highlightjs"; import ejs from "ejs"; import escapeHtml from "escape-html"; @@ -16,7 +15,7 @@ import BNote from "../becca/entities/bnote.js"; import assetPath, { assetUrlFragment } from "../services/asset_path.js"; import log from "../services/log.js"; import options from "../services/options.js"; -import utils, { getResourceDir, isDev, safeExtractMessageAndStackFromError } from "../services/utils.js"; +import { getResourceDir, isDev } from "../services/utils.js"; import SAttachment from "./shaca/entities/sattachment.js"; import SBranch from "./shaca/entities/sbranch.js"; import type SNote from "./shaca/entities/snote.js"; @@ -224,7 +223,7 @@ function renderNoteContentInternal(note: SNote | BNote, renderArgs: RenderArgs) return ejs.render(content, opts, { includer }); } } catch (e: unknown) { - const [errMessage, errStack] = safeExtractMessageAndStackFromError(e); + const [errMessage, errStack] = utils.safeExtractMessageAndStackFromError(e); log.error(`Rendering user provided share template (${templateId}) threw exception ${errMessage} with stacktrace: ${errStack}`); } } diff --git a/apps/server/src/www.ts b/apps/server/src/www.ts index 868b3877ad..67ab16b9e4 100644 --- a/apps/server/src/www.ts +++ b/apps/server/src/www.ts @@ -1,4 +1,4 @@ -import { getMessagingProvider, utils } from "@triliumnext/core"; +import { getMessagingProvider, getPlatform, utils } from "@triliumnext/core"; import type { Express } from "express"; import fs from "fs"; import http from "http"; @@ -61,6 +61,9 @@ export default async function startTriliumServer() { const sessionParser = (await import("./routes/session_parser.js")).default; (getMessagingProvider() as WebSocketMessagingProvider).init(httpServer, sessionParser); + const ws = (await import("./services/ws.js")).default; + ws.init(); + if (utils.isElectron()) { const electronRouting = await import("./routes/electron.js"); electronRouting.default(app); @@ -155,20 +158,15 @@ function startHttpServer(app: Express) { } if (utils.isElectron()) { - import("electron").then(({ app, dialog }) => { - // Not all situations require showing an error dialog. When Trilium is already open, - // clicking the shortcut, the software icon, or the taskbar icon, or when creating a new window, - // should simply focus on the existing window or open a new one, without displaying an error message. + import("electron").then(({ app }) => { if ("code" in error && error.code === "EADDRINUSE" && (process.argv.includes("--new-window") || !app.requestSingleInstanceLock())) { console.error(message); } else { - dialog.showErrorBox("Error while initializing the server", message); + getPlatform().crash(`Error while initializing the server: ${message}`); } - process.exit(1); }); } else { - console.error(message); - process.exit(1); + getPlatform().crash(message); } }); diff --git a/apps/server/src/zip_provider.ts b/apps/server/src/zip_provider.ts index 1f858e8de4..c046baceda 100644 --- a/apps/server/src/zip_provider.ts +++ b/apps/server/src/zip_provider.ts @@ -1,6 +1,30 @@ -import type { ZipEntry, ZipProvider } from "@triliumnext/core/src/services/import/zip_provider.js"; +import type { FileStream, ZipArchive, ZipEntry, ZipProvider } from "@triliumnext/core/src/services/zip_provider.js"; +import archiver, { type Archiver } from "archiver"; +import fs from "fs"; import type { Stream } from "stream"; -import yauzl from "yauzl"; +import * as yauzl from "yauzl"; + +class NodejsZipArchive implements ZipArchive { + readonly #archive: Archiver; + + constructor() { + this.#archive = archiver("zip", { + zlib: { level: 9 } + }); + } + + append(content: string | Uint8Array, options: { name: string; date?: Date }) { + this.#archive.append(typeof content === "string" ? content : Buffer.from(content), options); + } + + pipe(destination: unknown) { + this.#archive.pipe(destination as NodeJS.WritableStream); + } + + finalize(): Promise { + return this.#archive.finalize(); + } +} function streamToBuffer(stream: Stream): Promise { const chunks: Uint8Array[] = []; @@ -12,6 +36,21 @@ function streamToBuffer(stream: Stream): Promise { } export default class NodejsZipProvider implements ZipProvider { + createZipArchive(): ZipArchive { + return new NodejsZipArchive(); + } + + createFileStream(filePath: string): FileStream { + const stream = fs.createWriteStream(filePath); + return { + destination: stream, + waitForFinish: () => new Promise((resolve, reject) => { + stream.on("finish", resolve); + stream.on("error", reject); + }) + }; + } + readZipFile( buffer: Uint8Array, processEntry: (entry: ZipEntry, readContent: () => Promise) => Promise diff --git a/packages/trilium-core/package.json b/packages/trilium-core/package.json index 3a79abdcb0..2b77ebcdff 100644 --- a/packages/trilium-core/package.json +++ b/packages/trilium-core/package.json @@ -9,6 +9,7 @@ "dependencies": { "@braintree/sanitize-url": "7.1.1", "@triliumnext/commons": "workspace:*", + "@triliumnext/turndown-plugin-gfm": "workspace:*", "async-mutex": "0.5.0", "chardet": "2.1.1", "escape-html": "1.0.3", diff --git a/packages/trilium-core/src/index.ts b/packages/trilium-core/src/index.ts index d1f713884e..3c1c2071fe 100644 --- a/packages/trilium-core/src/index.ts +++ b/packages/trilium-core/src/index.ts @@ -9,7 +9,8 @@ import { initTranslations, TranslationProvider } from "./services/i18n"; import { initSchema, initDemoArchive } from "./services/sql_init"; import appInfo from "./services/app_info"; import { type PlatformProvider, initPlatform } from "./services/platform"; -import { type ZipProvider, initZipProvider } from "./services/import/zip_provider"; +import { type ZipProvider, initZipProvider } from "./services/zip_provider"; +import { type ZipExportProviderFactory, initZipExportProviderFactory } from "./services/export/zip_export_provider_factory"; import markdown from "./services/import/markdown"; export { getLog } from "./services/log"; @@ -101,15 +102,20 @@ export type { RequestProvider, ExecOpts, CookieJar } from "./services/request"; export type * from "./meta"; export * as routeHelpers from "./routes/helpers"; -export { getZipProvider, type ZipProvider } from "./services/import/zip_provider"; +export { getZipProvider, type ZipArchive, type ZipProvider } from "./services/zip_provider"; export { default as zipImportService } from "./services/import/zip"; +export { default as zipExportService } from "./services/export/zip"; +export { type AdvancedExportOptions, type ZipExportProviderData } from "./services/export/zip/abstract_provider"; +export { ZipExportProvider } from "./services/export/zip/abstract_provider"; +export { type ZipExportProviderFactory } from "./services/export/zip_export_provider_factory"; +export { type ExportFormat } from "./meta"; export * as becca_easy_mocking from "./test/becca_easy_mocking"; export * as becca_mocking from "./test/becca_mocking"; export { default as markdownImportService } from "./services/import/markdown"; -export async function initializeCore({ dbConfig, executionContext, crypto, zip, translations, messaging, request, schema, extraAppInfo, platform, getDemoArchive }: { +export async function initializeCore({ dbConfig, executionContext, crypto, zip, zipExportProviderFactory, translations, messaging, request, schema, extraAppInfo, platform, getDemoArchive }: { dbConfig: SqlServiceParams, executionContext: ExecutionContext, crypto: CryptoProvider, @@ -117,6 +123,7 @@ export async function initializeCore({ dbConfig, executionContext, crypto, zip, translations: TranslationProvider, platform: PlatformProvider, schema: string, + zipExportProviderFactory: ZipExportProviderFactory, messaging?: MessagingProvider, request?: RequestProvider, getDemoArchive?: () => Promise, @@ -130,6 +137,7 @@ export async function initializeCore({ dbConfig, executionContext, crypto, zip, await initTranslations(translations); initCrypto(crypto); initZipProvider(zip); + initZipExportProviderFactory(zipExportProviderFactory); initContext(executionContext); initSql(new SqlService(dbConfig, getLog())); initSchema(schema); diff --git a/apps/server/src/routes/api/export.ts b/packages/trilium-core/src/routes/api/export.ts similarity index 80% rename from apps/server/src/routes/api/export.ts rename to packages/trilium-core/src/routes/api/export.ts index 9d90b49eaf..88af5f2e32 100644 --- a/apps/server/src/routes/api/export.ts +++ b/packages/trilium-core/src/routes/api/export.ts @@ -1,21 +1,21 @@ -import { NotFoundError, ValidationError } from "@triliumnext/core"; import type { Request, Response } from "express"; import becca from "../../becca/becca.js"; import opmlExportService from "../../services/export/opml.js"; import singleExportService from "../../services/export/single.js"; import zipExportService from "../../services/export/zip.js"; -import log from "../../services/log.js"; +import { getLog } from "../../services/log.js"; import TaskContext from "../../services/task_context.js"; -import { safeExtractMessageAndStackFromError } from "../../services/utils.js"; +import { safeExtractMessageAndStackFromError } from "../../services/utils/index.js"; +import { NotFoundError, ValidationError } from "../../errors.js"; -function exportBranch(req: Request<{ branchId: string; type: string; format: string; version: string; taskId: string }>, res: Response) { +async function exportBranch(req: Request<{ branchId: string; type: string; format: string; version: string; taskId: string }>, res: Response) { const { branchId, type, format, version, taskId } = req.params; const branch = becca.getBranch(branchId); if (!branch) { const message = `Cannot export branch '${branchId}' since it does not exist.`; - log.error(message); + getLog().error(message); res.setHeader("Content-Type", "text/plain").status(500).send(message); return; @@ -25,7 +25,7 @@ function exportBranch(req: Request<{ branchId: string; type: string; format: str try { if (type === "subtree" && (format === "html" || format === "markdown" || format === "share")) { - zipExportService.exportToZip(taskContext, branch, format, res); + await zipExportService.exportToZip(taskContext, branch, format, res); } else if (type === "single") { if (format !== "html" && format !== "markdown") { throw new ValidationError("Invalid export type."); @@ -41,7 +41,7 @@ function exportBranch(req: Request<{ branchId: string; type: string; format: str const message = `Export failed with following error: '${errMessage}'. More details might be in the logs.`; taskContext.reportError(message); - log.error(errMessage + errStack); + getLog().error(errMessage + errStack); res.setHeader("Content-Type", "text/plain").status(500).send(message); } diff --git a/packages/trilium-core/src/routes/api/import.ts b/packages/trilium-core/src/routes/api/import.ts index 89b3018881..15ce84860a 100644 --- a/packages/trilium-core/src/routes/api/import.ts +++ b/packages/trilium-core/src/routes/api/import.ts @@ -5,7 +5,7 @@ type ImportRequest

= Omit, "file"> & { file?: File }; import becca from "../../becca/becca.js"; import type BNote from "../../becca/entities/bnote.js"; -// import enexImportService from "../../services/import/enex.js"; +import enexImportService from "../../services/import/enex.js"; import opmlImportService from "../../services/import/opml.js"; import singleImportService from "../../services/import/single.js"; import zipImportService from "../../services/import/zip.js"; @@ -62,18 +62,18 @@ async function importNotesToBranch(req: ImportRequest<{ parentNoteId: string }>) return importResult; } } else if (extension === ".enex" && options.explodeArchives) { - throw "ENEX import is currently not supported. Please use the desktop app to import ENEX files and then sync with the server."; - // const importResult = await enexImportService.importEnex(taskContext, file, parentNote); - // if (!Array.isArray(importResult)) { - // note = importResult; - // } else { - // return importResult; - // } + const importResult = await enexImportService.importEnex(taskContext, file, parentNote); + if (!Array.isArray(importResult)) { + note = importResult; + } else { + return importResult; + } } else { note = singleImportService.importSingleFile(taskContext, file, parentNote); } } catch (e: unknown) { const [errMessage, errStack] = safeExtractMessageAndStackFromError(e); + console.warn(e); const message = `Import failed with following error: '${errMessage}'. More details might be in the logs.`; taskContext.reportError(message); diff --git a/packages/trilium-core/src/routes/api/others.ts b/packages/trilium-core/src/routes/api/others.ts index edc4ef63ab..087391ffa0 100644 --- a/packages/trilium-core/src/routes/api/others.ts +++ b/packages/trilium-core/src/routes/api/others.ts @@ -1,5 +1,31 @@ import becca from "../../becca/becca"; +import { RenderMarkdownResponse, ToMarkdownResponse } from "@triliumnext/commons"; +import type { Request } from "express"; + +import markdown from "../../services/export/markdown.js"; +import { markdownImportService, ValidationError } from "../.."; + +function renderMarkdown(req: Request) { + const { markdownContent } = req.body; + if (typeof markdownContent !== 'string') { + throw new ValidationError('markdownContent parameter is required and must be a string'); + } + return { + htmlContent: markdownImportService.renderToHtml(markdownContent, "") + } satisfies RenderMarkdownResponse; +} + +function toMarkdown(req: Request) { + const { htmlContent } = req.body; + if (typeof htmlContent !== 'string') { + throw new ValidationError('htmlContent parameter is required and must be a string'); + } + return { + markdownContent: markdown.toMarkdown(htmlContent) + } satisfies ToMarkdownResponse; +} + function getIconUsage() { const iconClassToCountMap: Record = {}; @@ -25,5 +51,7 @@ function getIconUsage() { } export default { - getIconUsage + getIconUsage, + renderMarkdown, + toMarkdown } diff --git a/packages/trilium-core/src/routes/index.ts b/packages/trilium-core/src/routes/index.ts index f5e9e3e6c7..2c276b40ce 100644 --- a/packages/trilium-core/src/routes/index.ts +++ b/packages/trilium-core/src/routes/index.ts @@ -26,6 +26,7 @@ import imageRoute from "./api/image"; import setupApiRoute from "./api/setup"; import filesRoute from "./api/files"; import importRoute from "./api/import"; +import exportRoute from "./api/export"; // TODO: Deduplicate with routes.ts const GET = "get", @@ -116,6 +117,7 @@ export function buildSharedApiRoutes({ route, asyncRoute, apiRoute, asyncApiRout apiRoute(PUT, "/api/branches/:branchId/set-prefix", branchesApiRoute.setPrefix); apiRoute(PUT, "/api/branches/set-prefix-batch", branchesApiRoute.setPrefixBatch); + // :filename is not used by trilium, but instead used for "save as" to assign a human-readable filename route(GET, "/api/revisions/:revisionId/image/:filename", [checkApiAuthOrElectron], imageRoute.returnImageFromRevision); route(GET, "/api/attachments/:attachmentId/image/:filename", [checkApiAuthOrElectron], imageRoute.returnAttachedImage); route(GET, "/api/images/:noteId/:filename", [checkApiAuthOrElectron], imageRoute.returnImageFromNote); @@ -139,8 +141,11 @@ export function buildSharedApiRoutes({ route, asyncRoute, apiRoute, asyncApiRout route(PST, "/api/sync/queue-sector/:entityName/:sector", [checkApiAuth], syncApiRoute.queueSector, apiResultHandler); route(GET, "/api/sync/stats", [], syncApiRoute.getStats, apiResultHandler); + //#region Import/export asyncRoute(PST, "/api/notes/:parentNoteId/notes-import", [checkApiAuthOrElectron, uploadMiddlewareWithErrorHandling, csrfMiddleware], importRoute.importNotesToBranch, apiResultHandler); route(PST, "/api/notes/:parentNoteId/attachments-import", [checkApiAuthOrElectron, uploadMiddlewareWithErrorHandling, csrfMiddleware], importRoute.importAttachmentsToNote, apiResultHandler); + asyncRoute(GET, "/api/branches/:branchId/export/:type/:format/:version/:taskId", [checkApiAuthOrElectron], exportRoute.exportBranch); + //#endregion apiRoute(GET, "/api/quick-search/:searchString", searchRoute.quickSearch); apiRoute(GET, "/api/search-note/:noteId", searchRoute.searchFromNote); @@ -193,7 +198,11 @@ export function buildSharedApiRoutes({ route, asyncRoute, apiRoute, asyncApiRout apiRoute(PST, "/api/bulk-action/affected-notes", bulkActionRoute.getAffectedNoteCount); apiRoute(GET, "/api/app-info", appInfoRoute.getAppInfo); + apiRoute(GET, "/api/other/icon-usage", otherRoute.getIconUsage); + apiRoute(PST, "/api/other/render-markdown", otherRoute.renderMarkdown); + apiRoute(PST, "/api/other/to-markdown", otherRoute.toMarkdown); + asyncApiRoute(GET, "/api/similar-notes/:noteId", similarNotesRoute.getSimilarNotes); apiRoute(PST, "/api/relation-map", relationMapApiRoute.getRelationMap); apiRoute(GET, "/api/recent-changes/:ancestorNoteId", recentChangesApiRoute.getRecentChanges); diff --git a/apps/server/src/services/export/markdown.spec.ts b/packages/trilium-core/src/services/export/markdown.spec.ts similarity index 100% rename from apps/server/src/services/export/markdown.spec.ts rename to packages/trilium-core/src/services/export/markdown.spec.ts diff --git a/packages/trilium-core/src/services/export/markdown.ts b/packages/trilium-core/src/services/export/markdown.ts index 5889033a5f..35e08ee11f 100644 --- a/packages/trilium-core/src/services/export/markdown.ts +++ b/packages/trilium-core/src/services/export/markdown.ts @@ -1,3 +1,8 @@ +import { gfm } from "@triliumnext/turndown-plugin-gfm"; +import Turnish, { type Rule } from "turnish"; + +let instance: Turnish | null = null; + // TODO: Move this to a dedicated file someday. export const ADMONITION_TYPE_MAPPINGS: Record = { note: "NOTE", @@ -8,3 +13,272 @@ export const ADMONITION_TYPE_MAPPINGS: Record = { }; export const DEFAULT_ADMONITION_TYPE = ADMONITION_TYPE_MAPPINGS.note; + +const fencedCodeBlockFilter: Rule = { + filter (node, options) { + return options.codeBlockStyle === "fenced" && node.nodeName === "PRE" && node.firstChild !== null && node.firstChild.nodeName === "CODE"; + }, + + replacement (content, node, options) { + if (!node.firstChild || !("getAttribute" in node.firstChild) || typeof node.firstChild.getAttribute !== "function") { + return content; + } + + const className = node.firstChild.getAttribute("class") || ""; + const language = rewriteLanguageTag((className.match(/language-(\S+)/) || [null, ""])[1]); + + return `\n\n${options.fence}${language}\n${node.firstChild.textContent}\n${options.fence}\n\n`; + } +}; + +function toMarkdown(content: string) { + if (instance === null) { + instance = new Turnish({ + headingStyle: "atx", + bulletListMarker: "*", + emDelimiter: "_", + codeBlockStyle: "fenced", + blankReplacement(_content, node) { + if (node.nodeName === "SECTION" && node.classList.contains("include-note")) { + return node.outerHTML; + } + + // Original implementation as per https://github.com/mixmark-io/turndown/blob/master/src/turndown.js. + return ("isBlock" in node && node.isBlock) ? '\n\n' : ''; + }, + }); + // Filter is heavily based on: https://github.com/mixmark-io/turndown/issues/274#issuecomment-458730974 + instance.addRule("fencedCodeBlock", fencedCodeBlockFilter); + instance.addRule("img", buildImageFilter()); + instance.addRule("admonition", buildAdmonitionFilter()); + instance.addRule("inlineLink", buildInlineLinkFilter()); + instance.addRule("figure", buildFigureFilter()); + instance.addRule("math", buildMathFilter()); + instance.addRule("li", buildListItemFilter()); + instance.use(gfm); + instance.keep([ "kbd", "sup", "sub" ]); + } + + return instance.render(content); +} + +function rewriteLanguageTag(source: string) { + if (!source) { + return source; + } + + switch (source) { + case "text-x-trilium-auto": + return ""; + case "application-javascript-env-frontend": + case "application-javascript-env-backend": + return "javascript"; + case "text-x-nginx-conf": + return "nginx"; + default: + return source.split("-").at(-1); + } +} + +// TODO: Remove once upstream delivers a fix for https://github.com/mixmark-io/turndown/issues/467. +function buildImageFilter() { + const ESCAPE_PATTERNS = { + before: /([\\*`[\]_]|(?:^[-+>])|(?:^~~~)|(?:^#{1-6}))/g, + after: /((?:^\d+(?=\.)))/ + }; + + const escapePattern = new RegExp(`(?:${ESCAPE_PATTERNS.before.source}|${ESCAPE_PATTERNS.after.source})`, 'g'); + + function escapeMarkdown (content: string) { + return content.replace(escapePattern, (match, before, after) => { + return before ? `\\${before}` : `${after}\\`; + }); + } + + function escapeLinkDestination(destination: string) { + return destination + .replace(/([()])/g, '\\$1') + .replace(/ /g, "%20"); + } + + function escapeLinkTitle (title: string) { + return title.replace(/"/g, '\\"'); + } + + const imageFilter: Rule = { + filter: "img", + replacement(content, _node) { + const node = _node as HTMLElement; + + // Preserve image verbatim if it has a width or height attribute. + if (node.hasAttribute("width") || node.hasAttribute("height")) { + return node.outerHTML; + } + + // TODO: Deduplicate with upstream. + const untypedNode = (node as any); + const alt = escapeMarkdown(cleanAttribute(untypedNode.getAttribute('alt'))); + const src = escapeLinkDestination(untypedNode.getAttribute('src') || ''); + const title = cleanAttribute(untypedNode.getAttribute('title')); + const titlePart = title ? ` "${escapeLinkTitle(title)}"` : ''; + + return src ? `![${alt}](${src}${titlePart})` : ''; + } + }; + return imageFilter; +} + +function buildAdmonitionFilter() { + function parseAdmonitionType(_node: Node) { + if (!("getAttribute" in _node)) { + return DEFAULT_ADMONITION_TYPE; + } + + const node = _node as Element; + const classList = node.getAttribute("class")?.split(" ") ?? []; + + for (const className of classList) { + if (className === "admonition") { + continue; + } + + const mappedType = ADMONITION_TYPE_MAPPINGS[className]; + if (mappedType) { + return mappedType; + } + } + + return DEFAULT_ADMONITION_TYPE; + } + + const admonitionFilter: Rule = { + filter(node, options) { + return node.nodeName === "ASIDE" && node.classList.contains("admonition"); + }, + replacement(content, node) { + // Parse the admonition type. + const admonitionType = parseAdmonitionType(node); + + content = content.replace(/^\n+|\n+$/g, ''); + content = content.replace(/^/gm, '> '); + content = `> [!${admonitionType}]\n${content}`; + + return `\n\n${content}\n\n`; + } + }; + return admonitionFilter; +} + +/** + * Variation of the original ruleset: https://github.com/mixmark-io/turndown/blob/master/src/commonmark-rules.js. + * + * Detects if the URL is a Trilium reference link and returns it verbatim if that's the case. + * + * @returns + */ +function buildInlineLinkFilter(): Rule { + return { + filter (node, options) { + return ( + options.linkStyle === 'inlined' && + node.nodeName === 'A' && + !!node.getAttribute('href') + ); + }, + + replacement (content, _node) { + const node = _node as HTMLElement; + + // Return reference links verbatim. + if (node.classList.contains("reference-link")) { + return node.outerHTML; + } + + // Otherwise treat as normal. + // TODO: Call super() somehow instead of duplicating the implementation. + let href = node.getAttribute('href'); + if (href) href = href.replace(/([()])/g, '\\$1'); + let title = cleanAttribute(node.getAttribute('title')); + if (title) title = ` "${title.replace(/"/g, '\\"')}"`; + return `[${content}](${href}${title})`; + } + }; +} + +function buildFigureFilter(): Rule { + return { + filter(node, options) { + return node.nodeName === 'FIGURE' + && node.classList.contains("image"); + }, + replacement(content, node) { + return (node as HTMLElement).outerHTML; + } + }; +} + +// Keep in line with https://github.com/mixmark-io/turndown/blob/master/src/commonmark-rules.js. +function buildListItemFilter(): Rule { + return { + filter: "li", + replacement(content, node, options) { + content = content + .trim() + .replace(/\n/gm, '\n '); // indent + let prefix = `${options.bulletListMarker} `; + const parent = node.parentNode as HTMLElement; + if (parent.nodeName === 'OL') { + const start = parent.getAttribute('start'); + const index = Array.prototype.indexOf.call(parent.children, node); + prefix = `${start ? Number(start) + index : index + 1}. `; + } else if (parent.classList.contains("todo-list")) { + const isChecked = node.querySelector("input[type=checkbox]:checked"); + prefix = (isChecked ? "- [x] " : "- [ ] "); + } + + const result = prefix + content + (node.nextSibling && !/\n$/.test(content) ? '\n' : ''); + return result; + } + }; +} + +function buildMathFilter(): Rule { + const MATH_INLINE_PREFIX = "\\("; + const MATH_INLINE_SUFFIX = "\\)"; + + const MATH_DISPLAY_PREFIX = "\\["; + const MATH_DISPLAY_SUFFIX = "\\]"; + + return { + filter(node) { + return node.nodeName === "SPAN" && node.classList.contains("math-tex"); + }, + replacement(_, node) { + // We have to use the raw HTML text, otherwise the content is escaped too much. + const content = (node as HTMLElement).innerText; + + // Inline math + if (content.startsWith(MATH_INLINE_PREFIX) && content.endsWith(MATH_INLINE_SUFFIX)) { + return `$${content.substring(MATH_INLINE_PREFIX.length, content.length - MATH_INLINE_SUFFIX.length)}$`; + } + + // Display math + if (content.startsWith(MATH_DISPLAY_PREFIX) && content.endsWith(MATH_DISPLAY_SUFFIX)) { + return `$$${content.substring(MATH_DISPLAY_PREFIX.length, content.length - MATH_DISPLAY_SUFFIX.length)}$$`; + } + + // Unknown. + return content; + } + }; +} + +// Taken from upstream since it's not exposed. +// https://github.com/mixmark-io/turndown/blob/master/src/commonmark-rules.js +function cleanAttribute(attribute: string | null | undefined) { + return attribute ? attribute.replace(/(\n+\s*)+/g, '\n') : ''; +} + +export default { + toMarkdown +}; diff --git a/apps/server/src/services/export/opml.ts b/packages/trilium-core/src/services/export/opml.ts similarity index 93% rename from apps/server/src/services/export/opml.ts rename to packages/trilium-core/src/services/export/opml.ts index 7d83a7d6f2..e6ca1ce3c2 100644 --- a/apps/server/src/services/export/opml.ts +++ b/packages/trilium-core/src/services/export/opml.ts @@ -1,9 +1,9 @@ -import { utils } from "@triliumnext/core"; import type { Response } from "express"; import becca from "../../becca/becca.js"; import type BBranch from "../../becca/entities/bbranch.js"; import type TaskContext from "../task_context.js"; +import { getContentDisposition, stripTags } from "../utils/index.js"; function exportToOpml(taskContext: TaskContext<"export">, branch: BBranch, version: string, res: Response) { if (!["1.0", "2.0"].includes(version)) { @@ -58,7 +58,7 @@ function exportToOpml(taskContext: TaskContext<"export">, branch: BBranch, versi const filename = `${branch.prefix ? `${branch.prefix} - ` : ""}${note.title}.opml`; - res.setHeader("Content-Disposition", utils.getContentDisposition(filename)); + res.setHeader("Content-Disposition", getContentDisposition(filename)); res.setHeader("Content-Type", "text/x-opml"); res.write(` @@ -82,7 +82,7 @@ function exportToOpml(taskContext: TaskContext<"export">, branch: BBranch, versi function prepareText(text: string) { const newLines = text.replace(/(]*>|)/g, "\n").replace(/ /g, " "); // nbsp isn't in XML standard (only HTML) - const stripped = utils.stripTags(newLines); + const stripped = stripTags(newLines); const escaped = escapeXmlAttribute(stripped); diff --git a/apps/server/src/services/export/single.spec.ts b/packages/trilium-core/src/services/export/single.spec.ts similarity index 100% rename from apps/server/src/services/export/single.spec.ts rename to packages/trilium-core/src/services/export/single.spec.ts diff --git a/apps/server/src/services/export/single.ts b/packages/trilium-core/src/services/export/single.ts similarity index 89% rename from apps/server/src/services/export/single.ts rename to packages/trilium-core/src/services/export/single.ts index e115e0e54b..301112825c 100644 --- a/apps/server/src/services/export/single.ts +++ b/packages/trilium-core/src/services/export/single.ts @@ -8,9 +8,10 @@ import becca from "../../becca/becca.js"; import type BBranch from "../../becca/entities/bbranch.js"; import type BNote from "../../becca/entities/bnote.js"; import type TaskContext from "../task_context.js"; -import { escapeHtml,getContentDisposition } from "../utils.js"; +import { escapeHtml,getContentDisposition } from "../utils/index.js"; import mdService from "./markdown.js"; -import type { ExportFormat } from "./zip/abstract_provider.js"; +import { ExportFormat } from "../../meta.js"; +import { encodeBase64 } from "../utils/binary.js"; function exportSingleNote(taskContext: TaskContext<"export">, branch: BBranch, format: ExportFormat, res: Response) { const note = branch.getNote(); @@ -88,11 +89,11 @@ function inlineAttachments(content: string) { } const imageContent = note.getContent(); - if (!Buffer.isBuffer(imageContent)) { + if (typeof imageContent === "string") { return match; } - const base64Content = imageContent.toString("base64"); + const base64Content = encodeBase64(imageContent); const srcValue = `data:${note.mime};base64,${base64Content}`; return `src="${srcValue}"`; @@ -105,11 +106,11 @@ function inlineAttachments(content: string) { } const attachmentContent = attachment.getContent(); - if (!Buffer.isBuffer(attachmentContent)) { + if (typeof attachmentContent === "string") { return match; } - const base64Content = attachmentContent.toString("base64"); + const base64Content = encodeBase64(attachmentContent); const srcValue = `data:${attachment.mime};base64,${base64Content}`; return `src="${srcValue}"`; @@ -122,11 +123,11 @@ function inlineAttachments(content: string) { } const attachmentContent = attachment.getContent(); - if (!Buffer.isBuffer(attachmentContent)) { + if (typeof attachmentContent === "string") { return match; } - const base64Content = attachmentContent.toString("base64"); + const base64Content = encodeBase64(attachmentContent); const hrefValue = `data:${attachment.mime};base64,${base64Content}`; return `href="${hrefValue}" download="${escapeHtml(attachment.title)}"`; diff --git a/apps/server/src/services/export/zip.ts b/packages/trilium-core/src/services/export/zip.ts similarity index 88% rename from apps/server/src/services/export/zip.ts rename to packages/trilium-core/src/services/export/zip.ts index 5efd4afbe8..8aa4914750 100644 --- a/apps/server/src/services/export/zip.ts +++ b/packages/trilium-core/src/services/export/zip.ts @@ -1,43 +1,32 @@ import { NoteType } from "@triliumnext/commons"; -import { ValidationError } from "@triliumnext/core"; -import archiver from "archiver"; -import type { Response } from "express"; -import fs from "fs"; -import path from "path"; import sanitize from "sanitize-filename"; import packageInfo from "../../../package.json" with { type: "json" }; import becca from "../../becca/becca.js"; import BBranch from "../../becca/entities/bbranch.js"; import type BNote from "../../becca/entities/bnote.js"; -import dateUtils from "../date_utils.js"; -import log from "../log.js"; -import type AttachmentMeta from "../meta/attachment_meta.js"; -import type AttributeMeta from "../meta/attribute_meta.js"; -import type NoteMeta from "../meta/note_meta.js"; -import type { NoteMetaFile } from "../meta/note_meta.js"; +import dateUtils from "../utils/date.js"; +import { getLog } from "../log.js"; import protectedSessionService from "../protected_session.js"; import TaskContext from "../task_context.js"; -import { getContentDisposition, waitForStreamToFinish } from "../utils.js"; -import { AdvancedExportOptions, type ExportFormat, ZipExportProviderData } from "./zip/abstract_provider.js"; -import HtmlExportProvider from "./zip/html.js"; -import MarkdownExportProvider from "./zip/markdown.js"; -import ShareThemeExportProvider from "./zip/share_theme.js"; +import { getZipProvider } from "../zip_provider.js"; +import { getContentDisposition } from "../utils/index" +import { AdvancedExportOptions, ZipExportProviderData } from "./zip/abstract_provider.js"; +import { getZipExportProviderFactory } from "./zip_export_provider_factory.js"; +import { AttachmentMeta, AttributeMeta, ExportFormat, NoteMeta, NoteMetaFile } from "../../meta"; +import { ValidationError } from "../../errors"; +import { extname } from "../utils/path"; -async function exportToZip(taskContext: TaskContext<"export">, branch: BBranch, format: ExportFormat, res: Response | fs.WriteStream, setHeaders = true, zipExportOptions?: AdvancedExportOptions) { - if (!["html", "markdown", "share"].includes(format)) { - throw new ValidationError(`Only 'html', 'markdown' and 'share' allowed as export format, '${format}' given`); - } - - const archive = archiver("zip", { - zlib: { level: 9 } // Sets the compression level. - }); +// eslint-disable-next-line @typescript-eslint/no-explicit-any +async function exportToZip(taskContext: TaskContext<"export">, branch: BBranch, format: ExportFormat, res: Record, setHeaders = true, zipExportOptions?: AdvancedExportOptions) { + const archive = getZipProvider().createZipArchive(); const rewriteFn = (zipExportOptions?.customRewriteLinks ? zipExportOptions?.customRewriteLinks(rewriteLinks, getNoteTargetUrl) : rewriteLinks); - const provider = buildProvider(); + const provider = await buildProvider(); + const log = getLog(); const noteIdToMeta: Record = {}; - function buildProvider() { + async function buildProvider() { const providerData: ZipExportProviderData = { getNoteTargetUrl, archive, @@ -46,16 +35,7 @@ async function exportToZip(taskContext: TaskContext<"export">, branch: BBranch, zipExportOptions }; - switch (format) { - case "html": - return new HtmlExportProvider(providerData); - case "markdown": - return new MarkdownExportProvider(providerData); - case "share": - return new ShareThemeExportProvider(providerData); - default: - throw new Error(); - } + return getZipExportProviderFactory()(format, providerData); } function getUniqueFilename(existingFileNames: Record, fileName: string) { @@ -96,7 +76,7 @@ async function exportToZip(taskContext: TaskContext<"export">, branch: BBranch, fileName = fileName.slice(0, 30 - croppedExt.length) + croppedExt; } - const existingExtension = path.extname(fileName).toLowerCase(); + const existingExtension = extname(fileName).toLowerCase(); const newExtension = provider.mapExtension(type, mime, existingExtension, format); // if the note is already named with the extension (e.g. "image.jpg"), then it's silly to append the exact same extension again @@ -343,7 +323,7 @@ async function exportToZip(taskContext: TaskContext<"export">, branch: BBranch, content = prepareContent(noteMeta.title, content, noteMeta, undefined); - archive.append(typeof content === "string" ? content : Buffer.from(content), { + archive.append(typeof content === "string" ? content : new Uint8Array(content), { name: filePathPrefix + noteMeta.dataFileName }); @@ -361,7 +341,7 @@ async function exportToZip(taskContext: TaskContext<"export">, branch: BBranch, if (noteMeta.dataFileName) { const content = prepareContent(noteMeta.title, note.getContent(), noteMeta, note); - archive.append(content as string | Buffer, { + archive.append(content as string | Uint8Array, { name: filePathPrefix + noteMeta.dataFileName, date: dateUtils.parseDateTime(note.utcDateModified) }); @@ -377,7 +357,7 @@ async function exportToZip(taskContext: TaskContext<"export">, branch: BBranch, const attachment = note.getAttachmentById(attachmentMeta.attachmentId); const content = attachment.getContent(); - archive.append(typeof content === "string" ? content : Buffer.from(content), { + archive.append(typeof content === "string" ? content : new Uint8Array(content), { name: filePathPrefix + attachmentMeta.dataFileName, date: dateUtils.parseDateTime(note.utcDateModified) }); @@ -471,7 +451,7 @@ async function exportToZip(taskContext: TaskContext<"export">, branch: BBranch, async function exportToZipFile(noteId: string, format: ExportFormat, zipFilePath: string, zipExportOptions?: AdvancedExportOptions) { - const fileOutputStream = fs.createWriteStream(zipFilePath); + const { destination, waitForFinish } = getZipProvider().createFileStream(zipFilePath); const taskContext = new TaskContext("no-progress-reporting", "export", null); const note = becca.getNote(noteId); @@ -480,10 +460,10 @@ async function exportToZipFile(noteId: string, format: ExportFormat, zipFilePath throw new ValidationError(`Note ${noteId} not found.`); } - await exportToZip(taskContext, note.getParentBranches()[0], format, fileOutputStream, false, zipExportOptions); - await waitForStreamToFinish(fileOutputStream); + await exportToZip(taskContext, note.getParentBranches()[0], format, destination as Record, false, zipExportOptions); + await waitForFinish(); - log.info(`Exported '${noteId}' with format '${format}' to '${zipFilePath}'`); + getLog().info(`Exported '${noteId}' with format '${format}' to '${zipFilePath}'`); } export default { diff --git a/apps/server/src/services/export/zip/abstract_provider.ts b/packages/trilium-core/src/services/export/zip/abstract_provider.ts similarity index 93% rename from apps/server/src/services/export/zip/abstract_provider.ts rename to packages/trilium-core/src/services/export/zip/abstract_provider.ts index dd2fdcafa2..42a388281b 100644 --- a/apps/server/src/services/export/zip/abstract_provider.ts +++ b/packages/trilium-core/src/services/export/zip/abstract_provider.ts @@ -1,13 +1,10 @@ import { NoteType } from "@triliumnext/commons"; -import { ExportFormat } from "@triliumnext/core"; -import { Archiver } from "archiver"; import mimeTypes from "mime-types"; import type BBranch from "../../../becca/entities/bbranch.js"; import type BNote from "../../../becca/entities/bnote.js"; -import type { default as NoteMeta, NoteMetaFile } from "../../meta/note_meta.js"; - -export type { ExportFormat, NoteMeta } from "@triliumnext/core"; +import { ExportFormat, NoteMeta, NoteMetaFile } from "../../../meta.js"; +import type { ZipArchive } from "../../zip_provider.js"; type RewriteLinksFn = (content: string, noteMeta: NoteMeta) => string; @@ -32,7 +29,7 @@ export interface AdvancedExportOptions { export interface ZipExportProviderData { branch: BBranch; getNoteTargetUrl: (targetNoteId: string, sourceMeta: NoteMeta) => string | null; - archive: Archiver; + archive: ZipArchive; zipExportOptions: AdvancedExportOptions | undefined; rewriteFn: RewriteLinksFn; } @@ -40,7 +37,7 @@ export interface ZipExportProviderData { export abstract class ZipExportProvider { branch: BBranch; getNoteTargetUrl: (targetNoteId: string, sourceMeta: NoteMeta) => string | null; - archive: Archiver; + archive: ZipArchive; zipExportOptions?: AdvancedExportOptions; rewriteFn: RewriteLinksFn; diff --git a/apps/server/src/services/export/zip/html.ts b/packages/trilium-core/src/services/export/zip/html.ts similarity index 89% rename from apps/server/src/services/export/zip/html.ts rename to packages/trilium-core/src/services/export/zip/html.ts index 0ba9ed2136..d5c165d23e 100644 --- a/apps/server/src/services/export/zip/html.ts +++ b/packages/trilium-core/src/services/export/zip/html.ts @@ -1,16 +1,24 @@ -import fs from "fs"; import html from "html"; -import path from "path"; -import type NoteMeta from "../../meta/note_meta.js"; -import { escapeHtml, getResourceDir, isDev } from "../../utils"; -import { ZipExportProvider } from "./abstract_provider.js"; +import { escapeHtml } from "../../utils/index"; +import { ZipExportProvider, ZipExportProviderData } from "./abstract_provider.js"; +import { NoteMeta } from "../../../meta"; + +export interface HtmlExportProviderOptions { + contentCss?: string; +} export default class HtmlExportProvider extends ZipExportProvider { private navigationMeta: NoteMeta | null = null; private indexMeta: NoteMeta | null = null; private cssMeta: NoteMeta | null = null; + private options: HtmlExportProviderOptions; + + constructor(data: ZipExportProviderData, options?: HtmlExportProviderOptions) { + super(data); + this.options = options ?? {}; + } prepareMeta(metaFile) { if (this.zipExportOptions?.skipExtraFiles) return; @@ -170,11 +178,9 @@ export default class HtmlExportProvider extends ZipExportProvider { return; } - const cssFile = isDev - ? path.join(__dirname, "../../../../../../node_modules/ckeditor5/dist/ckeditor5-content.css") - : path.join(getResourceDir(), "ckeditor5-content.css"); - const cssContent = fs.readFileSync(cssFile, "utf-8"); - this.archive.append(cssContent, { name: cssMeta.dataFileName }); + if (this.options.contentCss) { + this.archive.append(this.options.contentCss, { name: cssMeta.dataFileName }); + } } } diff --git a/apps/server/src/services/export/zip/markdown.ts b/packages/trilium-core/src/services/export/zip/markdown.ts similarity index 93% rename from apps/server/src/services/export/zip/markdown.ts rename to packages/trilium-core/src/services/export/zip/markdown.ts index 382bef8747..3079ff0dcf 100644 --- a/apps/server/src/services/export/zip/markdown.ts +++ b/packages/trilium-core/src/services/export/zip/markdown.ts @@ -1,4 +1,4 @@ -import NoteMeta from "../../meta/note_meta"; +import { NoteMeta } from "../../../meta.js"; import mdService from "../markdown.js"; import { ZipExportProvider } from "./abstract_provider.js"; diff --git a/packages/trilium-core/src/services/export/zip_export_provider_factory.ts b/packages/trilium-core/src/services/export/zip_export_provider_factory.ts new file mode 100644 index 0000000000..6f97f329da --- /dev/null +++ b/packages/trilium-core/src/services/export/zip_export_provider_factory.ts @@ -0,0 +1,15 @@ +import type { ExportFormat } from "../../meta.js"; +import type { ZipExportProvider, ZipExportProviderData } from "./zip/abstract_provider.js"; + +export type ZipExportProviderFactory = (format: ExportFormat, data: ZipExportProviderData) => Promise; + +let factory: ZipExportProviderFactory | null = null; + +export function initZipExportProviderFactory(f: ZipExportProviderFactory) { + factory = f; +} + +export function getZipExportProviderFactory(): ZipExportProviderFactory { + if (!factory) throw new Error("ZipExportProviderFactory not initialized."); + return factory; +} diff --git a/packages/trilium-core/src/services/import/enex.ts b/packages/trilium-core/src/services/import/enex.ts index 18ab4e888d..1672ff24ff 100644 --- a/packages/trilium-core/src/services/import/enex.ts +++ b/packages/trilium-core/src/services/import/enex.ts @@ -1,8 +1,6 @@ import type { AttributeType } from "@triliumnext/commons"; import { dayjs } from "@triliumnext/commons"; import sax from "sax"; -import stream from "stream"; -import { Throttle } from "stream-throttle"; import type BNote from "../../becca/entities/bnote.js"; import date_utils from "../utils/date.js"; @@ -59,8 +57,8 @@ interface Note { let note: Partial = {}; let resource: Resource; -function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentNote: BNote): Promise { - const saxStream = sax.createStream(true); +async function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentNote: BNote): Promise { + const parser = sax.parser(true); const rootNoteTitle = file.originalname.toLowerCase().endsWith(".enex") ? file.originalname.substr(0, file.originalname.length - 5) : file.originalname; @@ -138,15 +136,14 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN } } - saxStream.on("error", (e) => { - // unhandled errors will throw, since this is a proper node event emitter. + parser.onerror = (e) => { getLog().error(`error when parsing ENEX file: ${e}`); - // clear the error - (saxStream._parser as any).error = null; - saxStream._parser.resume(); - }); + // clear the error and resume + parser.error = null; + parser.resume(); + }; - saxStream.on("text", (text) => { + parser.ontext = (text) => { const currentTag = getCurrentTag(); const previousTag = getPreviousTag(); @@ -209,13 +206,9 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN } // unknown tags are just ignored } - }); + }; - saxStream.on("attribute", (attr) => { - // an attribute. attr has "name" and "value" - }); - - saxStream.on("opentag", (tag) => { + parser.onopentag = (tag) => { path.push(tag.name); if (tag.name === "note") { @@ -235,7 +228,7 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN note.resources.push(resource); } } - }); + }; const sql = getSql(); @@ -381,38 +374,29 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN updateDates(noteEntity, utcDateCreated, utcDateModified); } - saxStream.on("closetag", (tag) => { + parser.onclosetag = (tag) => { path.pop(); if (tag === "note") { saveNote(); } - }); + }; - saxStream.on("opencdata", () => { - //console.log("opencdata"); - }); - - saxStream.on("cdata", (text) => { + parser.oncdata = (text) => { note.content += text; - }); + }; - saxStream.on("closecdata", () => { - //console.log("closecdata"); - }); + const content = typeof file.buffer === "string" ? file.buffer : new TextDecoder().decode(file.buffer); - return new Promise((resolve, reject) => { - // resolve only when we parse the whole document AND saving of all notes have been finished - saxStream.on("end", () => resolve(rootNote)); + const CHUNK_SIZE = 64 * 1024; + for (let i = 0; i < content.length; i += CHUNK_SIZE) { + parser.write(content.slice(i, i + CHUNK_SIZE)); + // Yield to the event loop between chunks to avoid blocking the server. + await new Promise((resolve) => setTimeout(resolve, 0)); + } + parser.close(); - const bufferStream = new stream.PassThrough(); - bufferStream.end(file.buffer); - - bufferStream - // rate limiting to improve responsiveness during / after import - .pipe(new Throttle({ rate: 500000 })) - .pipe(saxStream); - }); + return rootNote; } function formatDateTimeToLocalDbFormat( diff --git a/packages/trilium-core/src/services/import/single.spec.ts b/packages/trilium-core/src/services/import/single.spec.ts index 4720bff43e..a749647fb6 100644 --- a/packages/trilium-core/src/services/import/single.spec.ts +++ b/packages/trilium-core/src/services/import/single.spec.ts @@ -1,6 +1,5 @@ import { beforeAll, describe, expect, it, vi } from "vitest"; import fs from "fs"; -import path from "path"; import { fileURLToPath } from "url"; import { dirname } from "path"; import becca from "../../becca/becca.js"; @@ -13,7 +12,7 @@ import { getContext } from "../context.js"; const scriptDir = dirname(fileURLToPath(import.meta.url)); async function testImport(fileName: string, mimetype: string) { - const buffer = fs.readFileSync(path.join(scriptDir, "samples", fileName)); + const buffer = fs.readFileSync(`${scriptDir}/samples/${fileName}`); const taskContext = TaskContext.getInstance("import-mdx", "importNotes", { textImportedAsText: true, codeImportedAsCode: true diff --git a/packages/trilium-core/src/services/import/zip.spec.ts b/packages/trilium-core/src/services/import/zip.spec.ts index 397bce0db1..4921a529ff 100644 --- a/packages/trilium-core/src/services/import/zip.spec.ts +++ b/packages/trilium-core/src/services/import/zip.spec.ts @@ -1,6 +1,5 @@ import { beforeAll, describe, expect, it, vi } from "vitest"; import fs from "fs"; -import path from "path"; import { fileURLToPath } from "url"; import { dirname } from "path"; import zip, { removeTriliumTags } from "./zip.js"; @@ -13,7 +12,7 @@ import { getContext } from "../context.js"; const scriptDir = dirname(fileURLToPath(import.meta.url)); async function testImport(fileName: string) { - const mdxSample = fs.readFileSync(path.join(scriptDir, "samples", fileName)); + const mdxSample = fs.readFileSync(`${scriptDir}/samples/${fileName}`); const taskContext = TaskContext.getInstance("import-mdx", "importNotes", { textImportedAsText: true }); diff --git a/packages/trilium-core/src/services/import/zip.ts b/packages/trilium-core/src/services/import/zip.ts index 94df44ce32..c51a561196 100644 --- a/packages/trilium-core/src/services/import/zip.ts +++ b/packages/trilium-core/src/services/import/zip.ts @@ -1,6 +1,6 @@ import { ALLOWED_NOTE_TYPES, type NoteType } from "@triliumnext/commons"; import { basename, dirname } from "../utils/path.js"; -import { getZipProvider } from "./zip_provider.js"; +import { getZipProvider } from "../zip_provider.js"; import becca from "../../becca/becca.js"; import BAttachment from "../../becca/entities/battachment.js"; diff --git a/packages/trilium-core/src/services/log.ts b/packages/trilium-core/src/services/log.ts index e3cc68af2e..2447811c9b 100644 --- a/packages/trilium-core/src/services/log.ts +++ b/packages/trilium-core/src/services/log.ts @@ -14,7 +14,8 @@ export default class LogService { banner(message: string | undefined) { if (!message) return; - const maxContent = 76; // 80 - 4 (border + padding) + const termWidth = (typeof process !== "undefined" && process.stdout?.columns) || 80; + const maxContent = termWidth - 4; // border + padding const words = message.split(" "); const lines: string[] = []; let current = ""; diff --git a/packages/trilium-core/src/services/notes.ts b/packages/trilium-core/src/services/notes.ts index 69f7601351..502ef1beb1 100644 --- a/packages/trilium-core/src/services/notes.ts +++ b/packages/trilium-core/src/services/notes.ts @@ -2,7 +2,6 @@ import { type AttachmentRow, type AttributeRow, type BranchRow, dayjs, type Note import fs from "fs"; import html2plaintext from "html2plaintext"; import { t } from "i18next"; -import path from "path"; import url from "url"; import becca from "../becca/becca.js"; @@ -28,6 +27,7 @@ import { getSql } from "./sql/index.js"; import { sanitizeHtml } from "./sanitizer.js"; import { ValidationError } from "../errors.js"; import * as cls from "./context.js"; +import { basename } from "./utils/path.js"; interface FoundLink { name: "imageLink" | "internalLink" | "includeNoteLink" | "relationMapLink"; @@ -552,7 +552,7 @@ async function downloadImage(noteId: string, imageUrl: string) { } const parsedUrl = url.parse(unescapedUrl); - const title = path.basename(parsedUrl.pathname || ""); + const title = basename(parsedUrl.pathname || ""); const attachment = imageService.saveImageToAttachment(noteId, imageBuffer, title, true, true); diff --git a/packages/trilium-core/src/services/utils/index.ts b/packages/trilium-core/src/services/utils/index.ts index ec64a693f5..a1427becbf 100644 --- a/packages/trilium-core/src/services/utils/index.ts +++ b/packages/trilium-core/src/services/utils/index.ts @@ -8,6 +8,7 @@ import unescape from "unescape"; import { basename, extname } from "./path"; import { NoteMeta } from "../../meta"; +export function isDev() { return getPlatform().getEnv("TRILIUM_ENV") === "dev"; } export function isElectron() { return getPlatform().isElectron; } export function isMac() { return getPlatform().isMac; } export function isWindows() { return getPlatform().isWindows; } @@ -198,7 +199,15 @@ export function randomSecureToken(bytes = 32) { } export function safeExtractMessageAndStackFromError(err: unknown): [errMessage: string, errStack: string | undefined] { - return (err instanceof Error) ? [err.message, err.stack] as const : ["Unknown Error", undefined] as const; + if (err instanceof Error) { + return [err.message, err.stack] as const; + } + + if (typeof err === "string") { + return [err, undefined] as const; + } + + return ["Unknown Error", undefined] as const; } export function isEmptyOrWhitespace(str: string | null | undefined) { diff --git a/packages/trilium-core/src/services/import/zip_provider.ts b/packages/trilium-core/src/services/zip_provider.ts similarity index 51% rename from packages/trilium-core/src/services/import/zip_provider.ts rename to packages/trilium-core/src/services/zip_provider.ts index 7b6cc9d04e..cd75d93d37 100644 --- a/packages/trilium-core/src/services/import/zip_provider.ts +++ b/packages/trilium-core/src/services/zip_provider.ts @@ -2,6 +2,24 @@ export interface ZipEntry { fileName: string; } +export interface ZipArchiveEntryOptions { + name: string; + date?: Date; +} + +export interface ZipArchive { + append(content: string | Uint8Array, options: ZipArchiveEntryOptions): void; + pipe(destination: unknown): void; + finalize(): Promise; +} + +export interface FileStream { + /** An opaque writable destination that can be passed to {@link ZipArchive.pipe}. */ + destination: unknown; + /** Resolves when the stream has finished writing (or rejects on error). */ + waitForFinish(): Promise; +} + export interface ZipProvider { /** * Iterates over every entry in a ZIP buffer, calling `processEntry` for each one. @@ -11,6 +29,11 @@ export interface ZipProvider { buffer: Uint8Array, processEntry: (entry: ZipEntry, readContent: () => Promise) => Promise ): Promise; + + createZipArchive(): ZipArchive; + + /** Creates a writable file stream for the given path. */ + createFileStream(filePath: string): FileStream; } let zipProvider: ZipProvider | null = null; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 63bc0cd559..c94c380a4a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -803,9 +803,6 @@ importers: '@triliumnext/highlightjs': specifier: workspace:* version: link:../../packages/highlightjs - '@triliumnext/turndown-plugin-gfm': - specifier: workspace:* - version: link:../../packages/turndown-plugin-gfm '@types/archiver': specifier: 7.0.0 version: 7.0.0 @@ -1704,6 +1701,9 @@ importers: '@triliumnext/commons': specifier: workspace:* version: link:../commons + '@triliumnext/turndown-plugin-gfm': + specifier: workspace:* + version: link:../turndown-plugin-gfm async-mutex: specifier: 0.5.0 version: 0.5.0 @@ -17587,8 +17587,6 @@ snapshots: '@ckeditor/ckeditor5-utils': 47.6.1 '@ckeditor/ckeditor5-widget': 47.6.1 ckeditor5: 47.6.1 - transitivePeerDependencies: - - supports-color '@ckeditor/ckeditor5-html-embed@47.6.1': dependencies: @@ -17598,8 +17596,6 @@ snapshots: '@ckeditor/ckeditor5-utils': 47.6.1 '@ckeditor/ckeditor5-widget': 47.6.1 ckeditor5: 47.6.1 - transitivePeerDependencies: - - supports-color '@ckeditor/ckeditor5-html-support@47.6.1': dependencies: @@ -17615,8 +17611,6 @@ snapshots: '@ckeditor/ckeditor5-widget': 47.6.1 ckeditor5: 47.6.1 es-toolkit: 1.39.5 - transitivePeerDependencies: - - supports-color '@ckeditor/ckeditor5-icons@47.6.1': {} @@ -17634,8 +17628,6 @@ snapshots: '@ckeditor/ckeditor5-widget': 47.6.1 ckeditor5: 47.6.1 es-toolkit: 1.39.5 - transitivePeerDependencies: - - supports-color '@ckeditor/ckeditor5-import-word@47.6.1': dependencies: @@ -17659,8 +17651,6 @@ snapshots: '@ckeditor/ckeditor5-ui': 47.6.1 '@ckeditor/ckeditor5-utils': 47.6.1 ckeditor5: 47.6.1 - transitivePeerDependencies: - - supports-color '@ckeditor/ckeditor5-inspector@5.0.0': {} @@ -17671,8 +17661,6 @@ snapshots: '@ckeditor/ckeditor5-ui': 47.6.1 '@ckeditor/ckeditor5-utils': 47.6.1 ckeditor5: 47.6.1 - transitivePeerDependencies: - - supports-color '@ckeditor/ckeditor5-line-height@47.6.1': dependencies: @@ -17697,8 +17685,6 @@ snapshots: '@ckeditor/ckeditor5-widget': 47.6.1 ckeditor5: 47.6.1 es-toolkit: 1.39.5 - transitivePeerDependencies: - - supports-color '@ckeditor/ckeditor5-list-multi-level@47.6.1': dependencies: