From 2ac85a1d1c9268f28dec4b8c47782832c737e3af Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sun, 22 Mar 2026 19:32:51 +0200 Subject: [PATCH] chore(core): add provider for requests --- .../src/lightweight/request_provider.ts | 94 +++++ .../src/local-server-worker.ts | 5 + apps/server/src/main.ts | 2 + apps/server/src/services/request.ts | 331 +++++++++--------- apps/server/src/services/setup.ts | 2 +- apps/server/src/services/sync.ts | 4 +- packages/trilium-core/src/index.ts | 9 +- packages/trilium-core/src/services/request.ts | 54 ++- 8 files changed, 327 insertions(+), 174 deletions(-) create mode 100644 apps/client-standalone/src/lightweight/request_provider.ts diff --git a/apps/client-standalone/src/lightweight/request_provider.ts b/apps/client-standalone/src/lightweight/request_provider.ts new file mode 100644 index 0000000000..46c0988d54 --- /dev/null +++ b/apps/client-standalone/src/lightweight/request_provider.ts @@ -0,0 +1,94 @@ +import type { ExecOpts, RequestProvider } from "@triliumnext/core"; + +/** + * Fetch-based implementation of RequestProvider for browser environments. + * + * Uses the Fetch API instead of Node's http/https modules. + * Proxy support is not available in browsers, so the proxy option is ignored. + */ +export default class FetchRequestProvider implements RequestProvider { + + async exec(opts: ExecOpts): Promise { + const paging = opts.paging || { + pageCount: 1, + pageIndex: 0, + requestId: "n/a" + }; + + const headers: Record = { + "Content-Type": paging.pageCount === 1 ? "application/json" : "text/plain", + "pageCount": String(paging.pageCount), + "pageIndex": String(paging.pageIndex), + "requestId": paging.requestId + }; + + if (opts.cookieJar?.header) { + headers["Cookie"] = opts.cookieJar.header; + } + + if (opts.auth?.password) { + headers["trilium-cred"] = btoa(`dummy:${opts.auth.password}`); + } + + let body: string | undefined; + if (opts.body) { + body = typeof opts.body === "object" ? JSON.stringify(opts.body) : opts.body; + } + + const controller = new AbortController(); + const timeoutId = opts.timeout + ? setTimeout(() => controller.abort(), opts.timeout) + : undefined; + + try { + const response = await fetch(opts.url, { + method: opts.method, + headers, + body, + signal: controller.signal + }); + + // Handle set-cookie from response (limited in browser, but keep for API compat) + if (opts.cookieJar) { + const setCookie = response.headers.get("set-cookie"); + if (setCookie) { + opts.cookieJar.header = setCookie; + } + } + + if ([200, 201, 204].includes(response.status)) { + const text = await response.text(); + return text.trim() ? JSON.parse(text) : null; + } else { + const text = await response.text(); + let errorMessage: string; + try { + const json = JSON.parse(text); + errorMessage = json?.message || ""; + } catch { + errorMessage = text.substring(0, 100); + } + throw new Error(`Request to ${opts.method} ${opts.url} failed, error: ${response.status} ${response.statusText} ${errorMessage}`); + } + } catch (e: any) { + if (e.name === "AbortError") { + throw new Error(`Request to ${opts.method} ${opts.url} failed, error: timeout after ${opts.timeout}ms`); + } + throw e; + } finally { + if (timeoutId) { + clearTimeout(timeoutId); + } + } + } + + async getImage(imageUrl: string): Promise { + const response = await fetch(imageUrl); + + if (!response.ok) { + throw new Error(`Request to GET ${imageUrl} failed, error: ${response.status} ${response.statusText}`); + } + + return await response.arrayBuffer(); + } +} diff --git a/apps/client-standalone/src/local-server-worker.ts b/apps/client-standalone/src/local-server-worker.ts index d70e9f5d97..02d6d46bdc 100644 --- a/apps/client-standalone/src/local-server-worker.ts +++ b/apps/client-standalone/src/local-server-worker.ts @@ -55,6 +55,7 @@ let BrowserSqlProvider: typeof import('./lightweight/sql_provider').default; let WorkerMessagingProvider: typeof import('./lightweight/messaging_provider').default; let BrowserExecutionContext: typeof import('./lightweight/cls_provider').default; let BrowserCryptoProvider: typeof import('./lightweight/crypto_provider').default; +let FetchRequestProvider: typeof import('./lightweight/request_provider').default; let translationProvider: typeof import('./lightweight/translation_provider').default; let createConfiguredRouter: typeof import('./lightweight/browser_routes').createConfiguredRouter; @@ -79,6 +80,7 @@ async function loadModules(): Promise { messagingModule, clsModule, cryptoModule, + requestModule, translationModule, routesModule ] = await Promise.all([ @@ -86,6 +88,7 @@ async function loadModules(): Promise { import('./lightweight/messaging_provider.js'), import('./lightweight/cls_provider.js'), import('./lightweight/crypto_provider.js'), + import('./lightweight/request_provider.js'), import('./lightweight/translation_provider.js'), import('./lightweight/browser_routes.js') ]); @@ -94,6 +97,7 @@ async function loadModules(): Promise { WorkerMessagingProvider = messagingModule.default; BrowserExecutionContext = clsModule.default; BrowserCryptoProvider = cryptoModule.default; + FetchRequestProvider = requestModule.default; translationProvider = translationModule.default; createConfiguredRouter = routesModule.createConfiguredRouter; @@ -152,6 +156,7 @@ async function initialize(): Promise { executionContext: new BrowserExecutionContext(), crypto: new BrowserCryptoProvider(), messaging: messagingProvider!, + request: new FetchRequestProvider(), translations: translationProvider, dbConfig: { provider: sqlProvider!, diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index d32ff11afa..3b3cc8aee9 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -8,6 +8,7 @@ import path from "path"; import ClsHookedExecutionContext from "./cls_provider.js"; import NodejsCryptoProvider from "./crypto_provider.js"; +import NodeRequestProvider from "./services/request.js"; import dataDirs from "./services/data_dir.js"; import BetterSqlite3Provider from "./sql_provider.js"; @@ -45,6 +46,7 @@ async function startApplication() { }, }, crypto: new NodejsCryptoProvider(), + request: new NodeRequestProvider(), executionContext: new ClsHookedExecutionContext(), translations: (await import("./services/i18n.js")).initializeTranslations, extraAppInfo: { diff --git a/apps/server/src/services/request.ts b/apps/server/src/services/request.ts index 688c617531..96fd8d8f7c 100644 --- a/apps/server/src/services/request.ts +++ b/apps/server/src/services/request.ts @@ -1,10 +1,10 @@ "use strict"; +import type { ExecOpts, RequestProvider } from "@triliumnext/core"; import { isElectron } from "./utils.js"; import log from "./log.js"; import url from "url"; import syncOptions from "./sync_options.js"; -import type { ExecOpts } from "./request_interface.js"; // this service provides abstraction over node's HTTP/HTTPS and electron net.client APIs // this allows supporting system proxy @@ -33,160 +33,19 @@ interface Client { request(opts: ClientOpts): Request; } -async function exec(opts: ExecOpts): Promise { - const client = getClient(opts); - - // hack for cases where electron.net does not work, but we don't want to set proxy - if (opts.proxy === "noproxy") { - opts.proxy = null; - } - - const paging = opts.paging || { - pageCount: 1, - pageIndex: 0, - requestId: "n/a" - }; - - const proxyAgent = await getProxyAgent(opts); - const parsedTargetUrl = url.parse(opts.url); - - return new Promise(async (resolve, reject) => { - try { - const headers: Record = { - Cookie: (opts.cookieJar && opts.cookieJar.header) || "", - "Content-Type": paging.pageCount === 1 ? "application/json" : "text/plain", - pageCount: paging.pageCount, - pageIndex: paging.pageIndex, - requestId: paging.requestId - }; - - if (opts.auth) { - headers["trilium-cred"] = Buffer.from(`dummy:${opts.auth.password}`).toString("base64"); - } - - const request = (await client).request({ - method: opts.method, - // url is used by electron net module - url: opts.url, - // 4 fields below are used by http and https node modules - protocol: parsedTargetUrl.protocol, - host: parsedTargetUrl.hostname, - port: parsedTargetUrl.port, - path: parsedTargetUrl.path, - timeout: opts.timeout, // works only for node.js client - headers, - agent: proxyAgent - }); - - request.on("error", (err) => reject(generateError(opts, err))); - - request.on("response", (response) => { - if (opts.cookieJar && response.headers["set-cookie"]) { - opts.cookieJar.header = response.headers["set-cookie"]; - } - - let responseStr = ""; - let chunks: Buffer[] = []; - - response.on("data", (chunk: Buffer) => chunks.push(chunk)); - - response.on("end", () => { - // use Buffer instead of string concatenation to avoid implicit decoding for each chunk - // decode the entire data chunks explicitly as utf-8 - responseStr = Buffer.concat(chunks).toString("utf-8"); - - if ([200, 201, 204].includes(response.statusCode)) { - try { - const jsonObj = responseStr.trim() ? JSON.parse(responseStr) : null; - - resolve(jsonObj); - } catch (e: any) { - log.error(`Failed to deserialize sync response: ${responseStr}`); - - reject(generateError(opts, e.message)); - } - } else { - let errorMessage; - - try { - const jsonObj = JSON.parse(responseStr); - - errorMessage = jsonObj?.message || ""; - } catch (e: any) { - errorMessage = responseStr.substr(0, Math.min(responseStr.length, 100)); - } - - reject(generateError(opts, `${response.statusCode} ${response.statusMessage} ${errorMessage}`)); - } - }); - }); - - let payload; - - if (opts.body) { - payload = typeof opts.body === "object" ? JSON.stringify(opts.body) : opts.body; - } - - request.end(payload as string); - } catch (e: any) { - reject(generateError(opts, e.message)); - } - }); -} - -async function getImage(imageUrl: string): Promise { - const proxyConf = syncOptions.getSyncProxy(); - const opts: ClientOpts = { - method: "GET", - url: imageUrl, - proxy: proxyConf !== "noproxy" ? proxyConf : null - }; - - const client = await getClient(opts); - const proxyAgent = await getProxyAgent(opts); - const parsedTargetUrl = url.parse(opts.url); - - return new Promise((resolve, reject) => { - try { - const request = client.request({ - method: opts.method, - // url is used by electron net module - url: opts.url, - // 4 fields below are used by http and https node modules - protocol: parsedTargetUrl.protocol, - host: parsedTargetUrl.hostname, - port: parsedTargetUrl.port, - path: parsedTargetUrl.path, - timeout: opts.timeout, // works only for the node client - headers: {}, - agent: proxyAgent - }); - - request.on("error", (err) => reject(generateError(opts, err))); - - request.on("abort", (err) => reject(generateError(opts, err))); - - request.on("response", (response) => { - if (![200, 201, 204].includes(response.statusCode)) { - reject(generateError(opts, `${response.statusCode} ${response.statusMessage}`)); - } - - const chunks: Buffer[] = []; - - response.on("data", (chunk: Buffer) => chunks.push(chunk)); - response.on("end", () => resolve(Buffer.concat(chunks))); - }); - - request.end(undefined); - } catch (e: any) { - reject(generateError(opts, e.message)); - } - }); -} - const HTTP = "http:", HTTPS = "https:"; +function generateError( + opts: { + method: string; + url: string; + }, + message: string +) { + return new Error(`Request to ${opts.method} ${opts.url} failed, error: ${message}`); +} + async function getProxyAgent(opts: ClientOpts) { if (!opts.proxy) { return null; @@ -219,17 +78,159 @@ async function getClient(opts: ClientOpts): Promise { } } -function generateError( - opts: { - method: string; - url: string; - }, - message: string -) { - return new Error(`Request to ${opts.method} ${opts.url} failed, error: ${message}`); -} +export default class NodeRequestProvider implements RequestProvider { -export default { - exec, - getImage -}; + async exec(opts: ExecOpts): Promise { + const client = getClient(opts); + + // hack for cases where electron.net does not work, but we don't want to set proxy + if (opts.proxy === "noproxy") { + opts.proxy = null; + } + + const paging = opts.paging || { + pageCount: 1, + pageIndex: 0, + requestId: "n/a" + }; + + const proxyAgent = await getProxyAgent(opts); + const parsedTargetUrl = url.parse(opts.url); + + return new Promise(async (resolve, reject) => { + try { + const headers: Record = { + Cookie: (opts.cookieJar && opts.cookieJar.header) || "", + "Content-Type": paging.pageCount === 1 ? "application/json" : "text/plain", + pageCount: paging.pageCount, + pageIndex: paging.pageIndex, + requestId: paging.requestId + }; + + if (opts.auth) { + headers["trilium-cred"] = Buffer.from(`dummy:${opts.auth.password}`).toString("base64"); + } + + const request = (await client).request({ + method: opts.method, + // url is used by electron net module + url: opts.url, + // 4 fields below are used by http and https node modules + protocol: parsedTargetUrl.protocol, + host: parsedTargetUrl.hostname, + port: parsedTargetUrl.port, + path: parsedTargetUrl.path, + timeout: opts.timeout, // works only for node.js client + headers, + agent: proxyAgent + }); + + request.on("error", (err) => reject(generateError(opts, err))); + + request.on("response", (response) => { + if (opts.cookieJar && response.headers["set-cookie"]) { + opts.cookieJar.header = response.headers["set-cookie"]; + } + + let responseStr = ""; + let chunks: Buffer[] = []; + + response.on("data", (chunk: Buffer) => chunks.push(chunk)); + + response.on("end", () => { + // use Buffer instead of string concatenation to avoid implicit decoding for each chunk + // decode the entire data chunks explicitly as utf-8 + responseStr = Buffer.concat(chunks).toString("utf-8"); + + if ([200, 201, 204].includes(response.statusCode)) { + try { + const jsonObj = responseStr.trim() ? JSON.parse(responseStr) : null; + + resolve(jsonObj); + } catch (e: any) { + log.error(`Failed to deserialize sync response: ${responseStr}`); + + reject(generateError(opts, e.message)); + } + } else { + let errorMessage; + + try { + const jsonObj = JSON.parse(responseStr); + + errorMessage = jsonObj?.message || ""; + } catch (e: any) { + errorMessage = responseStr.substr(0, Math.min(responseStr.length, 100)); + } + + reject(generateError(opts, `${response.statusCode} ${response.statusMessage} ${errorMessage}`)); + } + }); + }); + + let payload; + + if (opts.body) { + payload = typeof opts.body === "object" ? JSON.stringify(opts.body) : opts.body; + } + + request.end(payload as string); + } catch (e: any) { + reject(generateError(opts, e.message)); + } + }); + } + + async getImage(imageUrl: string): Promise { + const proxyConf = syncOptions.getSyncProxy(); + const opts: ClientOpts = { + method: "GET", + url: imageUrl, + proxy: proxyConf !== "noproxy" ? proxyConf : null + }; + + const client = await getClient(opts); + const proxyAgent = await getProxyAgent(opts); + const parsedTargetUrl = url.parse(opts.url); + + return new Promise((resolve, reject) => { + try { + const request = client.request({ + method: opts.method, + // url is used by electron net module + url: opts.url, + // 4 fields below are used by http and https node modules + protocol: parsedTargetUrl.protocol, + host: parsedTargetUrl.hostname, + port: parsedTargetUrl.port, + path: parsedTargetUrl.path, + timeout: opts.timeout, // works only for the node client + headers: {}, + agent: proxyAgent + }); + + request.on("error", (err) => reject(generateError(opts, err))); + + request.on("abort", (err) => reject(generateError(opts, err))); + + request.on("response", (response) => { + if (![200, 201, 204].includes(response.statusCode)) { + reject(generateError(opts, `${response.statusCode} ${response.statusMessage}`)); + } + + const chunks: Buffer[] = []; + + response.on("data", (chunk: Buffer) => chunks.push(chunk)); + response.on("end", () => { + const buf = Buffer.concat(chunks); + resolve(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength)); + }); + }); + + request.end(undefined); + } catch (e: any) { + reject(generateError(opts, e.message)); + } + }); + } +} diff --git a/apps/server/src/services/setup.ts b/apps/server/src/services/setup.ts index a20fb670f9..8acf314731 100644 --- a/apps/server/src/services/setup.ts +++ b/apps/server/src/services/setup.ts @@ -3,7 +3,7 @@ import log from "./log.js"; import sqlInit from "./sql_init.js"; import optionService from "./options.js"; import syncOptions from "./sync_options.js"; -import request from "./request.js"; +import { request } from "@triliumnext/core"; import appInfo from "./app_info.js"; import { timeLimit } from "./utils.js"; import becca from "../becca/becca.js"; diff --git a/apps/server/src/services/sync.ts b/apps/server/src/services/sync.ts index 1b446c9bf3..3a0cb1a00c 100644 --- a/apps/server/src/services/sync.ts +++ b/apps/server/src/services/sync.ts @@ -1,5 +1,5 @@ import type { EntityChange, EntityChangeRecord, EntityRow } from "@triliumnext/commons"; -import { becca_loader, binary_utils, entity_constructor, getInstanceId } from "@triliumnext/core"; +import { becca_loader, binary_utils, entity_constructor, getInstanceId, request, type CookieJar, type ExecOpts } from "@triliumnext/core"; import becca from "../becca/becca.js"; import appInfo from "./app_info.js"; @@ -10,8 +10,6 @@ import dateUtils from "./date_utils.js"; import entityChangesService from "./entity_changes.js"; import log from "./log.js"; import optionService from "./options.js"; -import request from "./request.js"; -import type { CookieJar, ExecOpts } from "./request_interface.js"; import setupService from "./setup.js"; import sql from "./sql.js"; import syncMutexService from "./sync_mutex.js"; diff --git a/packages/trilium-core/src/index.ts b/packages/trilium-core/src/index.ts index e42bba571f..a299f3888f 100644 --- a/packages/trilium-core/src/index.ts +++ b/packages/trilium-core/src/index.ts @@ -4,6 +4,7 @@ import { getLog, initLog } from "./services/log"; import { initSql } from "./services/sql/index"; import { SqlService, SqlServiceParams } from "./services/sql/sql"; import { initMessaging, MessagingProvider } from "./services/messaging/index"; +import { initRequest, RequestProvider } from "./services/request"; import { initTranslations, TranslationProvider } from "./services/i18n"; import appInfo from "./services/app_info"; @@ -74,13 +75,16 @@ export type { NoteParams } from "./services/notes"; export * as sanitize from "./services/sanitizer"; export * as routes from "./routes"; export { default as ws } from "./services/ws"; +export { default as request } from "./services/request"; +export type { RequestProvider, ExecOpts, CookieJar } from "./services/request"; -export async function initializeCore({ dbConfig, executionContext, crypto, translations, messaging, extraAppInfo }: { +export async function initializeCore({ dbConfig, executionContext, crypto, translations, messaging, request, extraAppInfo }: { dbConfig: SqlServiceParams, executionContext: ExecutionContext, crypto: CryptoProvider, translations: TranslationProvider, messaging?: MessagingProvider, + request?: RequestProvider, extraAppInfo?: { nodeVersion: string; dataDirectory: string; @@ -95,4 +99,7 @@ export async function initializeCore({ dbConfig, executionContext, crypto, trans if (messaging) { initMessaging(messaging); } + if (request) { + initRequest(request); + } }; diff --git a/packages/trilium-core/src/services/request.ts b/packages/trilium-core/src/services/request.ts index bf67168e0c..ac0a5ec944 100644 --- a/packages/trilium-core/src/services/request.ts +++ b/packages/trilium-core/src/services/request.ts @@ -1,5 +1,51 @@ -export default { - getImage(url: string) { - console.warn("Image download ignored ", url); - } +export interface CookieJar { + header?: string; } + +export interface ExecOpts { + proxy: string | null; + method: string; + url: string; + paging?: { + pageCount: number; + pageIndex: number; + requestId: string; + }; + cookieJar?: CookieJar; + auth?: { + password?: string; + }; + timeout: number; + body?: string | {}; +} + +export interface RequestProvider { + exec(opts: ExecOpts): Promise; + getImage(imageUrl: string): Promise; +} + +let requestProvider: RequestProvider | null = null; + +export function initRequest(provider: RequestProvider): void { + requestProvider = provider; +} + +export function getRequestProvider(): RequestProvider { + if (!requestProvider) { + throw new Error("Request provider not initialized. Call initRequest() first."); + } + return requestProvider; +} + +export function isRequestInitialized(): boolean { + return requestProvider !== null; +} + +export default { + exec(opts: ExecOpts): Promise { + return getRequestProvider().exec(opts); + }, + getImage(imageUrl: string): Promise { + return getRequestProvider().getImage(imageUrl); + } +};