Merge branch 'develop' into feat/add-rootless-dockerfiles

This commit is contained in:
Elian Doran
2025-05-27 19:34:49 +03:00
committed by GitHub
237 changed files with 2645 additions and 2302 deletions

Binary file not shown.

File diff suppressed because one or more lines are too long

View File

@@ -275,9 +275,9 @@
content via the injected <code>now</code> and <code>parentNote</code> variables.</p>
<p>Examples:</p>
<ul>
<li><code>${parentNote.getLabel('authorName')}'s literary works</code>
<li><code><span class="math-tex">\({parentNote.getLabel('authorName')}'s literary works</span></code>
</li>
<li><code>Log for ${now.format('YYYY-MM-DD HH:mm:ss')}</code>
<li><code>Log for \){now.format('YYYY-MM-DD HH:mm:ss')}</code>
</li>
<li>to mirror the parent's template.</li>
</ul>

View File

@@ -1,5 +1,5 @@
<p>Trilium supports configuration via a file named <code>config.ini</code> and
environment variables. Please review the file named <a href="https://github.com/TriliumNext/Notes/blob/develop/config-sample.ini">config-sample.ini</a> in
environment variables. Please review the file named <a href="https://github.com/TriliumNext/Notes/blob/develop/apps/server/src/assets/config-sample.ini">config-sample.ini</a> in
the <a href="https://github.com/TriliumNext/Notes">Notes</a> repository to
see what values are supported.</p>
<p>You can provide the same values via environment variables instead of the <code>config.ini</code> file,

View File

@@ -0,0 +1,87 @@
<p>&nbsp;</p>
<h1><strong>Trilium Metrics API</strong></h1>
<p>The Trilium metrics API provides comprehensive monitoring data about your
Trilium instance, designed for external monitoring systems like Prometheus.</p>
<h2><strong>Endpoint</strong></h2>
<ul>
<li><strong>URL</strong>: <code>/etapi/metrics</code>
</li>
<li><strong>Method</strong>: <code>GET</code>
</li>
<li><strong>Authentication</strong>: ETAPI token required</li>
<li><strong>Default Format</strong>: Prometheus text format</li>
</ul>
<h2><strong>Authentication</strong></h2>
<p>You need an ETAPI token to access the metrics endpoint. Get one by:</p><pre><code class="language-text-x-trilium-auto"># Get an ETAPI token
curl -X POST http://localhost:8080/etapi/auth/login \
-H "Content-Type: application/json" \
-d '{"password": "your_password"}'
</code></pre>
<h2><strong>Usage</strong></h2>
<h3><strong>Prometheus Format (Default)</strong></h3><pre><code class="language-text-x-trilium-auto">curl -H "Authorization: YOUR_ETAPI_TOKEN" \
http://localhost:8080/etapi/metrics
</code></pre>
<p>Returns metrics in Prometheus text format:</p><pre><code class="language-text-x-trilium-auto"># HELP trilium_info Trilium instance information
# TYPE trilium_info gauge
trilium_info{version="0.91.6",db_version="231",node_version="v18.17.0"} 1 1701432000
# HELP trilium_notes_total Total number of notes including deleted
# TYPE trilium_notes_total gauge
trilium_notes_total 1234 1701432000
</code></pre>
<h3><strong>JSON Format</strong></h3><pre><code class="language-text-x-trilium-auto">curl -H "Authorization: YOUR_ETAPI_TOKEN" \
"http://localhost:8080/etapi/metrics?format=json"
</code></pre>
<p>Returns detailed metrics in JSON format for debugging or custom integrations.</p>
<h2><strong>Available Metrics</strong></h2>
<h3><strong>Instance Information</strong></h3>
<ul>
<li><code>trilium_info</code> - Version and build information with labels</li>
</ul>
<h3><strong>Database Metrics</strong></h3>
<ul>
<li><code>trilium_notes_total</code> - Total notes (including deleted)</li>
<li><code>trilium_notes_deleted</code> - Number of deleted notes</li>
<li><code>trilium_notes_active</code> - Number of active notes</li>
<li><code>trilium_notes_protected</code> - Number of protected notes</li>
<li><code>trilium_attachments_total</code> - Total attachments</li>
<li><code>trilium_attachments_active</code> - Active attachments</li>
<li><code>trilium_revisions_total</code> - Total note revisions</li>
<li><code>trilium_branches_total</code> - Active branches</li>
<li><code>trilium_attributes_total</code> - Active attributes</li>
<li><code>trilium_blobs_total</code> - Total blob records</li>
<li><code>trilium_etapi_tokens_total</code> - Active ETAPI tokens</li>
<li><code>trilium_embeddings_total</code> - Note embeddings (if available)</li>
</ul>
<h3><strong>Categorized Metrics</strong></h3>
<ul>
<li><code>trilium_notes_by_type{type="text|code|image|file"}</code> - Notes
by type</li>
<li><code>trilium_attachments_by_type{mime_type="..."}</code> - Attachments
by MIME type</li>
</ul>
<h3><strong>Statistics</strong></h3>
<ul>
<li><code>trilium_database_size_bytes</code> - Database size in bytes</li>
<li><code>trilium_oldest_note_timestamp</code> - Timestamp of oldest note</li>
<li><code>trilium_newest_note_timestamp</code> - Timestamp of newest note</li>
<li><code>trilium_last_modified_timestamp</code> - Last modification timestamp</li>
</ul>
<h2><strong>Prometheus Configuration</strong></h2>
<p>Add to your <code>prometheus.yml</code>:</p><pre><code class="language-text-x-trilium-auto">scrape_configs:
- job_name: 'trilium'
static_configs:
- targets: ['localhost:8080']
metrics_path: '/etapi/metrics'
bearer_token: 'YOUR_ETAPI_TOKEN'
scrape_interval: 30s
</code></pre>
<h2><strong>Error Responses</strong></h2>
<ul>
<li><code>400</code> - Invalid format parameter</li>
<li><code>401</code> - Missing or invalid ETAPI token</li>
<li><code>500</code> - Internal server error</li>
</ul>
<p>&nbsp;</p>
<p>&nbsp;</p>
<p>&nbsp;</p>

View File

@@ -0,0 +1,268 @@
import type { Router, Request, Response, NextFunction } from "express";
import eu from "./etapi_utils.js";
import sql from "../services/sql.js";
import appInfo from "../services/app_info.js";
interface MetricsData {
version: {
app: string;
db: number;
node: string;
sync: number;
buildDate: string;
buildRevision: string;
};
database: {
totalNotes: number;
deletedNotes: number;
activeNotes: number;
protectedNotes: number;
totalAttachments: number;
deletedAttachments: number;
activeAttachments: number;
totalRevisions: number;
totalBranches: number;
totalAttributes: number;
totalBlobs: number;
totalEtapiTokens: number;
totalRecentNotes: number;
totalEmbeddings: number;
totalEmbeddingProviders: number;
};
noteTypes: Record<string, number>;
attachmentTypes: Record<string, number>;
statistics: {
oldestNote: string | null;
newestNote: string | null;
lastModified: string | null;
databaseSizeBytes: number | null;
};
timestamp: string;
}
/**
* Converts metrics data to Prometheus text format
*/
function formatPrometheusMetrics(data: MetricsData): string {
const lines: string[] = [];
const timestamp = Math.floor(new Date(data.timestamp).getTime() / 1000);
// Helper function to add a metric
const addMetric = (name: string, value: number | null, help: string, type: string = 'gauge', labels: Record<string, string> = {}) => {
if (value === null) return;
lines.push(`# HELP ${name} ${help}`);
lines.push(`# TYPE ${name} ${type}`);
const labelStr = Object.entries(labels).length > 0
? `{${Object.entries(labels).map(([k, v]) => `${k}="${v}"`).join(',')}}`
: '';
lines.push(`${name}${labelStr} ${value} ${timestamp}`);
lines.push('');
};
// Version info
addMetric('trilium_info', 1, 'Trilium instance information', 'gauge', {
version: data.version.app,
db_version: data.version.db.toString(),
node_version: data.version.node,
sync_version: data.version.sync.toString(),
build_date: data.version.buildDate,
build_revision: data.version.buildRevision
});
// Database metrics
addMetric('trilium_notes_total', data.database.totalNotes, 'Total number of notes including deleted');
addMetric('trilium_notes_deleted', data.database.deletedNotes, 'Number of deleted notes');
addMetric('trilium_notes_active', data.database.activeNotes, 'Number of active notes');
addMetric('trilium_notes_protected', data.database.protectedNotes, 'Number of protected notes');
addMetric('trilium_attachments_total', data.database.totalAttachments, 'Total number of attachments including deleted');
addMetric('trilium_attachments_deleted', data.database.deletedAttachments, 'Number of deleted attachments');
addMetric('trilium_attachments_active', data.database.activeAttachments, 'Number of active attachments');
addMetric('trilium_revisions_total', data.database.totalRevisions, 'Total number of note revisions');
addMetric('trilium_branches_total', data.database.totalBranches, 'Number of active branches');
addMetric('trilium_attributes_total', data.database.totalAttributes, 'Number of active attributes');
addMetric('trilium_blobs_total', data.database.totalBlobs, 'Total number of blob records');
addMetric('trilium_etapi_tokens_total', data.database.totalEtapiTokens, 'Number of active ETAPI tokens');
addMetric('trilium_recent_notes_total', data.database.totalRecentNotes, 'Number of recent notes tracked');
addMetric('trilium_embeddings_total', data.database.totalEmbeddings, 'Number of note embeddings');
addMetric('trilium_embedding_providers_total', data.database.totalEmbeddingProviders, 'Number of embedding providers');
// Note types
for (const [type, count] of Object.entries(data.noteTypes)) {
addMetric('trilium_notes_by_type', count, 'Number of notes by type', 'gauge', { type });
}
// Attachment types
for (const [mime, count] of Object.entries(data.attachmentTypes)) {
addMetric('trilium_attachments_by_type', count, 'Number of attachments by MIME type', 'gauge', { mime_type: mime });
}
// Statistics
if (data.statistics.databaseSizeBytes !== null) {
addMetric('trilium_database_size_bytes', data.statistics.databaseSizeBytes, 'Database size in bytes');
}
if (data.statistics.oldestNote) {
const oldestTimestamp = Math.floor(new Date(data.statistics.oldestNote).getTime() / 1000);
addMetric('trilium_oldest_note_timestamp', oldestTimestamp, 'Timestamp of the oldest note');
}
if (data.statistics.newestNote) {
const newestTimestamp = Math.floor(new Date(data.statistics.newestNote).getTime() / 1000);
addMetric('trilium_newest_note_timestamp', newestTimestamp, 'Timestamp of the newest note');
}
if (data.statistics.lastModified) {
const lastModifiedTimestamp = Math.floor(new Date(data.statistics.lastModified).getTime() / 1000);
addMetric('trilium_last_modified_timestamp', lastModifiedTimestamp, 'Timestamp of the last modification');
}
return lines.join('\n');
}
/**
* Collects comprehensive metrics about the Trilium instance
*/
function collectMetrics(): MetricsData {
// Version information
const version = {
app: appInfo.appVersion,
db: appInfo.dbVersion,
node: appInfo.nodeVersion,
sync: appInfo.syncVersion,
buildDate: appInfo.buildDate,
buildRevision: appInfo.buildRevision
};
// Database counts
const totalNotes = sql.getValue<number>("SELECT COUNT(*) FROM notes");
const deletedNotes = sql.getValue<number>("SELECT COUNT(*) FROM notes WHERE isDeleted = 1");
const activeNotes = totalNotes - deletedNotes;
const protectedNotes = sql.getValue<number>("SELECT COUNT(*) FROM notes WHERE isProtected = 1 AND isDeleted = 0");
const totalAttachments = sql.getValue<number>("SELECT COUNT(*) FROM attachments");
const deletedAttachments = sql.getValue<number>("SELECT COUNT(*) FROM attachments WHERE isDeleted = 1");
const activeAttachments = totalAttachments - deletedAttachments;
const totalRevisions = sql.getValue<number>("SELECT COUNT(*) FROM revisions");
const totalBranches = sql.getValue<number>("SELECT COUNT(*) FROM branches WHERE isDeleted = 0");
const totalAttributes = sql.getValue<number>("SELECT COUNT(*) FROM attributes WHERE isDeleted = 0");
const totalBlobs = sql.getValue<number>("SELECT COUNT(*) FROM blobs");
const totalEtapiTokens = sql.getValue<number>("SELECT COUNT(*) FROM etapi_tokens WHERE isDeleted = 0");
const totalRecentNotes = sql.getValue<number>("SELECT COUNT(*) FROM recent_notes");
// Embedding-related metrics (these tables might not exist in older versions)
let totalEmbeddings = 0;
let totalEmbeddingProviders = 0;
try {
totalEmbeddings = sql.getValue<number>("SELECT COUNT(*) FROM note_embeddings");
totalEmbeddingProviders = sql.getValue<number>("SELECT COUNT(*) FROM embedding_providers");
} catch (e) {
// Tables don't exist, keep defaults
}
const database = {
totalNotes,
deletedNotes,
activeNotes,
protectedNotes,
totalAttachments,
deletedAttachments,
activeAttachments,
totalRevisions,
totalBranches,
totalAttributes,
totalBlobs,
totalEtapiTokens,
totalRecentNotes,
totalEmbeddings,
totalEmbeddingProviders
};
// Note types breakdown
const noteTypesRows = sql.getRows<{ type: string; count: number }>(
"SELECT type, COUNT(*) as count FROM notes WHERE isDeleted = 0 GROUP BY type ORDER BY count DESC"
);
const noteTypes: Record<string, number> = {};
for (const row of noteTypesRows) {
noteTypes[row.type] = row.count;
}
// Attachment types breakdown
const attachmentTypesRows = sql.getRows<{ mime: string; count: number }>(
"SELECT mime, COUNT(*) as count FROM attachments WHERE isDeleted = 0 GROUP BY mime ORDER BY count DESC"
);
const attachmentTypes: Record<string, number> = {};
for (const row of attachmentTypesRows) {
attachmentTypes[row.mime] = row.count;
}
// Statistics
const oldestNote = sql.getValue<string | null>(
"SELECT utcDateCreated FROM notes WHERE isDeleted = 0 ORDER BY utcDateCreated ASC LIMIT 1"
);
const newestNote = sql.getValue<string | null>(
"SELECT utcDateCreated FROM notes WHERE isDeleted = 0 ORDER BY utcDateCreated DESC LIMIT 1"
);
const lastModified = sql.getValue<string | null>(
"SELECT utcDateModified FROM notes WHERE isDeleted = 0 ORDER BY utcDateModified DESC LIMIT 1"
);
// Database size (this might not work on all systems)
let databaseSizeBytes: number | null = null;
try {
const sizeResult = sql.getValue<number>("SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size()");
databaseSizeBytes = sizeResult;
} catch (e) {
// Pragma might not be available
}
const statistics = {
oldestNote,
newestNote,
lastModified,
databaseSizeBytes
};
return {
version,
database,
noteTypes,
attachmentTypes,
statistics,
timestamp: new Date().toISOString()
};
}
function register(router: Router): void {
eu.route(router, "get", "/etapi/metrics", (req: Request, res: Response, next: NextFunction) => {
try {
const metrics = collectMetrics();
const format = (req.query.format as string)?.toLowerCase() || 'prometheus';
if (format === 'json') {
res.status(200).json(metrics);
} else if (format === 'prometheus') {
const prometheusText = formatPrometheusMetrics(metrics);
res.status(200)
.set('Content-Type', 'text/plain; version=0.0.4; charset=utf-8')
.send(prometheusText);
} else {
throw new eu.EtapiError(400, "INVALID_FORMAT", "Supported formats: 'prometheus' (default), 'json'");
}
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
throw new eu.EtapiError(500, "METRICS_ERROR", `Failed to collect metrics: ${errorMessage}`);
}
});
}
export default {
register,
collectMetrics,
formatPrometheusMetrics
};

View File

@@ -0,0 +1,171 @@
import type { Request, Response } from "express";
import etapiMetrics from "../../etapi/metrics.js";
type MetricsData = ReturnType<typeof etapiMetrics.collectMetrics>;
/**
* @swagger
* /api/metrics:
* get:
* summary: Get Trilium instance metrics
* operationId: metrics
* parameters:
* - in: query
* name: format
* schema:
* type: string
* enum: [prometheus, json]
* default: prometheus
* description: Response format - 'prometheus' (default) for Prometheus text format, 'json' for JSON
* responses:
* '200':
* description: Instance metrics
* content:
* text/plain:
* schema:
* type: string
* example: |
* # HELP trilium_info Trilium instance information
* # TYPE trilium_info gauge
* trilium_info{version="0.91.6",db_version="231",node_version="v18.17.0"} 1 1701432000
*
* # HELP trilium_notes_total Total number of notes including deleted
* # TYPE trilium_notes_total gauge
* trilium_notes_total 1234 1701432000
* application/json:
* schema:
* type: object
* properties:
* version:
* type: object
* properties:
* app:
* type: string
* example: "0.91.6"
* db:
* type: integer
* example: 231
* node:
* type: string
* example: "v18.17.0"
* sync:
* type: integer
* example: 35
* buildDate:
* type: string
* example: "2024-09-07T18:36:34Z"
* buildRevision:
* type: string
* example: "7c0d6930fa8f20d269dcfbcbc8f636a25f6bb9a7"
* database:
* type: object
* properties:
* totalNotes:
* type: integer
* example: 1234
* deletedNotes:
* type: integer
* example: 56
* activeNotes:
* type: integer
* example: 1178
* protectedNotes:
* type: integer
* example: 23
* totalAttachments:
* type: integer
* example: 89
* deletedAttachments:
* type: integer
* example: 5
* activeAttachments:
* type: integer
* example: 84
* totalRevisions:
* type: integer
* example: 567
* totalBranches:
* type: integer
* example: 1200
* totalAttributes:
* type: integer
* example: 345
* totalBlobs:
* type: integer
* example: 678
* totalEtapiTokens:
* type: integer
* example: 3
* totalRecentNotes:
* type: integer
* example: 50
* totalEmbeddings:
* type: integer
* example: 123
* totalEmbeddingProviders:
* type: integer
* example: 2
* noteTypes:
* type: object
* additionalProperties:
* type: integer
* example:
* text: 800
* code: 200
* image: 100
* file: 50
* attachmentTypes:
* type: object
* additionalProperties:
* type: integer
* example:
* "image/png": 45
* "image/jpeg": 30
* "application/pdf": 14
* statistics:
* type: object
* properties:
* oldestNote:
* type: string
* nullable: true
* example: "2020-01-01T00:00:00.000Z"
* newestNote:
* type: string
* nullable: true
* example: "2024-12-01T12:00:00.000Z"
* lastModified:
* type: string
* nullable: true
* example: "2024-12-01T11:30:00.000Z"
* databaseSizeBytes:
* type: integer
* nullable: true
* example: 52428800
* timestamp:
* type: string
* example: "2024-12-01T12:00:00.000Z"
* '400':
* description: Invalid format parameter
* '500':
* description: Error collecting metrics
* security:
* - session: []
*/
function getMetrics(req: Request, res: Response): string | MetricsData {
const format = (req.query?.format as string)?.toLowerCase() || 'prometheus';
if (format === 'json') {
return etapiMetrics.collectMetrics();
} else if (format === 'prometheus') {
const metrics = etapiMetrics.collectMetrics();
const prometheusText = etapiMetrics.formatPrometheusMetrics(metrics);
res.set('Content-Type', 'text/plain; version=0.0.4; charset=utf-8');
return prometheusText;
} else {
throw new Error("Supported formats: 'prometheus' (default), 'json'");
}
}
export default {
getMetrics
};

View File

@@ -7,6 +7,7 @@ import ValidationError from "../../errors/validation_error.js";
import type { Request } from "express";
import { changeLanguage, getLocales } from "../../services/i18n.js";
import type { OptionNames } from "@triliumnext/commons";
import config from "../../services/config.js";
// options allowed to be updated directly in the Options dialog
const ALLOWED_OPTIONS = new Set<OptionNames>([
@@ -127,6 +128,12 @@ function getOptions() {
}
resultMap["isPasswordSet"] = optionMap["passwordVerificationHash"] ? "true" : "false";
// if database is read-only, disable editing in UI by setting 0 here
if (config.General.readOnly) {
resultMap["autoReadonlySizeText"] = "0";
resultMap["autoReadonlySizeCode"] = "0";
resultMap["databaseReadonly"] = "true";
}
return resultMap;
}

View File

@@ -52,6 +52,7 @@ import fontsRoute from "./api/fonts.js";
import etapiTokensApiRoutes from "./api/etapi_tokens.js";
import relationMapApiRoute from "./api/relation-map.js";
import otherRoute from "./api/other.js";
import metricsRoute from "./api/metrics.js";
import shareRoutes from "../share/routes.js";
import embeddingsRoute from "./api/embeddings.js";
import ollamaRoute from "./api/ollama.js";
@@ -68,6 +69,7 @@ import etapiNoteRoutes from "../etapi/notes.js";
import etapiSpecialNoteRoutes from "../etapi/special_notes.js";
import etapiSpecRoute from "../etapi/spec.js";
import etapiBackupRoute from "../etapi/backup.js";
import etapiMetricsRoute from "../etapi/metrics.js";
import apiDocsRoute from "./api_docs.js";
import { apiResultHandler, apiRoute, asyncApiRoute, asyncRoute, route, router, uploadMiddlewareWithErrorHandling } from "./route_api.js";
@@ -236,6 +238,7 @@ function register(app: express.Application) {
apiRoute(PST, "/api/recent-notes", recentNotesRoute.addRecentNote);
apiRoute(GET, "/api/app-info", appInfoRoute.getAppInfo);
apiRoute(GET, "/api/metrics", metricsRoute.getMetrics);
// docker health check
route(GET, "/api/health-check", [], () => ({ status: "ok" }), apiResultHandler);
@@ -363,6 +366,7 @@ function register(app: express.Application) {
etapiSpecialNoteRoutes.register(router);
etapiSpecRoute.register(router);
etapiBackupRoute.register(router);
etapiMetricsRoute.register(router);
// LLM Chat API
asyncApiRoute(PST, "/api/llm/chat", llmRoute.createSession);

View File

@@ -21,6 +21,7 @@ export interface TriliumConfig {
noAuthentication: boolean;
noBackup: boolean;
noDesktopIcon: boolean;
readOnly: boolean;
};
Network: {
host: string;
@@ -62,7 +63,10 @@ const config: TriliumConfig = {
envToBoolean(process.env.TRILIUM_GENERAL_NOBACKUP) || iniConfig.General.noBackup || false,
noDesktopIcon:
envToBoolean(process.env.TRILIUM_GENERAL_NODESKTOPICON) || iniConfig.General.noDesktopIcon || false
envToBoolean(process.env.TRILIUM_GENERAL_NODESKTOPICON) || iniConfig.General.noDesktopIcon || false,
readOnly:
envToBoolean(process.env.TRILIUM_GENERAL_READONLY) || iniConfig.General.readOnly || false
},
Network: {

View File

@@ -13,18 +13,20 @@ import Database from "better-sqlite3";
import ws from "./ws.js";
import becca_loader from "../becca/becca_loader.js";
import entity_changes from "./entity_changes.js";
import config from "./config.js";
let dbConnection: DatabaseType = buildDatabase();
let statementCache: Record<string, Statement> = {};
function buildDatabase() {
// for integration tests, ignore the config's readOnly setting
if (process.env.TRILIUM_INTEGRATION_TEST === "memory") {
return buildIntegrationTestDatabase();
} else if (process.env.TRILIUM_INTEGRATION_TEST === "memory-no-store") {
return new Database(":memory:");
}
return new Database(dataDir.DOCUMENT_PATH);
return new Database(dataDir.DOCUMENT_PATH, { readonly: config.General.readOnly });
}
function buildIntegrationTestDatabase(dbPath?: string) {
@@ -208,6 +210,13 @@ function getColumn<T>(query: string, params: Params = []): T[] {
}
function execute(query: string, params: Params = []): RunResult {
if (config.General.readOnly && (query.startsWith("UPDATE") || query.startsWith("INSERT") || query.startsWith("DELETE"))) {
log.error(`read-only DB ignored: ${query} with parameters ${JSON.stringify(params)}`);
return {
changes: 0,
lastInsertRowid: 0
};
}
return wrap(query, (s) => s.run(params)) as RunResult;
}

View File

@@ -186,6 +186,9 @@ function setDbAsInitialized() {
}
function optimize() {
if (config.General.readOnly) {
return;
}
log.info("Optimizing database");
const start = Date.now();