From 9fe23442f594399ff5e711ea33bfaf2ac5827ebc Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sun, 22 Mar 2026 20:10:59 +0200 Subject: [PATCH] chore(core): integrate content_hash --- apps/server/src/services/content_hash.ts | 91 +------------------ packages/trilium-core/src/index.ts | 1 + .../trilium-core/src/services/content_hash.ts | 90 ++++++++++++++++++ 3 files changed, 93 insertions(+), 89 deletions(-) create mode 100644 packages/trilium-core/src/services/content_hash.ts diff --git a/apps/server/src/services/content_hash.ts b/apps/server/src/services/content_hash.ts index 6b189bbe61..1897547b53 100644 --- a/apps/server/src/services/content_hash.ts +++ b/apps/server/src/services/content_hash.ts @@ -1,89 +1,2 @@ -import { erase as eraseService,utils } from "@triliumnext/core"; - -import log from "./log.js"; -import sql from "./sql.js"; - -type SectorHash = Record; - -interface FailedCheck { - entityName: string; - sector: string[1]; -} - -function getEntityHashes() { - // blob erasure is not synced, we should check before each sync if there's some blob to erase - eraseService.eraseUnusedBlobs(); - - const startTime = new Date(); - - // we know this is slow and the total content hash calculation time is logged - type HashRow = [string, string, string, boolean]; - const hashRows = sql.disableSlowQueryLogging(() => - sql.getRawRows(` - SELECT entityName, - entityId, - hash, - isErased - FROM entity_changes - WHERE isSynced = 1 - AND entityName != 'note_reordering'`) - ); - - // sorting is faster in memory - // sorting by entityId is enough, hashes will be segmented by entityName later on anyway - hashRows.sort((a, b) => (a[1] < b[1] ? -1 : 1)); - - const hashMap: Record = {}; - - for (const [entityName, entityId, hash, isErased] of hashRows) { - const entityHashMap = (hashMap[entityName] = hashMap[entityName] || {}); - - const sector = entityId[0]; - - // if the entity is erased, its hash is not updated, so it has to be added extra - entityHashMap[sector] = (entityHashMap[sector] || "") + hash + isErased; - } - - for (const entityHashMap of Object.values(hashMap)) { - for (const key in entityHashMap) { - entityHashMap[key] = utils.hash(entityHashMap[key]); - } - } - - const elapsedTimeMs = Date.now() - startTime.getTime(); - - log.info(`Content hash computation took ${elapsedTimeMs}ms`); - - return hashMap; -} - -function checkContentHashes(otherHashes: Record) { - const entityHashes = getEntityHashes(); - const failedChecks: FailedCheck[] = []; - - for (const entityName in entityHashes) { - const thisSectorHashes: SectorHash = entityHashes[entityName] || {}; - const otherSectorHashes: SectorHash = otherHashes[entityName] || {}; - - const sectors = new Set(Object.keys(thisSectorHashes).concat(Object.keys(otherSectorHashes))); - - for (const sector of sectors) { - if (thisSectorHashes[sector] !== otherSectorHashes[sector]) { - log.info(`Content hash check for ${entityName} sector ${sector} FAILED. Local is ${thisSectorHashes[sector]}, remote is ${otherSectorHashes[sector]}`); - - failedChecks.push({ entityName, sector }); - } - } - } - - if (failedChecks.length === 0) { - log.info("Content hash checks PASSED"); - } - - return failedChecks; -} - -export default { - getEntityHashes, - checkContentHashes -}; +import { content_hash } from "@triliumnext/core"; +export default content_hash; diff --git a/packages/trilium-core/src/index.ts b/packages/trilium-core/src/index.ts index e9de1b3ba1..bd48549de3 100644 --- a/packages/trilium-core/src/index.ts +++ b/packages/trilium-core/src/index.ts @@ -80,6 +80,7 @@ export { default as sync_options } from "./services/sync_options"; export { default as sync_update } from "./services/sync_update"; export { default as sync } from "./services/sync"; export { default as consistency_checks } from "./services/consistency_checks"; +export { default as content_hash } from "./services/content_hash"; export type { RequestProvider, ExecOpts, CookieJar } from "./services/request"; export async function initializeCore({ dbConfig, executionContext, crypto, translations, messaging, request, extraAppInfo }: { diff --git a/packages/trilium-core/src/services/content_hash.ts b/packages/trilium-core/src/services/content_hash.ts new file mode 100644 index 0000000000..138d0f6763 --- /dev/null +++ b/packages/trilium-core/src/services/content_hash.ts @@ -0,0 +1,90 @@ +import eraseService from "./erase.js"; +import { getLog } from "./log.js"; +import { getSql } from "./sql/index.js"; +import { hash } from "./utils/index.js"; + +type SectorHash = Record; + +interface FailedCheck { + entityName: string; + sector: string[1]; +} + +function getEntityHashes() { + // blob erasure is not synced, we should check before each sync if there's some blob to erase + eraseService.eraseUnusedBlobs(); + + const startTime = new Date(); + + // we know this is slow and the total content hash calculation time is logged + type HashRow = [string, string, string, boolean]; + const sql = getSql(); + const hashRows = sql.disableSlowQueryLogging(() => + sql.getRawRows(` + SELECT entityName, + entityId, + hash, + isErased + FROM entity_changes + WHERE isSynced = 1 + AND entityName != 'note_reordering'`) + ); + + // sorting is faster in memory + // sorting by entityId is enough, hashes will be segmented by entityName later on anyway + hashRows.sort((a, b) => (a[1] < b[1] ? -1 : 1)); + + const hashMap: Record = {}; + + for (const [entityName, entityId, hash, isErased] of hashRows) { + const entityHashMap = (hashMap[entityName] = hashMap[entityName] || {}); + + const sector = entityId[0]; + + // if the entity is erased, its hash is not updated, so it has to be added extra + entityHashMap[sector] = (entityHashMap[sector] || "") + hash + isErased; + } + + for (const entityHashMap of Object.values(hashMap)) { + for (const key in entityHashMap) { + entityHashMap[key] = hash(entityHashMap[key]); + } + } + + const elapsedTimeMs = Date.now() - startTime.getTime(); + + getLog().info(`Content hash computation took ${elapsedTimeMs}ms`); + + return hashMap; +} + +function checkContentHashes(otherHashes: Record) { + const entityHashes = getEntityHashes(); + const failedChecks: FailedCheck[] = []; + + for (const entityName in entityHashes) { + const thisSectorHashes: SectorHash = entityHashes[entityName] || {}; + const otherSectorHashes: SectorHash = otherHashes[entityName] || {}; + + const sectors = new Set(Object.keys(thisSectorHashes).concat(Object.keys(otherSectorHashes))); + + for (const sector of sectors) { + if (thisSectorHashes[sector] !== otherSectorHashes[sector]) { + getLog().info(`Content hash check for ${entityName} sector ${sector} FAILED. Local is ${thisSectorHashes[sector]}, remote is ${otherSectorHashes[sector]}`); + + failedChecks.push({ entityName, sector }); + } + } + } + + if (failedChecks.length === 0) { + getLog().info("Content hash checks PASSED"); + } + + return failedChecks; +} + +export default { + getEntityHashes, + checkContentHashes +};