From 28a56ff7bfc90a073aa5ee8eb9b2c253caf8a46e Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sun, 22 Mar 2026 14:03:48 +0200 Subject: [PATCH] feat(core): integrate search with route --- apps/server/src/routes/routes.ts | 7 - apps/server/src/services/hoisted_note.ts | 42 +- apps/server/src/services/search/note_set.ts | 2 - .../src/services/search/services/search.ts | 794 ----------------- apps/server/src/services/utils.ts | 6 +- packages/trilium-core/src/index.ts | 1 + .../trilium-core}/src/routes/api/search.ts | 7 +- packages/trilium-core/src/routes/index.ts | 8 + .../trilium-core/src/services/hoisted_note.ts | 40 + packages/trilium-core/src/services/script.ts | 8 + .../services/search/expressions/ancestor.ts | 6 +- .../src/services/search/expressions/and.ts | 0 .../search/expressions/attribute_exists.ts | 0 .../services/search/expressions/child_of.ts | 0 .../search/expressions/descendant_of.ts | 0 .../services/search/expressions/expression.ts | 0 .../services/search/expressions/is_hidden.ts | 0 .../search/expressions/label_comparison.ts | 0 .../src/services/search/expressions/not.ts | 0 .../expressions/note_content_fulltext.spec.ts | 0 .../expressions/note_content_fulltext.ts | 10 +- ...note_content_fulltext_preprocessor.spec.ts | 0 .../note_content_fulltext_preprocessor.ts | 6 +- .../search/expressions/note_flat_text.ts | 4 +- .../src/services/search/expressions/or.ts | 0 .../search/expressions/order_by_and_limit.ts | 0 .../services/search/expressions/parent_of.ts | 0 .../search/expressions/property_comparison.ts | 0 .../search/expressions/relation_where.ts | 0 .../src/services/search/expressions/true.ts | 0 .../src/services/search/search_context.ts | 0 .../src/services/search/search_result.ts | 2 +- .../search/services/build_comparator.ts | 0 .../search/services/handle_parens.spec.ts | 0 .../services/search/services/handle_parens.ts | 0 .../src/services/search/services/lex.spec.ts | 0 .../src/services/search/services/lex.ts | 0 .../services/search/services/parse.spec.ts | 0 .../src/services/search/services/parse.ts | 2 +- .../services/progressive_search.spec.ts | 0 .../services/search/services/search.spec.ts | 0 .../src/services/search/services/search.ts | 798 +++++++++++++++++- .../src/services/search/services/types.ts | 0 .../services/search/utils/text_utils.spec.ts | 0 .../src/services/search/utils/text_utils.ts | 56 +- .../services/search/value_extractor.spec.ts | 0 .../src/services/search/value_extractor.ts | 0 .../trilium-core/src/services/utils/index.ts | 4 + 48 files changed, 902 insertions(+), 901 deletions(-) delete mode 100644 apps/server/src/services/search/note_set.ts delete mode 100644 apps/server/src/services/search/services/search.ts rename {apps/server => packages/trilium-core}/src/routes/api/search.ts (95%) create mode 100644 packages/trilium-core/src/services/hoisted_note.ts rename {apps/server => packages/trilium-core}/src/services/search/expressions/ancestor.ts (90%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/and.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/attribute_exists.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/child_of.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/descendant_of.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/expression.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/is_hidden.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/label_comparison.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/not.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/note_content_fulltext.spec.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/note_content_fulltext.ts (98%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/note_content_fulltext_preprocessor.spec.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/note_content_fulltext_preprocessor.ts (96%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/note_flat_text.ts (98%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/or.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/order_by_and_limit.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/parent_of.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/property_comparison.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/relation_where.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/expressions/true.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/search_context.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/search_result.ts (99%) rename {apps/server => packages/trilium-core}/src/services/search/services/build_comparator.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/services/handle_parens.spec.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/services/handle_parens.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/services/lex.spec.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/services/lex.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/services/parse.spec.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/services/parse.ts (99%) rename {apps/server => packages/trilium-core}/src/services/search/services/progressive_search.spec.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/services/search.spec.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/services/types.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/utils/text_utils.spec.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/utils/text_utils.ts (97%) rename {apps/server => packages/trilium-core}/src/services/search/value_extractor.spec.ts (100%) rename {apps/server => packages/trilium-core}/src/services/search/value_extractor.ts (100%) diff --git a/apps/server/src/routes/routes.ts b/apps/server/src/routes/routes.ts index 92bc03e1e3..6fbabf41fe 100644 --- a/apps/server/src/routes/routes.ts +++ b/apps/server/src/routes/routes.ts @@ -35,7 +35,6 @@ import otherRoute from "./api/other.js"; import passwordApiRoute from "./api/password.js"; import recoveryCodes from './api/recovery_codes.js'; import scriptRoute from "./api/script.js"; -import searchRoute from "./api/search.js"; import senderRoute from "./api/sender.js"; import setupApiRoute from "./api/setup.js"; import similarNotesRoute from "./api/similar_notes.js"; @@ -171,12 +170,6 @@ function register(app: express.Application) { apiRoute(GET, "/api/autocomplete", autocompleteApiRoute.getAutocomplete); apiRoute(GET, "/api/autocomplete/notesCount", autocompleteApiRoute.getNotesCount); - apiRoute(GET, "/api/quick-search/:searchString", searchRoute.quickSearch); - apiRoute(GET, "/api/search-note/:noteId", searchRoute.searchFromNote); - apiRoute(PST, "/api/search-and-execute-note/:noteId", searchRoute.searchAndExecute); - apiRoute(PST, "/api/search-related", searchRoute.getRelatedNotes); - apiRoute(GET, "/api/search/:searchString", searchRoute.search); - apiRoute(GET, "/api/search-templates", searchRoute.searchTemplates); route(PST, "/api/login/sync", [loginRateLimiter], loginApiRoute.loginSync, apiResultHandler); // this is for entering protected mode so user has to be already logged-in (that's the reason we don't require username) diff --git a/apps/server/src/services/hoisted_note.ts b/apps/server/src/services/hoisted_note.ts index ba68263fc8..ec76b03d65 100644 --- a/apps/server/src/services/hoisted_note.ts +++ b/apps/server/src/services/hoisted_note.ts @@ -1,40 +1,2 @@ -import cls from "./cls.js"; -import becca from "../becca/becca.js"; - -function getHoistedNoteId() { - return cls.getHoistedNoteId(); -} - -function isHoistedInHiddenSubtree() { - const hoistedNoteId = getHoistedNoteId(); - - if (hoistedNoteId === "root") { - return false; - } else if (hoistedNoteId === "_hidden") { - return true; - } - - const hoistedNote = becca.getNote(hoistedNoteId); - - if (!hoistedNote) { - throw new Error(`Cannot find hoisted note '${hoistedNoteId}'`); - } - - return hoistedNote.isHiddenCompletely(); -} - -function getWorkspaceNote() { - const hoistedNote = becca.getNote(cls.getHoistedNoteId()); - - if (hoistedNote && (hoistedNote.isRoot() || hoistedNote.hasLabel("workspace"))) { - return hoistedNote; - } else { - return becca.getRoot(); - } -} - -export default { - getHoistedNoteId, - getWorkspaceNote, - isHoistedInHiddenSubtree -}; +import { hoisted_note } from "@triliumnext/core"; +export default hoisted_note; diff --git a/apps/server/src/services/search/note_set.ts b/apps/server/src/services/search/note_set.ts deleted file mode 100644 index 10ecb41346..0000000000 --- a/apps/server/src/services/search/note_set.ts +++ /dev/null @@ -1,2 +0,0 @@ -import { NoteSet } from "@triliumnext/core"; -export default NoteSet; diff --git a/apps/server/src/services/search/services/search.ts b/apps/server/src/services/search/services/search.ts deleted file mode 100644 index 16eeb70ccd..0000000000 --- a/apps/server/src/services/search/services/search.ts +++ /dev/null @@ -1,794 +0,0 @@ -import { becca_service } from "@triliumnext/core"; -import normalizeString from "normalize-strings"; -import striptags from "striptags"; - -import becca from "../../../becca/becca.js"; -import type BNote from "../../../becca/entities/bnote.js"; -import hoistedNoteService from "../../hoisted_note.js"; -import log from "../../log.js"; -import protectedSessionService from "../../protected_session.js"; -import scriptService from "../../script.js"; -import sql from "../../sql.js"; -import { escapeHtml, escapeRegExp } from "../../utils.js"; -import type Expression from "../expressions/expression.js"; -import SearchContext from "../search_context.js"; -import SearchResult from "../search_result.js"; -import handleParens from "./handle_parens.js"; -import lex from "./lex.js"; -import parse from "./parse.js"; -import type { SearchParams, TokenStructure } from "./types.js"; - -export interface SearchNoteResult { - searchResultNoteIds: string[]; - highlightedTokens: string[]; - error: string | null; -} - -export const EMPTY_RESULT: SearchNoteResult = { - searchResultNoteIds: [], - highlightedTokens: [], - error: null -}; - -function searchFromNote(note: BNote): SearchNoteResult { - let searchResultNoteIds; - let highlightedTokens: string[]; - - const searchScript = note.getRelationValue("searchScript"); - const searchString = note.getLabelValue("searchString") || ""; - let error: string | null = null; - - if (searchScript) { - searchResultNoteIds = searchFromRelation(note, "searchScript"); - highlightedTokens = []; - } else { - const searchContext = new SearchContext({ - fastSearch: note.hasLabel("fastSearch"), - ancestorNoteId: note.getRelationValue("ancestor") || undefined, - ancestorDepth: note.getLabelValue("ancestorDepth") || undefined, - includeArchivedNotes: note.hasLabel("includeArchivedNotes"), - orderBy: note.getLabelValue("orderBy") || undefined, - orderDirection: note.getLabelValue("orderDirection") || undefined, - limit: parseInt(note.getLabelValue("limit") || "0", 10), - debug: note.hasLabel("debug"), - fuzzyAttributeSearch: false - }); - - searchResultNoteIds = findResultsWithQuery(searchString, searchContext).map((sr) => sr.noteId); - - highlightedTokens = searchContext.highlightedTokens; - error = searchContext.getError(); - } - - // we won't return search note's own noteId - // also don't allow root since that would force infinite cycle - return { - searchResultNoteIds: searchResultNoteIds.filter((resultNoteId) => !["root", note.noteId].includes(resultNoteId)), - highlightedTokens, - error - }; -} - -function searchFromRelation(note: BNote, relationName: string) { - const scriptNote = note.getRelationTarget(relationName); - - if (!scriptNote) { - log.info(`Search note's relation ${relationName} has not been found.`); - - return []; - } - - if (!scriptNote.isJavaScript() || scriptNote.getScriptEnv() !== "backend") { - log.info(`Note ${scriptNote.noteId} is not executable.`); - - return []; - } - - if (!note.isContentAvailable()) { - log.info(`Note ${scriptNote.noteId} is not available outside of protected session.`); - - return []; - } - - const result = scriptService.executeNote(scriptNote, { originEntity: note }); - - if (!Array.isArray(result)) { - log.info(`Result from ${scriptNote.noteId} is not an array.`); - - return []; - } - - if (result.length === 0) { - return []; - } - - // we expect either array of noteIds (strings) or notes, in that case we extract noteIds ourselves - return typeof result[0] === "string" ? result : result.map((item) => item.noteId); -} - -function loadNeededInfoFromDatabase() { - /** - * This complex structure is needed to calculate total occupied space by a note. Several object instances - * (note, revisions, attachments) can point to a single blobId, and thus the blob size should count towards the total - * only once. - * - * noteId => { blobId => blobSize } - */ - const noteBlobs: Record> = {}; - - type NoteContentLengthsRow = { - noteId: string; - blobId: string; - length: number; - }; - const noteContentLengths = sql.getRows(` - SELECT - noteId, - blobId, - LENGTH(content) AS length - FROM notes - JOIN blobs USING(blobId) - WHERE notes.isDeleted = 0`); - - for (const { noteId, blobId, length } of noteContentLengths) { - if (!(noteId in becca.notes)) { - log.error(`Note '${noteId}' not found in becca.`); - continue; - } - - becca.notes[noteId].contentSize = length; - becca.notes[noteId].revisionCount = 0; - - noteBlobs[noteId] = { [blobId]: length }; - } - - type AttachmentContentLengthsRow = { - noteId: string; - blobId: string; - length: number; - }; - const attachmentContentLengths = sql.getRows(` - SELECT - ownerId AS noteId, - attachments.blobId, - LENGTH(content) AS length - FROM attachments - JOIN notes ON attachments.ownerId = notes.noteId - JOIN blobs ON attachments.blobId = blobs.blobId - WHERE attachments.isDeleted = 0 - AND notes.isDeleted = 0`); - - for (const { noteId, blobId, length } of attachmentContentLengths) { - if (!(noteId in becca.notes)) { - log.error(`Note '${noteId}' not found in becca.`); - continue; - } - - if (!(noteId in noteBlobs)) { - log.error(`Did not find a '${noteId}' in the noteBlobs.`); - continue; - } - - noteBlobs[noteId][blobId] = length; - } - - for (const noteId in noteBlobs) { - becca.notes[noteId].contentAndAttachmentsSize = Object.values(noteBlobs[noteId]).reduce((acc, size) => acc + size, 0); - } - - type RevisionRow = { - noteId: string; - blobId: string; - length: number; - isNoteRevision: true; - }; - const revisionContentLengths = sql.getRows(` - SELECT - noteId, - revisions.blobId, - LENGTH(content) AS length, - 1 AS isNoteRevision - FROM notes - JOIN revisions USING(noteId) - JOIN blobs ON revisions.blobId = blobs.blobId - WHERE notes.isDeleted = 0 - UNION ALL - SELECT - noteId, - revisions.blobId, - LENGTH(content) AS length, - 0 AS isNoteRevision -- it's attachment not counting towards revision count - FROM notes - JOIN revisions USING(noteId) - JOIN attachments ON attachments.ownerId = revisions.revisionId - JOIN blobs ON attachments.blobId = blobs.blobId - WHERE notes.isDeleted = 0`); - - for (const { noteId, blobId, length, isNoteRevision } of revisionContentLengths) { - if (!(noteId in becca.notes)) { - log.error(`Note '${noteId}' not found in becca.`); - continue; - } - - if (!(noteId in noteBlobs)) { - log.error(`Did not find a '${noteId}' in the noteBlobs.`); - continue; - } - - noteBlobs[noteId][blobId] = length; - - if (isNoteRevision) { - const noteRevision = becca.notes[noteId]; - if (noteRevision && noteRevision.revisionCount) { - noteRevision.revisionCount++; - } - } - } - - for (const noteId in noteBlobs) { - becca.notes[noteId].contentAndAttachmentsAndRevisionsSize = Object.values(noteBlobs[noteId]).reduce((acc, size) => acc + size, 0); - } -} - -function findResultsWithExpression(expression: Expression, searchContext: SearchContext): SearchResult[] { - if (searchContext.dbLoadNeeded) { - loadNeededInfoFromDatabase(); - } - - // If there's an explicit orderBy clause, skip progressive search - // as it would interfere with the ordering - if (searchContext.orderBy) { - // For ordered queries, don't use progressive search but respect - // the original fuzzy matching setting - return performSearch(expression, searchContext, searchContext.enableFuzzyMatching); - } - - // If fuzzy matching is explicitly disabled, skip progressive search - if (!searchContext.enableFuzzyMatching) { - return performSearch(expression, searchContext, false); - } - - // Phase 1: Try exact matches first (without fuzzy matching) - const exactResults = performSearch(expression, searchContext, false); - - // Check if we have sufficient high-quality results - const minResultThreshold = 5; - const minScoreForQuality = 10; // Minimum score to consider a result "high quality" - - const highQualityResults = exactResults.filter(result => result.score >= minScoreForQuality); - - // If we have enough high-quality exact matches, return them - if (highQualityResults.length >= minResultThreshold) { - return exactResults; - } - - // Phase 2: Add fuzzy matching as fallback when exact matches are insufficient - const fuzzyResults = performSearch(expression, searchContext, true); - - // Merge results, ensuring exact matches always rank higher than fuzzy matches - return mergeExactAndFuzzyResults(exactResults, fuzzyResults); -} - -function performSearch(expression: Expression, searchContext: SearchContext, enableFuzzyMatching: boolean): SearchResult[] { - const allNoteSet = becca.getAllNoteSet(); - - const noteIdToNotePath: Record = {}; - const executionContext = { - noteIdToNotePath - }; - - // Store original fuzzy setting and temporarily override it - const originalFuzzyMatching = searchContext.enableFuzzyMatching; - searchContext.enableFuzzyMatching = enableFuzzyMatching; - - const noteSet = expression.execute(allNoteSet, executionContext, searchContext); - - const searchResults = noteSet.notes.map((note) => { - const notePathArray = executionContext.noteIdToNotePath[note.noteId] || note.getBestNotePath(); - - if (!notePathArray) { - throw new Error(`Can't find note path for note ${JSON.stringify(note.getPojo())}`); - } - - return new SearchResult(notePathArray); - }); - - for (const res of searchResults) { - res.computeScore(searchContext.fulltextQuery, searchContext.highlightedTokens, enableFuzzyMatching); - } - - // Restore original fuzzy setting - searchContext.enableFuzzyMatching = originalFuzzyMatching; - - if (!noteSet.sorted) { - searchResults.sort((a, b) => { - if (a.score > b.score) { - return -1; - } else if (a.score < b.score) { - return 1; - } - - // if score does not decide then sort results by depth of the note. - // This is based on the assumption that more important results are closer to the note root. - if (a.notePathArray.length === b.notePathArray.length) { - return a.notePathTitle < b.notePathTitle ? -1 : 1; - } - - return a.notePathArray.length < b.notePathArray.length ? -1 : 1; - }); - } - - return searchResults; -} - -function mergeExactAndFuzzyResults(exactResults: SearchResult[], fuzzyResults: SearchResult[]): SearchResult[] { - // Create a map of exact result note IDs for deduplication - const exactNoteIds = new Set(exactResults.map(result => result.noteId)); - - // Add fuzzy results that aren't already in exact results - const additionalFuzzyResults = fuzzyResults.filter(result => !exactNoteIds.has(result.noteId)); - - // Sort exact results by score (best exact matches first) - exactResults.sort((a, b) => { - if (a.score > b.score) { - return -1; - } else if (a.score < b.score) { - return 1; - } - - // if score does not decide then sort results by depth of the note. - if (a.notePathArray.length === b.notePathArray.length) { - return a.notePathTitle < b.notePathTitle ? -1 : 1; - } - - return a.notePathArray.length < b.notePathArray.length ? -1 : 1; - }); - - // Sort fuzzy results by score (best fuzzy matches first) - additionalFuzzyResults.sort((a, b) => { - if (a.score > b.score) { - return -1; - } else if (a.score < b.score) { - return 1; - } - - // if score does not decide then sort results by depth of the note. - if (a.notePathArray.length === b.notePathArray.length) { - return a.notePathTitle < b.notePathTitle ? -1 : 1; - } - - return a.notePathArray.length < b.notePathArray.length ? -1 : 1; - }); - - // CRITICAL: Always put exact matches before fuzzy matches, regardless of scores - return [...exactResults, ...additionalFuzzyResults]; -} - -function parseQueryToExpression(query: string, searchContext: SearchContext) { - const { fulltextQuery, fulltextTokens, expressionTokens, leadingOperator } = lex(query); - searchContext.fulltextQuery = fulltextQuery; - - let structuredExpressionTokens: TokenStructure; - - try { - structuredExpressionTokens = handleParens(expressionTokens); - } catch (e: any) { - structuredExpressionTokens = []; - searchContext.addError(e.message); - } - - const expression = parse({ - fulltextTokens, - expressionTokens: structuredExpressionTokens, - searchContext, - originalQuery: query, - leadingOperator - }); - - if (searchContext.debug) { - searchContext.debugInfo = { - fulltextTokens, - structuredExpressionTokens, - expression - }; - - log.info(`Search debug: ${JSON.stringify(searchContext.debugInfo, null, 4)}`); - } - - return expression; -} - -function searchNotes(query: string, params: SearchParams = {}): BNote[] { - const searchResults = findResultsWithQuery(query, new SearchContext(params)); - - return searchResults.map((sr) => becca.notes[sr.noteId]); -} - -function findResultsWithQuery(query: string, searchContext: SearchContext): SearchResult[] { - query = query || ""; - searchContext.originalQuery = query; - - const expression = parseQueryToExpression(query, searchContext); - - if (!expression) { - return []; - } - - // If the query starts with '#', it's a pure expression query. - // Don't use progressive search for these as they may have complex - // ordering or other logic that shouldn't be interfered with. - const isPureExpressionQuery = query.trim().startsWith('#'); - - if (isPureExpressionQuery) { - // For pure expression queries, use standard search without progressive phases - return performSearch(expression, searchContext, searchContext.enableFuzzyMatching); - } - - return findResultsWithExpression(expression, searchContext); -} - -function findFirstNoteWithQuery(query: string, searchContext: SearchContext): BNote | null { - const searchResults = findResultsWithQuery(query, searchContext); - - return searchResults.length > 0 ? becca.notes[searchResults[0].noteId] : null; -} - -function extractContentSnippet(noteId: string, searchTokens: string[], maxLength: number = 200): string { - const note = becca.notes[noteId]; - if (!note) { - return ""; - } - - // Only extract content for text-based notes - if (!["text", "code", "mermaid", "canvas", "mindMap"].includes(note.type)) { - return ""; - } - - try { - let content = note.getContent(); - - if (!content || typeof content !== "string") { - return ""; - } - - // Handle protected notes - if (note.isProtected && protectedSessionService.isProtectedSessionAvailable()) { - try { - content = protectedSessionService.decryptString(content) || ""; - } catch (e) { - return ""; // Can't decrypt, don't show content - } - } else if (note.isProtected) { - return ""; // Protected but no session available - } - - // Strip HTML tags for text notes - if (note.type === "text") { - content = striptags(content); - } - - // Normalize whitespace while preserving paragraph breaks - // First, normalize multiple newlines to double newlines (paragraph breaks) - content = content.replace(/\n\s*\n/g, "\n\n"); - // Then normalize spaces within lines - content = content.split('\n').map(line => line.replace(/\s+/g, " ").trim()).join('\n'); - // Finally trim the whole content - content = content.trim(); - - if (!content) { - return ""; - } - - // Try to find a snippet around the first matching token - const normalizedContent = normalizeString(content.toLowerCase()); - let snippetStart = 0; - let matchFound = false; - - for (const token of searchTokens) { - const normalizedToken = normalizeString(token.toLowerCase()); - const matchIndex = normalizedContent.indexOf(normalizedToken); - - if (matchIndex !== -1) { - // Center the snippet around the match - snippetStart = Math.max(0, matchIndex - maxLength / 2); - matchFound = true; - break; - } - } - - // Extract snippet - let snippet = content.substring(snippetStart, snippetStart + maxLength); - - // If snippet contains linebreaks, limit to max 4 lines and override character limit - const lines = snippet.split('\n'); - if (lines.length > 4) { - // Find which lines contain the search tokens to ensure they're included - const normalizedLines = lines.map(line => normalizeString(line.toLowerCase())); - const normalizedTokens = searchTokens.map(token => normalizeString(token.toLowerCase())); - - // Find the first line that contains a search token - let firstMatchLine = -1; - for (let i = 0; i < normalizedLines.length; i++) { - if (normalizedTokens.some(token => normalizedLines[i].includes(token))) { - firstMatchLine = i; - break; - } - } - - if (firstMatchLine !== -1) { - // Center the 4-line window around the first match - // Try to show 1 line before and 2 lines after the match - const startLine = Math.max(0, firstMatchLine - 1); - const endLine = Math.min(lines.length, startLine + 4); - snippet = lines.slice(startLine, endLine).join('\n'); - } else { - // No match found in lines (shouldn't happen), just take first 4 - snippet = lines.slice(0, 4).join('\n'); - } - // Add ellipsis if we truncated lines - snippet = `${snippet }...`; - } else if (lines.length > 1) { - // For multi-line snippets that are 4 or fewer lines, keep them as-is - // No need to truncate - } else { - // Single line content - apply original word boundary logic - // Try to start/end at word boundaries - if (snippetStart > 0) { - const firstSpace = snippet.search(/\s/); - if (firstSpace > 0 && firstSpace < 20) { - snippet = snippet.substring(firstSpace + 1); - } - snippet = `...${ snippet}`; - } - - if (snippetStart + maxLength < content.length) { - const lastSpace = snippet.search(/\s[^\s]*$/); - if (lastSpace > snippet.length - 20 && lastSpace > 0) { - snippet = snippet.substring(0, lastSpace); - } - snippet = `${snippet }...`; - } - } - - return snippet; - } catch (e) { - log.error(`Error extracting content snippet for note ${noteId}: ${e}`); - return ""; - } -} - -function extractAttributeSnippet(noteId: string, searchTokens: string[], maxLength: number = 200): string { - const note = becca.notes[noteId]; - if (!note) { - return ""; - } - - try { - // Get all attributes for this note - const attributes = note.getAttributes(); - if (!attributes || attributes.length === 0) { - return ""; - } - - const matchingAttributes: Array<{name: string, value: string, type: string}> = []; - - // Look for attributes that match the search tokens - for (const attr of attributes) { - const attrName = attr.name?.toLowerCase() || ""; - const attrValue = attr.value?.toLowerCase() || ""; - const attrType = attr.type || ""; - - // Check if any search token matches the attribute name or value - const hasMatch = searchTokens.some(token => { - const normalizedToken = normalizeString(token.toLowerCase()); - return attrName.includes(normalizedToken) || attrValue.includes(normalizedToken); - }); - - if (hasMatch) { - matchingAttributes.push({ - name: attr.name || "", - value: attr.value || "", - type: attrType - }); - } - } - - if (matchingAttributes.length === 0) { - return ""; - } - - // Limit to 4 lines maximum, similar to content snippet logic - const lines: string[] = []; - for (const attr of matchingAttributes.slice(0, 4)) { - let line = ""; - if (attr.type === "label") { - line = attr.value ? `#${attr.name}="${attr.value}"` : `#${attr.name}`; - } else if (attr.type === "relation") { - // For relations, show the target note title if possible - const targetNote = attr.value ? becca.notes[attr.value] : null; - const targetTitle = targetNote ? targetNote.title : attr.value; - line = `~${attr.name}="${targetTitle}"`; - } - - if (line) { - lines.push(line); - } - } - - let snippet = lines.join('\n'); - - // Apply length limit while preserving line structure - if (snippet.length > maxLength) { - // Try to truncate at word boundaries but keep lines intact - const truncated = snippet.substring(0, maxLength); - const lastNewline = truncated.lastIndexOf('\n'); - - if (lastNewline > maxLength / 2) { - // If we can keep most content by truncating to last complete line - snippet = truncated.substring(0, lastNewline); - } else { - // Otherwise just truncate and add ellipsis - const lastSpace = truncated.lastIndexOf(' '); - snippet = truncated.substring(0, lastSpace > maxLength / 2 ? lastSpace : maxLength - 3); - snippet = `${snippet }...`; - } - } - - return snippet; - } catch (e) { - log.error(`Error extracting attribute snippet for note ${noteId}: ${e}`); - return ""; - } -} - -function searchNotesForAutocomplete(query: string, fastSearch: boolean = true) { - const searchContext = new SearchContext({ - fastSearch, - includeArchivedNotes: false, - includeHiddenNotes: true, - fuzzyAttributeSearch: true, - ignoreInternalAttributes: true, - ancestorNoteId: hoistedNoteService.isHoistedInHiddenSubtree() ? "root" : hoistedNoteService.getHoistedNoteId() - }); - - const allSearchResults = findResultsWithQuery(query, searchContext); - - const trimmed = allSearchResults.slice(0, 200); - - // Extract content and attribute snippets - for (const result of trimmed) { - result.contentSnippet = extractContentSnippet(result.noteId, searchContext.highlightedTokens); - result.attributeSnippet = extractAttributeSnippet(result.noteId, searchContext.highlightedTokens); - } - - highlightSearchResults(trimmed, searchContext.highlightedTokens, searchContext.ignoreInternalAttributes); - - return trimmed.map((result) => { - const { title, icon } = becca_service.getNoteTitleAndIcon(result.noteId); - return { - notePath: result.notePath, - noteTitle: title, - notePathTitle: result.notePathTitle, - highlightedNotePathTitle: result.highlightedNotePathTitle, - contentSnippet: result.contentSnippet, - highlightedContentSnippet: result.highlightedContentSnippet, - attributeSnippet: result.attributeSnippet, - highlightedAttributeSnippet: result.highlightedAttributeSnippet, - icon: icon ?? "bx bx-note" - }; - }); -} - -/** - * @param ignoreInternalAttributes whether to ignore certain attributes from the search such as ~internalLink. - */ -function highlightSearchResults(searchResults: SearchResult[], highlightedTokens: string[], ignoreInternalAttributes = false) { - highlightedTokens = Array.from(new Set(highlightedTokens)); - - // we remove < signs because they can cause trouble in matching and overwriting existing highlighted chunks - // which would make the resulting HTML string invalid. - // { and } are used for marking and tag (to avoid matches on single 'b' character) - // < and > are used for marking and - highlightedTokens = highlightedTokens.map((token) => token.replace("/[<\{\}]/g", "")).filter((token) => !!token?.trim()); - - // sort by the longest, so we first highlight the longest matches - highlightedTokens.sort((a, b) => (a.length > b.length ? -1 : 1)); - - for (const result of searchResults) { - result.highlightedNotePathTitle = result.notePathTitle.replace(/[<{}]/g, ""); - - // Initialize highlighted content snippet - if (result.contentSnippet) { - // Escape HTML but preserve newlines for later conversion to
- result.highlightedContentSnippet = escapeHtml(result.contentSnippet); - // Remove any stray < { } that might interfere with our highlighting markers - result.highlightedContentSnippet = result.highlightedContentSnippet.replace(/[<{}]/g, ""); - } - - // Initialize highlighted attribute snippet - if (result.attributeSnippet) { - // Escape HTML but preserve newlines for later conversion to
- result.highlightedAttributeSnippet = escapeHtml(result.attributeSnippet); - // Remove any stray < { } that might interfere with our highlighting markers - result.highlightedAttributeSnippet = result.highlightedAttributeSnippet.replace(/[<{}]/g, ""); - } - } - - function wrapText(text: string, start: number, length: number, prefix: string, suffix: string) { - return text.substring(0, start) + prefix + text.substr(start, length) + suffix + text.substring(start + length); - } - - for (const token of highlightedTokens) { - if (!token) { - // Avoid empty tokens, which might cause an infinite loop. - continue; - } - - for (const result of searchResults) { - // Reset token - const tokenRegex = new RegExp(escapeRegExp(token), "gi"); - let match; - - // Highlight in note path title - if (result.highlightedNotePathTitle) { - const titleRegex = new RegExp(escapeRegExp(token), "gi"); - while ((match = titleRegex.exec(normalizeString(result.highlightedNotePathTitle))) !== null) { - result.highlightedNotePathTitle = wrapText(result.highlightedNotePathTitle, match.index, token.length, "{", "}"); - // 2 characters are added, so we need to adjust the index - titleRegex.lastIndex += 2; - } - } - - // Highlight in content snippet - if (result.highlightedContentSnippet) { - const contentRegex = new RegExp(escapeRegExp(token), "gi"); - while ((match = contentRegex.exec(normalizeString(result.highlightedContentSnippet))) !== null) { - result.highlightedContentSnippet = wrapText(result.highlightedContentSnippet, match.index, token.length, "{", "}"); - // 2 characters are added, so we need to adjust the index - contentRegex.lastIndex += 2; - } - } - - // Highlight in attribute snippet - if (result.highlightedAttributeSnippet) { - const attributeRegex = new RegExp(escapeRegExp(token), "gi"); - while ((match = attributeRegex.exec(normalizeString(result.highlightedAttributeSnippet))) !== null) { - result.highlightedAttributeSnippet = wrapText(result.highlightedAttributeSnippet, match.index, token.length, "{", "}"); - // 2 characters are added, so we need to adjust the index - attributeRegex.lastIndex += 2; - } - } - } - } - - for (const result of searchResults) { - if (result.highlightedNotePathTitle) { - result.highlightedNotePathTitle = result.highlightedNotePathTitle.replace(/{/g, "").replace(/}/g, ""); - } - - if (result.highlightedContentSnippet) { - // Replace highlighting markers with HTML tags - result.highlightedContentSnippet = result.highlightedContentSnippet.replace(/{/g, "").replace(/}/g, ""); - // Convert newlines to
tags for HTML display - result.highlightedContentSnippet = result.highlightedContentSnippet.replace(/\n/g, "
"); - } - - if (result.highlightedAttributeSnippet) { - // Replace highlighting markers with HTML tags - result.highlightedAttributeSnippet = result.highlightedAttributeSnippet.replace(/{/g, "").replace(/}/g, ""); - // Convert newlines to
tags for HTML display - result.highlightedAttributeSnippet = result.highlightedAttributeSnippet.replace(/\n/g, "
"); - } - } -} - -export default { - searchFromNote, - searchNotesForAutocomplete, - findResultsWithQuery, - findFirstNoteWithQuery, - searchNotes, - extractContentSnippet, - extractAttributeSnippet, - highlightSearchResults -}; diff --git a/apps/server/src/services/utils.ts b/apps/server/src/services/utils.ts index dbdde6c7ee..56175e5027 100644 --- a/apps/server/src/services/utils.ts +++ b/apps/server/src/services/utils.ts @@ -105,10 +105,6 @@ export function stripTags(text: string) { return text.replace(/<(?:.|\n)*?>/gm, ""); } -export function escapeRegExp(str: string) { - return str.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1"); -} - export async function crash(message: string) { if (isElectron) { const electron = await import("electron"); @@ -450,6 +446,8 @@ function slugify(text: string) { /** @deprecated */ export const escapeHtml = coreUtils.escapeHtml; /** @deprecated */ +export const escapeRegExp = coreUtils.escapeRegExp; +/** @deprecated */ export const unescapeHtml = coreUtils.unescapeHtml; /** @deprecated */ export const randomSecureToken = coreUtils.randomSecureToken; diff --git a/packages/trilium-core/src/index.ts b/packages/trilium-core/src/index.ts index 5e5d7f652a..4f9e0e6188 100644 --- a/packages/trilium-core/src/index.ts +++ b/packages/trilium-core/src/index.ts @@ -40,6 +40,7 @@ export { default as erase } from "./services/erase"; export { default as getSharedBootstrapItems } from "./services/bootstrap_utils"; export { default as branches } from "./services/branches"; export { default as bulk_actions } from "./services/bulk_actions"; +export { default as hoisted_note } from "./services/hoisted_note"; export { default as attribute_formatter} from "./services/attribute_formatter"; diff --git a/apps/server/src/routes/api/search.ts b/packages/trilium-core/src/routes/api/search.ts similarity index 95% rename from apps/server/src/routes/api/search.ts rename to packages/trilium-core/src/routes/api/search.ts index eb65116e34..bd2491f9af 100644 --- a/apps/server/src/routes/api/search.ts +++ b/packages/trilium-core/src/routes/api/search.ts @@ -1,14 +1,15 @@ -import { becca_service,ValidationError } from "@triliumnext/core"; import type { Request } from "express"; import becca from "../../becca/becca.js"; import attributeFormatter from "../../services/attribute_formatter.js"; import bulkActionService from "../../services/bulk_actions.js"; -import cls from "../../services/cls.js"; import hoistedNoteService from "../../services/hoisted_note.js"; import SearchContext from "../../services/search/search_context.js"; import type SearchResult from "../../services/search/search_result.js"; import searchService, { EMPTY_RESULT, type SearchNoteResult } from "../../services/search/services/search.js"; +import { ValidationError } from "../../errors.js"; +import becca_service from "../../becca/becca_service.js"; +import { getHoistedNoteId } from "../../services/context.js"; function searchFromNote(req: Request<{ noteId: string }>): SearchNoteResult { const note = becca.getNoteOrThrow(req.params.noteId); @@ -146,7 +147,7 @@ function getRelatedNotes(req: Request) { } function searchTemplates() { - const query = cls.getHoistedNoteId() === "root" ? "#template" : "#template OR #workspaceTemplate"; + const query = getHoistedNoteId() === "root" ? "#template" : "#template OR #workspaceTemplate"; return searchService .searchNotes(query, { diff --git a/packages/trilium-core/src/routes/index.ts b/packages/trilium-core/src/routes/index.ts index 429e9b3b3d..0008314f0d 100644 --- a/packages/trilium-core/src/routes/index.ts +++ b/packages/trilium-core/src/routes/index.ts @@ -17,6 +17,7 @@ import revisionsApiRoute from "./api/revisions"; import relationMapApiRoute from "./api/relation-map"; import recentChangesApiRoute from "./api/recent_changes"; import bulkActionRoute from "./api/bulk_action"; +import searchRoute from "./api/search"; // TODO: Deduplicate with routes.ts const GET = "get", @@ -92,6 +93,13 @@ export function buildSharedApiRoutes(apiRoute: any) { apiRoute(PUT, "/api/branches/:branchId/set-prefix", branchesApiRoute.setPrefix); apiRoute(PUT, "/api/branches/set-prefix-batch", branchesApiRoute.setPrefixBatch); + apiRoute(GET, "/api/quick-search/:searchString", searchRoute.quickSearch); + apiRoute(GET, "/api/search-note/:noteId", searchRoute.searchFromNote); + apiRoute(PST, "/api/search-and-execute-note/:noteId", searchRoute.searchAndExecute); + apiRoute(PST, "/api/search-related", searchRoute.getRelatedNotes); + apiRoute(GET, "/api/search/:searchString", searchRoute.search); + apiRoute(GET, "/api/search-templates", searchRoute.searchTemplates); + apiRoute(PUT, "/api/notes/:noteId/clone-to-branch/:parentBranchId", cloningApiRoute.cloneNoteToBranch); apiRoute(PUT, "/api/notes/:noteId/toggle-in-parent/:parentNoteId/:present", cloningApiRoute.toggleNoteInParent); apiRoute(PUT, "/api/notes/:noteId/clone-to-note/:parentNoteId", cloningApiRoute.cloneNoteToParentNote); diff --git a/packages/trilium-core/src/services/hoisted_note.ts b/packages/trilium-core/src/services/hoisted_note.ts new file mode 100644 index 0000000000..1e2ffc1008 --- /dev/null +++ b/packages/trilium-core/src/services/hoisted_note.ts @@ -0,0 +1,40 @@ +import becca from "../becca/becca.js"; +import * as cls from "./context.js"; + +function getHoistedNoteId() { + return cls.getHoistedNoteId(); +} + +function isHoistedInHiddenSubtree() { + const hoistedNoteId = getHoistedNoteId(); + + if (hoistedNoteId === "root") { + return false; + } else if (hoistedNoteId === "_hidden") { + return true; + } + + const hoistedNote = becca.getNote(hoistedNoteId); + + if (!hoistedNote) { + throw new Error(`Cannot find hoisted note '${hoistedNoteId}'`); + } + + return hoistedNote.isHiddenCompletely(); +} + +function getWorkspaceNote() { + const hoistedNote = becca.getNote(getHoistedNoteId()); + + if (hoistedNote && (hoistedNote.isRoot() || hoistedNote.hasLabel("workspace"))) { + return hoistedNote; + } else { + return becca.getRoot(); + } +} + +export default { + getHoistedNoteId, + getWorkspaceNote, + isHoistedInHiddenSubtree +}; diff --git a/packages/trilium-core/src/services/script.ts b/packages/trilium-core/src/services/script.ts index 4760e21cc1..83dc0272b6 100644 --- a/packages/trilium-core/src/services/script.ts +++ b/packages/trilium-core/src/services/script.ts @@ -1,3 +1,11 @@ +import type BNote from "../becca/entities/bnote"; + export function executeNoteNoException(script: unknown) { console.warn("Skipped script execution"); } + +export default { + executeNote(scriptNote: BNote, args: {}) { + console.warn("Note not executed"); + } +} diff --git a/apps/server/src/services/search/expressions/ancestor.ts b/packages/trilium-core/src/services/search/expressions/ancestor.ts similarity index 90% rename from apps/server/src/services/search/expressions/ancestor.ts rename to packages/trilium-core/src/services/search/expressions/ancestor.ts index 3c0ecf0df1..a5200e7e27 100644 --- a/apps/server/src/services/search/expressions/ancestor.ts +++ b/packages/trilium-core/src/services/search/expressions/ancestor.ts @@ -2,7 +2,7 @@ import Expression from "./expression.js"; import NoteSet from "../note_set.js"; -import log from "../../log.js"; +import log, { getLog } from "../../log.js"; import becca from "../../../becca/becca.js"; import type SearchContext from "../search_context.js"; @@ -24,7 +24,7 @@ class AncestorExp extends Expression { const ancestorNote = becca.notes[this.ancestorNoteId]; if (!ancestorNote) { - log.error(`Subtree note '${this.ancestorNoteId}' was not not found.`); + getLog().error(`Subtree note '${this.ancestorNoteId}' was not not found.`); return new NoteSet([]); } @@ -64,7 +64,7 @@ class AncestorExp extends Expression { } else if (depthCondition.startsWith("lt")) { return (depth) => depth < comparedDepth; } else { - log.error(`Unrecognized depth condition value ${depthCondition}`); + getLog().error(`Unrecognized depth condition value ${depthCondition}`); return null; } } diff --git a/apps/server/src/services/search/expressions/and.ts b/packages/trilium-core/src/services/search/expressions/and.ts similarity index 100% rename from apps/server/src/services/search/expressions/and.ts rename to packages/trilium-core/src/services/search/expressions/and.ts diff --git a/apps/server/src/services/search/expressions/attribute_exists.ts b/packages/trilium-core/src/services/search/expressions/attribute_exists.ts similarity index 100% rename from apps/server/src/services/search/expressions/attribute_exists.ts rename to packages/trilium-core/src/services/search/expressions/attribute_exists.ts diff --git a/apps/server/src/services/search/expressions/child_of.ts b/packages/trilium-core/src/services/search/expressions/child_of.ts similarity index 100% rename from apps/server/src/services/search/expressions/child_of.ts rename to packages/trilium-core/src/services/search/expressions/child_of.ts diff --git a/apps/server/src/services/search/expressions/descendant_of.ts b/packages/trilium-core/src/services/search/expressions/descendant_of.ts similarity index 100% rename from apps/server/src/services/search/expressions/descendant_of.ts rename to packages/trilium-core/src/services/search/expressions/descendant_of.ts diff --git a/apps/server/src/services/search/expressions/expression.ts b/packages/trilium-core/src/services/search/expressions/expression.ts similarity index 100% rename from apps/server/src/services/search/expressions/expression.ts rename to packages/trilium-core/src/services/search/expressions/expression.ts diff --git a/apps/server/src/services/search/expressions/is_hidden.ts b/packages/trilium-core/src/services/search/expressions/is_hidden.ts similarity index 100% rename from apps/server/src/services/search/expressions/is_hidden.ts rename to packages/trilium-core/src/services/search/expressions/is_hidden.ts diff --git a/apps/server/src/services/search/expressions/label_comparison.ts b/packages/trilium-core/src/services/search/expressions/label_comparison.ts similarity index 100% rename from apps/server/src/services/search/expressions/label_comparison.ts rename to packages/trilium-core/src/services/search/expressions/label_comparison.ts diff --git a/apps/server/src/services/search/expressions/not.ts b/packages/trilium-core/src/services/search/expressions/not.ts similarity index 100% rename from apps/server/src/services/search/expressions/not.ts rename to packages/trilium-core/src/services/search/expressions/not.ts diff --git a/apps/server/src/services/search/expressions/note_content_fulltext.spec.ts b/packages/trilium-core/src/services/search/expressions/note_content_fulltext.spec.ts similarity index 100% rename from apps/server/src/services/search/expressions/note_content_fulltext.spec.ts rename to packages/trilium-core/src/services/search/expressions/note_content_fulltext.spec.ts diff --git a/apps/server/src/services/search/expressions/note_content_fulltext.ts b/packages/trilium-core/src/services/search/expressions/note_content_fulltext.ts similarity index 98% rename from apps/server/src/services/search/expressions/note_content_fulltext.ts rename to packages/trilium-core/src/services/search/expressions/note_content_fulltext.ts index f3e0a39333..23603d80d0 100644 --- a/apps/server/src/services/search/expressions/note_content_fulltext.ts +++ b/packages/trilium-core/src/services/search/expressions/note_content_fulltext.ts @@ -1,9 +1,8 @@ import type { NoteRow } from "@triliumnext/commons"; import becca from "../../../becca/becca.js"; -import log from "../../log.js"; +import { getLog } from "../../log.js"; import protectedSessionService from "../../protected_session.js"; -import sql from "../../sql.js"; import NoteSet from "../note_set.js"; import type SearchContext from "../search_context.js"; import { @@ -14,6 +13,7 @@ import { validateFuzzySearchTokens} from "../utils/text_utils.js"; import Expression from "./expression.js"; import preprocessContent from "./note_content_fulltext_preprocessor.js"; +import { getSql } from "../../../services/sql/index.js"; const ALLOWED_OPERATORS = new Set(["=", "!=", "*=*", "*=", "=*", "%=", "~=", "~*"]); @@ -80,7 +80,7 @@ class NoteContentFulltextExp extends Expression { const resultNoteSet = new NoteSet(); // Search through notes with content - for (const row of sql.iterateRows(` + for (const row of getSql().iterateRows(` SELECT noteId, type, mime, content, isProtected FROM notes JOIN blobs USING (blobId) WHERE type IN ('text', 'code', 'mermaid', 'canvas', 'mindMap') @@ -204,7 +204,7 @@ class NoteContentFulltextExp extends Expression { try { content = protectedSessionService.decryptString(content) || undefined; } catch (e) { - log.info(`Cannot decrypt content of note ${noteId}`); + getLog().info(`Cannot decrypt content of note ${noteId}`); return; } } @@ -327,7 +327,7 @@ class NoteContentFulltextExp extends Expression { return this.fuzzyMatchToken(token, normalizedContent) || (this.flatText && this.fuzzyMatchToken(token, flatText)); } catch (error) { - log.error(`Error in fuzzy matching for note ${noteId}: ${error}`); + getLog().error(`Error in fuzzy matching for note ${noteId}: ${error}`); return false; } } diff --git a/apps/server/src/services/search/expressions/note_content_fulltext_preprocessor.spec.ts b/packages/trilium-core/src/services/search/expressions/note_content_fulltext_preprocessor.spec.ts similarity index 100% rename from apps/server/src/services/search/expressions/note_content_fulltext_preprocessor.spec.ts rename to packages/trilium-core/src/services/search/expressions/note_content_fulltext_preprocessor.spec.ts diff --git a/apps/server/src/services/search/expressions/note_content_fulltext_preprocessor.ts b/packages/trilium-core/src/services/search/expressions/note_content_fulltext_preprocessor.ts similarity index 96% rename from apps/server/src/services/search/expressions/note_content_fulltext_preprocessor.ts rename to packages/trilium-core/src/services/search/expressions/note_content_fulltext_preprocessor.ts index f0652e0722..f53a941be6 100644 --- a/apps/server/src/services/search/expressions/note_content_fulltext_preprocessor.ts +++ b/packages/trilium-core/src/services/search/expressions/note_content_fulltext_preprocessor.ts @@ -1,6 +1,6 @@ import striptags from "striptags"; - -import { normalize } from "../../utils.js"; +import { normalizeSearchText } from "../utils/text_utils"; +import { normalize } from "../../utils/index"; export default function preprocessContent(rawContent: string | Uint8Array, type: string, mime: string, raw?: boolean) { let content = normalize(rawContent.toString()); @@ -77,7 +77,7 @@ function processMindmapContent(content: string) { // Combine topics into a single string const topicsString = topicsArray.join(", "); - return normalize(topicsString.toString()); + return normalizeSearchText(topicsString.toString()); } function processCanvasContent(content: string) { diff --git a/apps/server/src/services/search/expressions/note_flat_text.ts b/packages/trilium-core/src/services/search/expressions/note_flat_text.ts similarity index 98% rename from apps/server/src/services/search/expressions/note_flat_text.ts rename to packages/trilium-core/src/services/search/expressions/note_flat_text.ts index 18bb0944d8..950fe59bee 100644 --- a/apps/server/src/services/search/expressions/note_flat_text.ts +++ b/packages/trilium-core/src/services/search/expressions/note_flat_text.ts @@ -1,8 +1,6 @@ -import { becca_service } from "@triliumnext/core"; - +import becca_service from "../../../becca/becca_service.js"; import becca from "../../../becca/becca.js"; import type BNote from "../../../becca/entities/bnote.js"; -import { normalize } from "../../utils.js"; import NoteSet from "../note_set.js"; import type SearchContext from "../search_context.js"; import { fuzzyMatchWord, fuzzyMatchWordWithResult,normalizeSearchText } from "../utils/text_utils.js"; diff --git a/apps/server/src/services/search/expressions/or.ts b/packages/trilium-core/src/services/search/expressions/or.ts similarity index 100% rename from apps/server/src/services/search/expressions/or.ts rename to packages/trilium-core/src/services/search/expressions/or.ts diff --git a/apps/server/src/services/search/expressions/order_by_and_limit.ts b/packages/trilium-core/src/services/search/expressions/order_by_and_limit.ts similarity index 100% rename from apps/server/src/services/search/expressions/order_by_and_limit.ts rename to packages/trilium-core/src/services/search/expressions/order_by_and_limit.ts diff --git a/apps/server/src/services/search/expressions/parent_of.ts b/packages/trilium-core/src/services/search/expressions/parent_of.ts similarity index 100% rename from apps/server/src/services/search/expressions/parent_of.ts rename to packages/trilium-core/src/services/search/expressions/parent_of.ts diff --git a/apps/server/src/services/search/expressions/property_comparison.ts b/packages/trilium-core/src/services/search/expressions/property_comparison.ts similarity index 100% rename from apps/server/src/services/search/expressions/property_comparison.ts rename to packages/trilium-core/src/services/search/expressions/property_comparison.ts diff --git a/apps/server/src/services/search/expressions/relation_where.ts b/packages/trilium-core/src/services/search/expressions/relation_where.ts similarity index 100% rename from apps/server/src/services/search/expressions/relation_where.ts rename to packages/trilium-core/src/services/search/expressions/relation_where.ts diff --git a/apps/server/src/services/search/expressions/true.ts b/packages/trilium-core/src/services/search/expressions/true.ts similarity index 100% rename from apps/server/src/services/search/expressions/true.ts rename to packages/trilium-core/src/services/search/expressions/true.ts diff --git a/apps/server/src/services/search/search_context.ts b/packages/trilium-core/src/services/search/search_context.ts similarity index 100% rename from apps/server/src/services/search/search_context.ts rename to packages/trilium-core/src/services/search/search_context.ts diff --git a/apps/server/src/services/search/search_result.ts b/packages/trilium-core/src/services/search/search_result.ts similarity index 99% rename from apps/server/src/services/search/search_result.ts rename to packages/trilium-core/src/services/search/search_result.ts index 984a6d97a8..ef99daba65 100644 --- a/apps/server/src/services/search/search_result.ts +++ b/packages/trilium-core/src/services/search/search_result.ts @@ -1,4 +1,4 @@ -import { becca_service } from "@triliumnext/core"; +import becca_service from "../../becca/becca_service"; import becca from "../../becca/becca.js"; import { diff --git a/apps/server/src/services/search/services/build_comparator.ts b/packages/trilium-core/src/services/search/services/build_comparator.ts similarity index 100% rename from apps/server/src/services/search/services/build_comparator.ts rename to packages/trilium-core/src/services/search/services/build_comparator.ts diff --git a/apps/server/src/services/search/services/handle_parens.spec.ts b/packages/trilium-core/src/services/search/services/handle_parens.spec.ts similarity index 100% rename from apps/server/src/services/search/services/handle_parens.spec.ts rename to packages/trilium-core/src/services/search/services/handle_parens.spec.ts diff --git a/apps/server/src/services/search/services/handle_parens.ts b/packages/trilium-core/src/services/search/services/handle_parens.ts similarity index 100% rename from apps/server/src/services/search/services/handle_parens.ts rename to packages/trilium-core/src/services/search/services/handle_parens.ts diff --git a/apps/server/src/services/search/services/lex.spec.ts b/packages/trilium-core/src/services/search/services/lex.spec.ts similarity index 100% rename from apps/server/src/services/search/services/lex.spec.ts rename to packages/trilium-core/src/services/search/services/lex.spec.ts diff --git a/apps/server/src/services/search/services/lex.ts b/packages/trilium-core/src/services/search/services/lex.ts similarity index 100% rename from apps/server/src/services/search/services/lex.ts rename to packages/trilium-core/src/services/search/services/lex.ts diff --git a/apps/server/src/services/search/services/parse.spec.ts b/packages/trilium-core/src/services/search/services/parse.spec.ts similarity index 100% rename from apps/server/src/services/search/services/parse.spec.ts rename to packages/trilium-core/src/services/search/services/parse.spec.ts diff --git a/apps/server/src/services/search/services/parse.ts b/packages/trilium-core/src/services/search/services/parse.ts similarity index 99% rename from apps/server/src/services/search/services/parse.ts rename to packages/trilium-core/src/services/search/services/parse.ts index f2ddf31647..00e918cdcf 100644 --- a/apps/server/src/services/search/services/parse.ts +++ b/packages/trilium-core/src/services/search/services/parse.ts @@ -17,7 +17,7 @@ import OrderByAndLimitExp from "../expressions/order_by_and_limit.js"; import AncestorExp from "../expressions/ancestor.js"; import buildComparator from "./build_comparator.js"; import ValueExtractor from "../value_extractor.js"; -import { removeDiacritic } from "../../utils.js"; +import { removeDiacritic } from "../../utils/index.js"; import TrueExp from "../expressions/true.js"; import IsHiddenExp from "../expressions/is_hidden.js"; import type SearchContext from "../search_context.js"; diff --git a/apps/server/src/services/search/services/progressive_search.spec.ts b/packages/trilium-core/src/services/search/services/progressive_search.spec.ts similarity index 100% rename from apps/server/src/services/search/services/progressive_search.spec.ts rename to packages/trilium-core/src/services/search/services/progressive_search.spec.ts diff --git a/apps/server/src/services/search/services/search.spec.ts b/packages/trilium-core/src/services/search/services/search.spec.ts similarity index 100% rename from apps/server/src/services/search/services/search.spec.ts rename to packages/trilium-core/src/services/search/services/search.spec.ts diff --git a/packages/trilium-core/src/services/search/services/search.ts b/packages/trilium-core/src/services/search/services/search.ts index f344e86565..6c0b3dc2c6 100644 --- a/packages/trilium-core/src/services/search/services/search.ts +++ b/packages/trilium-core/src/services/search/services/search.ts @@ -1,12 +1,796 @@ -import BNote from "src/becca/entities/bnote"; +import normalizeString from "normalize-strings"; +import striptags from "striptags"; -export default { - searchFromNote(note: BNote) { - console.warn("Ignore search ", note.title); - }, +import becca from "../../../becca/becca.js"; +import becca_service from "../../../becca/becca_service.js"; +import type BNote from "../../../becca/entities/bnote.js"; +import hoistedNoteService from "../../hoisted_note.js"; +import { getLog } from "../../log.js"; +import protectedSessionService from "../../protected_session.js"; +import scriptService from "../../script.js"; +import { escapeHtml, escapeRegExp } from "../../utils/index.js"; +import type Expression from "../expressions/expression.js"; +import SearchContext from "../search_context.js"; +import SearchResult from "../search_result.js"; +import handleParens from "./handle_parens.js"; +import lex from "./lex.js"; +import parse from "./parse.js"; +import type { SearchParams, TokenStructure } from "./types.js"; +import { getSql } from "../../sql/index.js"; + +export interface SearchNoteResult { + searchResultNoteIds: string[]; + highlightedTokens: string[]; + error: string | null; +} + +export const EMPTY_RESULT: SearchNoteResult = { + searchResultNoteIds: [], + highlightedTokens: [], + error: null +}; + +function searchFromNote(note: BNote): SearchNoteResult { + let searchResultNoteIds; + let highlightedTokens: string[]; + + const searchScript = note.getRelationValue("searchScript"); + const searchString = note.getLabelValue("searchString") || ""; + let error: string | null = null; + + if (searchScript) { + searchResultNoteIds = searchFromRelation(note, "searchScript"); + highlightedTokens = []; + } else { + const searchContext = new SearchContext({ + fastSearch: note.hasLabel("fastSearch"), + ancestorNoteId: note.getRelationValue("ancestor") || undefined, + ancestorDepth: note.getLabelValue("ancestorDepth") || undefined, + includeArchivedNotes: note.hasLabel("includeArchivedNotes"), + orderBy: note.getLabelValue("orderBy") || undefined, + orderDirection: note.getLabelValue("orderDirection") || undefined, + limit: parseInt(note.getLabelValue("limit") || "0", 10), + debug: note.hasLabel("debug"), + fuzzyAttributeSearch: false + }); + + searchResultNoteIds = findResultsWithQuery(searchString, searchContext).map((sr) => sr.noteId); + + highlightedTokens = searchContext.highlightedTokens; + error = searchContext.getError(); + } + + // we won't return search note's own noteId + // also don't allow root since that would force infinite cycle + return { + searchResultNoteIds: searchResultNoteIds.filter((resultNoteId) => !["root", note.noteId].includes(resultNoteId)), + highlightedTokens, + error + }; +} + +function searchFromRelation(note: BNote, relationName: string) { + const scriptNote = note.getRelationTarget(relationName); + + const log = getLog(); + if (!scriptNote) { + log.info(`Search note's relation ${relationName} has not been found.`); - searchNotes(searchString: string, opts?: {}): BNote[] { - console.warn("Ignore search", searchString); return []; } + + if (!scriptNote.isJavaScript() || scriptNote.getScriptEnv() !== "backend") { + log.info(`Note ${scriptNote.noteId} is not executable.`); + + return []; + } + + if (!note.isContentAvailable()) { + log.info(`Note ${scriptNote.noteId} is not available outside of protected session.`); + + return []; + } + + const result = scriptService.executeNote(scriptNote, { originEntity: note }); + + if (!Array.isArray(result)) { + log.info(`Result from ${scriptNote.noteId} is not an array.`); + + return []; + } + + if (result.length === 0) { + return []; + } + + // we expect either array of noteIds (strings) or notes, in that case we extract noteIds ourselves + return typeof result[0] === "string" ? result : result.map((item) => item.noteId); } + +function loadNeededInfoFromDatabase() { + /** + * This complex structure is needed to calculate total occupied space by a note. Several object instances + * (note, revisions, attachments) can point to a single blobId, and thus the blob size should count towards the total + * only once. + * + * noteId => { blobId => blobSize } + */ + const noteBlobs: Record> = {}; + + type NoteContentLengthsRow = { + noteId: string; + blobId: string; + length: number; + }; + const log = getLog(); + const noteContentLengths = getSql().getRows(` + SELECT + noteId, + blobId, + LENGTH(content) AS length + FROM notes + JOIN blobs USING(blobId) + WHERE notes.isDeleted = 0`); + + for (const { noteId, blobId, length } of noteContentLengths) { + if (!(noteId in becca.notes)) { + log.error(`Note '${noteId}' not found in becca.`); + continue; + } + + becca.notes[noteId].contentSize = length; + becca.notes[noteId].revisionCount = 0; + + noteBlobs[noteId] = { [blobId]: length }; + } + + type AttachmentContentLengthsRow = { + noteId: string; + blobId: string; + length: number; + }; + const attachmentContentLengths = getSql().getRows(` + SELECT + ownerId AS noteId, + attachments.blobId, + LENGTH(content) AS length + FROM attachments + JOIN notes ON attachments.ownerId = notes.noteId + JOIN blobs ON attachments.blobId = blobs.blobId + WHERE attachments.isDeleted = 0 + AND notes.isDeleted = 0`); + + for (const { noteId, blobId, length } of attachmentContentLengths) { + if (!(noteId in becca.notes)) { + log.error(`Note '${noteId}' not found in becca.`); + continue; + } + + if (!(noteId in noteBlobs)) { + log.error(`Did not find a '${noteId}' in the noteBlobs.`); + continue; + } + + noteBlobs[noteId][blobId] = length; + } + + for (const noteId in noteBlobs) { + becca.notes[noteId].contentAndAttachmentsSize = Object.values(noteBlobs[noteId]).reduce((acc, size) => acc + size, 0); + } + + type RevisionRow = { + noteId: string; + blobId: string; + length: number; + isNoteRevision: true; + }; + const revisionContentLengths = getSql().getRows(` + SELECT + noteId, + revisions.blobId, + LENGTH(content) AS length, + 1 AS isNoteRevision + FROM notes + JOIN revisions USING(noteId) + JOIN blobs ON revisions.blobId = blobs.blobId + WHERE notes.isDeleted = 0 + UNION ALL + SELECT + noteId, + revisions.blobId, + LENGTH(content) AS length, + 0 AS isNoteRevision -- it's attachment not counting towards revision count + FROM notes + JOIN revisions USING(noteId) + JOIN attachments ON attachments.ownerId = revisions.revisionId + JOIN blobs ON attachments.blobId = blobs.blobId + WHERE notes.isDeleted = 0`); + + for (const { noteId, blobId, length, isNoteRevision } of revisionContentLengths) { + if (!(noteId in becca.notes)) { + log.error(`Note '${noteId}' not found in becca.`); + continue; + } + + if (!(noteId in noteBlobs)) { + log.error(`Did not find a '${noteId}' in the noteBlobs.`); + continue; + } + + noteBlobs[noteId][blobId] = length; + + if (isNoteRevision) { + const noteRevision = becca.notes[noteId]; + if (noteRevision && noteRevision.revisionCount) { + noteRevision.revisionCount++; + } + } + } + + for (const noteId in noteBlobs) { + becca.notes[noteId].contentAndAttachmentsAndRevisionsSize = Object.values(noteBlobs[noteId]).reduce((acc, size) => acc + size, 0); + } +} + +function findResultsWithExpression(expression: Expression, searchContext: SearchContext): SearchResult[] { + if (searchContext.dbLoadNeeded) { + loadNeededInfoFromDatabase(); + } + + // If there's an explicit orderBy clause, skip progressive search + // as it would interfere with the ordering + if (searchContext.orderBy) { + // For ordered queries, don't use progressive search but respect + // the original fuzzy matching setting + return performSearch(expression, searchContext, searchContext.enableFuzzyMatching); + } + + // If fuzzy matching is explicitly disabled, skip progressive search + if (!searchContext.enableFuzzyMatching) { + return performSearch(expression, searchContext, false); + } + + // Phase 1: Try exact matches first (without fuzzy matching) + const exactResults = performSearch(expression, searchContext, false); + + // Check if we have sufficient high-quality results + const minResultThreshold = 5; + const minScoreForQuality = 10; // Minimum score to consider a result "high quality" + + const highQualityResults = exactResults.filter(result => result.score >= minScoreForQuality); + + // If we have enough high-quality exact matches, return them + if (highQualityResults.length >= minResultThreshold) { + return exactResults; + } + + // Phase 2: Add fuzzy matching as fallback when exact matches are insufficient + const fuzzyResults = performSearch(expression, searchContext, true); + + // Merge results, ensuring exact matches always rank higher than fuzzy matches + return mergeExactAndFuzzyResults(exactResults, fuzzyResults); +} + +function performSearch(expression: Expression, searchContext: SearchContext, enableFuzzyMatching: boolean): SearchResult[] { + const allNoteSet = becca.getAllNoteSet(); + + const noteIdToNotePath: Record = {}; + const executionContext = { + noteIdToNotePath + }; + + // Store original fuzzy setting and temporarily override it + const originalFuzzyMatching = searchContext.enableFuzzyMatching; + searchContext.enableFuzzyMatching = enableFuzzyMatching; + + const noteSet = expression.execute(allNoteSet, executionContext, searchContext); + + const searchResults = noteSet.notes.map((note) => { + const notePathArray = executionContext.noteIdToNotePath[note.noteId] || note.getBestNotePath(); + + if (!notePathArray) { + throw new Error(`Can't find note path for note ${JSON.stringify(note.getPojo())}`); + } + + return new SearchResult(notePathArray); + }); + + for (const res of searchResults) { + res.computeScore(searchContext.fulltextQuery, searchContext.highlightedTokens, enableFuzzyMatching); + } + + // Restore original fuzzy setting + searchContext.enableFuzzyMatching = originalFuzzyMatching; + + if (!noteSet.sorted) { + searchResults.sort((a, b) => { + if (a.score > b.score) { + return -1; + } else if (a.score < b.score) { + return 1; + } + + // if score does not decide then sort results by depth of the note. + // This is based on the assumption that more important results are closer to the note root. + if (a.notePathArray.length === b.notePathArray.length) { + return a.notePathTitle < b.notePathTitle ? -1 : 1; + } + + return a.notePathArray.length < b.notePathArray.length ? -1 : 1; + }); + } + + return searchResults; +} + +function mergeExactAndFuzzyResults(exactResults: SearchResult[], fuzzyResults: SearchResult[]): SearchResult[] { + // Create a map of exact result note IDs for deduplication + const exactNoteIds = new Set(exactResults.map(result => result.noteId)); + + // Add fuzzy results that aren't already in exact results + const additionalFuzzyResults = fuzzyResults.filter(result => !exactNoteIds.has(result.noteId)); + + // Sort exact results by score (best exact matches first) + exactResults.sort((a, b) => { + if (a.score > b.score) { + return -1; + } else if (a.score < b.score) { + return 1; + } + + // if score does not decide then sort results by depth of the note. + if (a.notePathArray.length === b.notePathArray.length) { + return a.notePathTitle < b.notePathTitle ? -1 : 1; + } + + return a.notePathArray.length < b.notePathArray.length ? -1 : 1; + }); + + // Sort fuzzy results by score (best fuzzy matches first) + additionalFuzzyResults.sort((a, b) => { + if (a.score > b.score) { + return -1; + } else if (a.score < b.score) { + return 1; + } + + // if score does not decide then sort results by depth of the note. + if (a.notePathArray.length === b.notePathArray.length) { + return a.notePathTitle < b.notePathTitle ? -1 : 1; + } + + return a.notePathArray.length < b.notePathArray.length ? -1 : 1; + }); + + // CRITICAL: Always put exact matches before fuzzy matches, regardless of scores + return [...exactResults, ...additionalFuzzyResults]; +} + +function parseQueryToExpression(query: string, searchContext: SearchContext) { + const { fulltextQuery, fulltextTokens, expressionTokens, leadingOperator } = lex(query); + searchContext.fulltextQuery = fulltextQuery; + + let structuredExpressionTokens: TokenStructure; + + try { + structuredExpressionTokens = handleParens(expressionTokens); + } catch (e: any) { + structuredExpressionTokens = []; + searchContext.addError(e.message); + } + + const expression = parse({ + fulltextTokens, + expressionTokens: structuredExpressionTokens, + searchContext, + originalQuery: query, + leadingOperator + }); + + if (searchContext.debug) { + searchContext.debugInfo = { + fulltextTokens, + structuredExpressionTokens, + expression + }; + + getLog().info(`Search debug: ${JSON.stringify(searchContext.debugInfo, null, 4)}`); + } + + return expression; +} + +function searchNotes(query: string, params: SearchParams = {}): BNote[] { + const searchResults = findResultsWithQuery(query, new SearchContext(params)); + + return searchResults.map((sr) => becca.notes[sr.noteId]); +} + +function findResultsWithQuery(query: string, searchContext: SearchContext): SearchResult[] { + query = query || ""; + searchContext.originalQuery = query; + + const expression = parseQueryToExpression(query, searchContext); + + if (!expression) { + return []; + } + + // If the query starts with '#', it's a pure expression query. + // Don't use progressive search for these as they may have complex + // ordering or other logic that shouldn't be interfered with. + const isPureExpressionQuery = query.trim().startsWith('#'); + + if (isPureExpressionQuery) { + // For pure expression queries, use standard search without progressive phases + return performSearch(expression, searchContext, searchContext.enableFuzzyMatching); + } + + return findResultsWithExpression(expression, searchContext); +} + +function findFirstNoteWithQuery(query: string, searchContext: SearchContext): BNote | null { + const searchResults = findResultsWithQuery(query, searchContext); + + return searchResults.length > 0 ? becca.notes[searchResults[0].noteId] : null; +} + +function extractContentSnippet(noteId: string, searchTokens: string[], maxLength: number = 200): string { + const note = becca.notes[noteId]; + if (!note) { + return ""; + } + + // Only extract content for text-based notes + if (!["text", "code", "mermaid", "canvas", "mindMap"].includes(note.type)) { + return ""; + } + + try { + let content = note.getContent(); + + if (!content || typeof content !== "string") { + return ""; + } + + // Handle protected notes + if (note.isProtected && protectedSessionService.isProtectedSessionAvailable()) { + try { + content = protectedSessionService.decryptString(content) || ""; + } catch (e) { + return ""; // Can't decrypt, don't show content + } + } else if (note.isProtected) { + return ""; // Protected but no session available + } + + // Strip HTML tags for text notes + if (note.type === "text") { + content = striptags(content); + } + + // Normalize whitespace while preserving paragraph breaks + // First, normalize multiple newlines to double newlines (paragraph breaks) + content = content.replace(/\n\s*\n/g, "\n\n"); + // Then normalize spaces within lines + content = content.split('\n').map(line => line.replace(/\s+/g, " ").trim()).join('\n'); + // Finally trim the whole content + content = content.trim(); + + if (!content) { + return ""; + } + + // Try to find a snippet around the first matching token + const normalizedContent = normalizeString(content.toLowerCase()); + let snippetStart = 0; + let matchFound = false; + + for (const token of searchTokens) { + const normalizedToken = normalizeString(token.toLowerCase()); + const matchIndex = normalizedContent.indexOf(normalizedToken); + + if (matchIndex !== -1) { + // Center the snippet around the match + snippetStart = Math.max(0, matchIndex - maxLength / 2); + matchFound = true; + break; + } + } + + // Extract snippet + let snippet = content.substring(snippetStart, snippetStart + maxLength); + + // If snippet contains linebreaks, limit to max 4 lines and override character limit + const lines = snippet.split('\n'); + if (lines.length > 4) { + // Find which lines contain the search tokens to ensure they're included + const normalizedLines = lines.map(line => normalizeString(line.toLowerCase())); + const normalizedTokens = searchTokens.map(token => normalizeString(token.toLowerCase())); + + // Find the first line that contains a search token + let firstMatchLine = -1; + for (let i = 0; i < normalizedLines.length; i++) { + if (normalizedTokens.some(token => normalizedLines[i].includes(token))) { + firstMatchLine = i; + break; + } + } + + if (firstMatchLine !== -1) { + // Center the 4-line window around the first match + // Try to show 1 line before and 2 lines after the match + const startLine = Math.max(0, firstMatchLine - 1); + const endLine = Math.min(lines.length, startLine + 4); + snippet = lines.slice(startLine, endLine).join('\n'); + } else { + // No match found in lines (shouldn't happen), just take first 4 + snippet = lines.slice(0, 4).join('\n'); + } + // Add ellipsis if we truncated lines + snippet = `${snippet }...`; + } else if (lines.length > 1) { + // For multi-line snippets that are 4 or fewer lines, keep them as-is + // No need to truncate + } else { + // Single line content - apply original word boundary logic + // Try to start/end at word boundaries + if (snippetStart > 0) { + const firstSpace = snippet.search(/\s/); + if (firstSpace > 0 && firstSpace < 20) { + snippet = snippet.substring(firstSpace + 1); + } + snippet = `...${ snippet}`; + } + + if (snippetStart + maxLength < content.length) { + const lastSpace = snippet.search(/\s[^\s]*$/); + if (lastSpace > snippet.length - 20 && lastSpace > 0) { + snippet = snippet.substring(0, lastSpace); + } + snippet = `${snippet }...`; + } + } + + return snippet; + } catch (e) { + getLog().error(`Error extracting content snippet for note ${noteId}: ${e}`); + return ""; + } +} + +function extractAttributeSnippet(noteId: string, searchTokens: string[], maxLength: number = 200): string { + const note = becca.notes[noteId]; + if (!note) { + return ""; + } + + try { + // Get all attributes for this note + const attributes = note.getAttributes(); + if (!attributes || attributes.length === 0) { + return ""; + } + + const matchingAttributes: Array<{name: string, value: string, type: string}> = []; + + // Look for attributes that match the search tokens + for (const attr of attributes) { + const attrName = attr.name?.toLowerCase() || ""; + const attrValue = attr.value?.toLowerCase() || ""; + const attrType = attr.type || ""; + + // Check if any search token matches the attribute name or value + const hasMatch = searchTokens.some(token => { + const normalizedToken = normalizeString(token.toLowerCase()); + return attrName.includes(normalizedToken) || attrValue.includes(normalizedToken); + }); + + if (hasMatch) { + matchingAttributes.push({ + name: attr.name || "", + value: attr.value || "", + type: attrType + }); + } + } + + if (matchingAttributes.length === 0) { + return ""; + } + + // Limit to 4 lines maximum, similar to content snippet logic + const lines: string[] = []; + for (const attr of matchingAttributes.slice(0, 4)) { + let line = ""; + if (attr.type === "label") { + line = attr.value ? `#${attr.name}="${attr.value}"` : `#${attr.name}`; + } else if (attr.type === "relation") { + // For relations, show the target note title if possible + const targetNote = attr.value ? becca.notes[attr.value] : null; + const targetTitle = targetNote ? targetNote.title : attr.value; + line = `~${attr.name}="${targetTitle}"`; + } + + if (line) { + lines.push(line); + } + } + + let snippet = lines.join('\n'); + + // Apply length limit while preserving line structure + if (snippet.length > maxLength) { + // Try to truncate at word boundaries but keep lines intact + const truncated = snippet.substring(0, maxLength); + const lastNewline = truncated.lastIndexOf('\n'); + + if (lastNewline > maxLength / 2) { + // If we can keep most content by truncating to last complete line + snippet = truncated.substring(0, lastNewline); + } else { + // Otherwise just truncate and add ellipsis + const lastSpace = truncated.lastIndexOf(' '); + snippet = truncated.substring(0, lastSpace > maxLength / 2 ? lastSpace : maxLength - 3); + snippet = `${snippet }...`; + } + } + + return snippet; + } catch (e) { + getLog().error(`Error extracting attribute snippet for note ${noteId}: ${e}`); + return ""; + } +} + +function searchNotesForAutocomplete(query: string, fastSearch: boolean = true) { + const searchContext = new SearchContext({ + fastSearch, + includeArchivedNotes: false, + includeHiddenNotes: true, + fuzzyAttributeSearch: true, + ignoreInternalAttributes: true, + ancestorNoteId: hoistedNoteService.isHoistedInHiddenSubtree() ? "root" : hoistedNoteService.getHoistedNoteId() + }); + + const allSearchResults = findResultsWithQuery(query, searchContext); + + const trimmed = allSearchResults.slice(0, 200); + + // Extract content and attribute snippets + for (const result of trimmed) { + result.contentSnippet = extractContentSnippet(result.noteId, searchContext.highlightedTokens); + result.attributeSnippet = extractAttributeSnippet(result.noteId, searchContext.highlightedTokens); + } + + highlightSearchResults(trimmed, searchContext.highlightedTokens, searchContext.ignoreInternalAttributes); + + return trimmed.map((result) => { + const { title, icon } = becca_service.getNoteTitleAndIcon(result.noteId); + return { + notePath: result.notePath, + noteTitle: title, + notePathTitle: result.notePathTitle, + highlightedNotePathTitle: result.highlightedNotePathTitle, + contentSnippet: result.contentSnippet, + highlightedContentSnippet: result.highlightedContentSnippet, + attributeSnippet: result.attributeSnippet, + highlightedAttributeSnippet: result.highlightedAttributeSnippet, + icon: icon ?? "bx bx-note" + }; + }); +} + +/** + * @param ignoreInternalAttributes whether to ignore certain attributes from the search such as ~internalLink. + */ +function highlightSearchResults(searchResults: SearchResult[], highlightedTokens: string[], ignoreInternalAttributes = false) { + highlightedTokens = Array.from(new Set(highlightedTokens)); + + // we remove < signs because they can cause trouble in matching and overwriting existing highlighted chunks + // which would make the resulting HTML string invalid. + // { and } are used for marking and tag (to avoid matches on single 'b' character) + // < and > are used for marking and + highlightedTokens = highlightedTokens.map((token) => token.replace("/[<\{\}]/g", "")).filter((token) => !!token?.trim()); + + // sort by the longest, so we first highlight the longest matches + highlightedTokens.sort((a, b) => (a.length > b.length ? -1 : 1)); + + for (const result of searchResults) { + result.highlightedNotePathTitle = result.notePathTitle.replace(/[<{}]/g, ""); + + // Initialize highlighted content snippet + if (result.contentSnippet) { + // Escape HTML but preserve newlines for later conversion to
+ result.highlightedContentSnippet = escapeHtml(result.contentSnippet); + // Remove any stray < { } that might interfere with our highlighting markers + result.highlightedContentSnippet = result.highlightedContentSnippet.replace(/[<{}]/g, ""); + } + + // Initialize highlighted attribute snippet + if (result.attributeSnippet) { + // Escape HTML but preserve newlines for later conversion to
+ result.highlightedAttributeSnippet = escapeHtml(result.attributeSnippet); + // Remove any stray < { } that might interfere with our highlighting markers + result.highlightedAttributeSnippet = result.highlightedAttributeSnippet.replace(/[<{}]/g, ""); + } + } + + function wrapText(text: string, start: number, length: number, prefix: string, suffix: string) { + return text.substring(0, start) + prefix + text.substr(start, length) + suffix + text.substring(start + length); + } + + for (const token of highlightedTokens) { + if (!token) { + // Avoid empty tokens, which might cause an infinite loop. + continue; + } + + for (const result of searchResults) { + // Reset token + const tokenRegex = new RegExp(escapeRegExp(token), "gi"); + let match; + + // Highlight in note path title + if (result.highlightedNotePathTitle) { + const titleRegex = new RegExp(escapeRegExp(token), "gi"); + while ((match = titleRegex.exec(normalizeString(result.highlightedNotePathTitle))) !== null) { + result.highlightedNotePathTitle = wrapText(result.highlightedNotePathTitle, match.index, token.length, "{", "}"); + // 2 characters are added, so we need to adjust the index + titleRegex.lastIndex += 2; + } + } + + // Highlight in content snippet + if (result.highlightedContentSnippet) { + const contentRegex = new RegExp(escapeRegExp(token), "gi"); + while ((match = contentRegex.exec(normalizeString(result.highlightedContentSnippet))) !== null) { + result.highlightedContentSnippet = wrapText(result.highlightedContentSnippet, match.index, token.length, "{", "}"); + // 2 characters are added, so we need to adjust the index + contentRegex.lastIndex += 2; + } + } + + // Highlight in attribute snippet + if (result.highlightedAttributeSnippet) { + const attributeRegex = new RegExp(escapeRegExp(token), "gi"); + while ((match = attributeRegex.exec(normalizeString(result.highlightedAttributeSnippet))) !== null) { + result.highlightedAttributeSnippet = wrapText(result.highlightedAttributeSnippet, match.index, token.length, "{", "}"); + // 2 characters are added, so we need to adjust the index + attributeRegex.lastIndex += 2; + } + } + } + } + + for (const result of searchResults) { + if (result.highlightedNotePathTitle) { + result.highlightedNotePathTitle = result.highlightedNotePathTitle.replace(/{/g, "").replace(/}/g, ""); + } + + if (result.highlightedContentSnippet) { + // Replace highlighting markers with HTML tags + result.highlightedContentSnippet = result.highlightedContentSnippet.replace(/{/g, "").replace(/}/g, ""); + // Convert newlines to
tags for HTML display + result.highlightedContentSnippet = result.highlightedContentSnippet.replace(/\n/g, "
"); + } + + if (result.highlightedAttributeSnippet) { + // Replace highlighting markers with HTML tags + result.highlightedAttributeSnippet = result.highlightedAttributeSnippet.replace(/{/g, "").replace(/}/g, ""); + // Convert newlines to
tags for HTML display + result.highlightedAttributeSnippet = result.highlightedAttributeSnippet.replace(/\n/g, "
"); + } + } +} + +export default { + searchFromNote, + searchNotesForAutocomplete, + findResultsWithQuery, + findFirstNoteWithQuery, + searchNotes, + extractContentSnippet, + extractAttributeSnippet, + highlightSearchResults +}; diff --git a/apps/server/src/services/search/services/types.ts b/packages/trilium-core/src/services/search/services/types.ts similarity index 100% rename from apps/server/src/services/search/services/types.ts rename to packages/trilium-core/src/services/search/services/types.ts diff --git a/apps/server/src/services/search/utils/text_utils.spec.ts b/packages/trilium-core/src/services/search/utils/text_utils.spec.ts similarity index 100% rename from apps/server/src/services/search/utils/text_utils.spec.ts rename to packages/trilium-core/src/services/search/utils/text_utils.spec.ts diff --git a/apps/server/src/services/search/utils/text_utils.ts b/packages/trilium-core/src/services/search/utils/text_utils.ts similarity index 97% rename from apps/server/src/services/search/utils/text_utils.ts rename to packages/trilium-core/src/services/search/utils/text_utils.ts index 9274241cbc..84b6a66d23 100644 --- a/apps/server/src/services/search/utils/text_utils.ts +++ b/packages/trilium-core/src/services/search/utils/text_utils.ts @@ -1,6 +1,6 @@ "use strict"; -import { normalize } from "../../utils.js"; +import { normalize } from "../../utils/index"; /** * Shared text processing utilities for search functionality @@ -31,12 +31,12 @@ export const FUZZY_SEARCH_CONFIG = { * Normalizes text by removing diacritics and converting to lowercase. * This is the centralized text normalization function used across all search components. * Uses the shared normalize function from utils for consistency. - * - * Examples: + * + * Examples: * - "café" -> "cafe" * - "naïve" -> "naive" * - "HELLO WORLD" -> "hello world" - * + * * @param text The text to normalize * @returns The normalized text */ @@ -44,7 +44,7 @@ export function normalizeSearchText(text: string): string { if (!text || typeof text !== 'string') { return ''; } - + // Use shared normalize function for consistency across the codebase return normalize(text); } @@ -53,7 +53,7 @@ export function normalizeSearchText(text: string): string { * Optimized edit distance calculation using single array and early termination. * This is significantly more memory efficient than the 2D matrix approach and includes * early termination optimizations for better performance. - * + * * @param str1 First string * @param str2 Second string * @param maxDistance Maximum allowed distance (for early termination) @@ -64,7 +64,7 @@ export function calculateOptimizedEditDistance(str1: string, str2: string, maxDi if (typeof str1 !== 'string' || typeof str2 !== 'string') { throw new Error('Both arguments must be strings'); } - + if (maxDistance < 0 || !Number.isInteger(maxDistance)) { throw new Error('maxDistance must be a non-negative integer'); } @@ -103,7 +103,7 @@ export function calculateOptimizedEditDistance(str1: string, str2: string, maxDi currentRow[j - 1] + 1, // insertion previousRow[j - 1] + cost // substitution ); - + // Track minimum value in current row for early termination if (currentRow[j] < minInRow) { minInRow = currentRow[j]; @@ -125,7 +125,7 @@ export function calculateOptimizedEditDistance(str1: string, str2: string, maxDi /** * Validates that tokens meet minimum requirements for fuzzy operators. - * + * * @param tokens Array of search tokens * @param operator The search operator being used * @returns Validation result with success status and error message @@ -153,10 +153,10 @@ export function validateFuzzySearchTokens(tokens: string[], operator: string): { } // Check for null, undefined, or non-string tokens - const invalidTypeTokens = tokens.filter(token => + const invalidTypeTokens = tokens.filter(token => token == null || typeof token !== 'string' ); - + if (invalidTypeTokens.length > 0) { return { isValid: false, @@ -166,7 +166,7 @@ export function validateFuzzySearchTokens(tokens: string[], operator: string): { // Check for empty string tokens const emptyTokens = tokens.filter(token => token.trim().length === 0); - + if (emptyTokens.length > 0) { return { isValid: false, @@ -180,7 +180,7 @@ export function validateFuzzySearchTokens(tokens: string[], operator: string): { // Check minimum token length for fuzzy operators const shortTokens = tokens.filter(token => token.length < FUZZY_SEARCH_CONFIG.MIN_FUZZY_TOKEN_LENGTH); - + if (shortTokens.length > 0) { return { isValid: false, @@ -191,7 +191,7 @@ export function validateFuzzySearchTokens(tokens: string[], operator: string): { // Check for excessively long tokens that could cause performance issues const maxTokenLength = 100; // Reasonable limit for search tokens const longTokens = tokens.filter(token => token.length > maxTokenLength); - + if (longTokens.length > 0) { return { isValid: false, @@ -205,7 +205,7 @@ export function validateFuzzySearchTokens(tokens: string[], operator: string): { /** * Validates and preprocesses content for search operations. * Philosophy: Try to search everything! Only block truly extreme cases that could crash the system. - * + * * @param content The content to validate and preprocess * @param noteId The note ID (for logging purposes) * @returns Processed content, only null for truly extreme cases that could cause system instability @@ -258,9 +258,9 @@ function escapeRegExp(string: string): string { /** * Checks if a word matches a token with fuzzy matching and returns the matched word. * Optimized for common case where distances are small. - * + * * @param token The search token (should be normalized) - * @param text The text to match against (should be normalized) + * @param text The text to match against (should be normalized) * @param maxDistance Maximum allowed edit distance * @returns The matched word if found, null otherwise */ @@ -269,49 +269,49 @@ export function fuzzyMatchWordWithResult(token: string, text: string, maxDistanc if (typeof token !== 'string' || typeof text !== 'string') { return null; } - + if (token.length === 0 || text.length === 0) { return null; } - + try { // Normalize both strings for comparison const normalizedToken = token.toLowerCase(); const normalizedText = text.toLowerCase(); - + // Exact match check first (most common case) if (normalizedText.includes(normalizedToken)) { // Find the exact match in the original text to preserve case const exactMatch = text.match(new RegExp(escapeRegExp(token), 'i')); return exactMatch ? exactMatch[0] : token; } - + // For fuzzy matching, we need to check individual words in the text // Split the text into words and check each word against the token const words = normalizedText.split(/\s+/).filter(word => word.length > 0); const originalWords = text.split(/\s+/).filter(word => word.length > 0); - + for (let i = 0; i < words.length; i++) { const word = words[i]; const originalWord = originalWords[i]; - + // Skip if word is too different in length for fuzzy matching if (Math.abs(word.length - normalizedToken.length) > maxDistance) { continue; } - + // For very short tokens or very different lengths, be more strict if (normalizedToken.length < 4 || Math.abs(word.length - normalizedToken.length) > 2) { continue; } - + // Use optimized edit distance calculation const distance = calculateOptimizedEditDistance(normalizedToken, word, maxDistance); if (distance <= maxDistance) { return originalWord; // Return the original word with case preserved } } - + return null; } catch (error) { // Log error and return null for safety @@ -323,7 +323,7 @@ export function fuzzyMatchWordWithResult(token: string, text: string, maxDistanc /** * Checks if a word matches a token with fuzzy matching. * Optimized for common case where distances are small. - * + * * @param token The search token (should be normalized) * @param word The word to match against (should be normalized) * @param maxDistance Maximum allowed edit distance @@ -331,4 +331,4 @@ export function fuzzyMatchWordWithResult(token: string, text: string, maxDistanc */ export function fuzzyMatchWord(token: string, text: string, maxDistance: number = FUZZY_SEARCH_CONFIG.MAX_EDIT_DISTANCE): boolean { return fuzzyMatchWordWithResult(token, text, maxDistance) !== null; -} \ No newline at end of file +} diff --git a/apps/server/src/services/search/value_extractor.spec.ts b/packages/trilium-core/src/services/search/value_extractor.spec.ts similarity index 100% rename from apps/server/src/services/search/value_extractor.spec.ts rename to packages/trilium-core/src/services/search/value_extractor.spec.ts diff --git a/apps/server/src/services/search/value_extractor.ts b/packages/trilium-core/src/services/search/value_extractor.ts similarity index 100% rename from apps/server/src/services/search/value_extractor.ts rename to packages/trilium-core/src/services/search/value_extractor.ts diff --git a/packages/trilium-core/src/services/utils/index.ts b/packages/trilium-core/src/services/utils/index.ts index bba53a0b93..ba0ad517d1 100644 --- a/packages/trilium-core/src/services/utils/index.ts +++ b/packages/trilium-core/src/services/utils/index.ts @@ -135,3 +135,7 @@ export function isEmptyOrWhitespace(str: string | null | undefined) { if (!str) return true; return str.match(/^ *$/) !== null; } + +export function escapeRegExp(str: string) { + return str.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1"); +}