Dockery/Trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2026-09-05 22:19:03 +02:00
Code Issues Packages Projects Releases Activity

feat(core): integrate search with route

Browse Source
This commit is contained in:
Elian Doran
2026-03-22 14:03:48 +02:00
parent d7d28bcf58
commit 28a56ff7bf
48 changed files with 902 additions and 901 deletions
Download Patch File Download Diff File Expand all files Collapse all files

166
apps/server/src/routes/api/search.ts
View File

@@ -1,166 +0,0 @@
import { becca_service,ValidationError } from "@triliumnext/core";
import type { Request } from "express";
import becca from "../../becca/becca.js";
import attributeFormatter from "../../services/attribute_formatter.js";
import bulkActionService from "../../services/bulk_actions.js";
import cls from "../../services/cls.js";
import hoistedNoteService from "../../services/hoisted_note.js";
import SearchContext from "../../services/search/search_context.js";
import type SearchResult from "../../services/search/search_result.js";
import searchService, { EMPTY_RESULT, type SearchNoteResult } from "../../services/search/services/search.js";
function searchFromNote(req: Request<{ noteId: string }>): SearchNoteResult {
const note = becca.getNoteOrThrow(req.params.noteId);
if (!note) {
// this can be triggered from recent changes, and it's harmless to return an empty list rather than fail
return EMPTY_RESULT;
}
if (note.type !== "search") {
throw new ValidationError(`Note '${req.params.noteId}' is not a search note.`);
}
return searchService.searchFromNote(note);
}
function searchAndExecute(req: Request<{ noteId: string }>) {
const note = becca.getNoteOrThrow(req.params.noteId);
if (!note) {
// this can be triggered from recent changes, and it's harmless to return an empty list rather than fail
return [];
}
if (note.type !== "search") {
throw new ValidationError(`Note '${req.params.noteId}' is not a search note.`);
}
const { searchResultNoteIds } = searchService.searchFromNote(note);
bulkActionService.executeActionsFromNote(note, searchResultNoteIds);
}
function quickSearch(req: Request<{ searchString: string }>) {
const { searchString } = req.params;
const searchContext = new SearchContext({
fastSearch: false,
includeArchivedNotes: false,
includeHiddenNotes: true,
fuzzyAttributeSearch: true,
ignoreInternalAttributes: true,
ancestorNoteId: hoistedNoteService.isHoistedInHiddenSubtree() ? "root" : hoistedNoteService.getHoistedNoteId()
});
// Execute search with our context
const allSearchResults = searchService.findResultsWithQuery(searchString, searchContext);
const trimmed = allSearchResults.slice(0, 200);
// Extract snippets using highlightedTokens from our context
for (const result of trimmed) {
result.contentSnippet = searchService.extractContentSnippet(result.noteId, searchContext.highlightedTokens);
result.attributeSnippet = searchService.extractAttributeSnippet(result.noteId, searchContext.highlightedTokens);
}
// Highlight the results
searchService.highlightSearchResults(trimmed, searchContext.highlightedTokens, searchContext.ignoreInternalAttributes);
// Map to API format
const searchResults = trimmed.map((result) => {
const { title, icon } = becca_service.getNoteTitleAndIcon(result.noteId);
return {
notePath: result.notePath,
noteTitle: title,
notePathTitle: result.notePathTitle,
highlightedNotePathTitle: result.highlightedNotePathTitle,
contentSnippet: result.contentSnippet,
highlightedContentSnippet: result.highlightedContentSnippet,
attributeSnippet: result.attributeSnippet,
highlightedAttributeSnippet: result.highlightedAttributeSnippet,
icon
};
});
const resultNoteIds = searchResults.map((result) => result.notePath.split("/").pop()).filter(Boolean) as string[];
return {
searchResultNoteIds: resultNoteIds,
searchResults,
error: searchContext.getError()
};
}
function search(req: Request<{ searchString: string }>) {
const { searchString } = req.params;
const searchContext = new SearchContext({
fastSearch: false,
includeArchivedNotes: true,
fuzzyAttributeSearch: false,
ignoreHoistedNote: true
});
return searchService.findResultsWithQuery(searchString, searchContext).map((sr) => sr.noteId);
}
function getRelatedNotes(req: Request) {
const attr = req.body;
const searchSettings = {
fastSearch: true,
includeArchivedNotes: false,
fuzzyAttributeSearch: false
};
const matchingNameAndValue = searchService.findResultsWithQuery(attributeFormatter.formatAttrForSearch(attr, true), new SearchContext(searchSettings));
const matchingName = searchService.findResultsWithQuery(attributeFormatter.formatAttrForSearch(attr, false), new SearchContext(searchSettings));
const results: SearchResult[] = [];
const allResults = matchingNameAndValue.concat(matchingName);
const allResultNoteIds = new Set();
for (const record of allResults) {
allResultNoteIds.add(record.noteId);
}
for (const record of allResults) {
if (results.length >= 20) {
break;
}
if (results.find((res) => res.noteId === record.noteId)) {
continue;
}
results.push(record);
}
return {
count: allResultNoteIds.size,
results
};
}
function searchTemplates() {
const query = cls.getHoistedNoteId() === "root" ? "#template" : "#template OR #workspaceTemplate";
return searchService
.searchNotes(query, {
includeArchivedNotes: true,
ignoreHoistedNote: false
})
.map((note) => note.noteId);
}
export default {
searchFromNote,
searchAndExecute,
getRelatedNotes,
quickSearch,
search,
searchTemplates
};

7
apps/server/src/routes/routes.ts
View File

@@ -35,7 +35,6 @@ import otherRoute from "./api/other.js";
import passwordApiRoute from "./api/password.js";
import recoveryCodes from './api/recovery_codes.js';
import scriptRoute from "./api/script.js";
import searchRoute from "./api/search.js";
import senderRoute from "./api/sender.js";
import setupApiRoute from "./api/setup.js";
import similarNotesRoute from "./api/similar_notes.js";
@@ -171,12 +170,6 @@ function register(app: express.Application) {
apiRoute(GET, "/api/autocomplete", autocompleteApiRoute.getAutocomplete);
apiRoute(GET, "/api/autocomplete/notesCount", autocompleteApiRoute.getNotesCount);
apiRoute(GET, "/api/quick-search/:searchString", searchRoute.quickSearch);
apiRoute(GET, "/api/search-note/:noteId", searchRoute.searchFromNote);
apiRoute(PST, "/api/search-and-execute-note/:noteId", searchRoute.searchAndExecute);
apiRoute(PST, "/api/search-related", searchRoute.getRelatedNotes);
apiRoute(GET, "/api/search/:searchString", searchRoute.search);
apiRoute(GET, "/api/search-templates", searchRoute.searchTemplates);
route(PST, "/api/login/sync", [loginRateLimiter], loginApiRoute.loginSync, apiResultHandler);
// this is for entering protected mode so user has to be already logged-in (that's the reason we don't require username)

42
apps/server/src/services/hoisted_note.ts
View File

@@ -1,40 +1,2 @@
import cls from "./cls.js";
import becca from "../becca/becca.js";
function getHoistedNoteId() {
return cls.getHoistedNoteId();
}
function isHoistedInHiddenSubtree() {
const hoistedNoteId = getHoistedNoteId();
if (hoistedNoteId === "root") {
return false;
} else if (hoistedNoteId === "_hidden") {
return true;
}
const hoistedNote = becca.getNote(hoistedNoteId);
if (!hoistedNote) {
throw new Error(`Cannot find hoisted note '${hoistedNoteId}'`);
}
return hoistedNote.isHiddenCompletely();
}
function getWorkspaceNote() {
const hoistedNote = becca.getNote(cls.getHoistedNoteId());
if (hoistedNote && (hoistedNote.isRoot() || hoistedNote.hasLabel("workspace"))) {
return hoistedNote;
} else {
return becca.getRoot();
}
}
export default {
getHoistedNoteId,
getWorkspaceNote,
isHoistedInHiddenSubtree
};
import { hoisted_note } from "@triliumnext/core";
export default hoisted_note;

73
apps/server/src/services/search/expressions/ancestor.ts
View File

@@ -1,73 +0,0 @@
"use strict";
import Expression from "./expression.js";
import NoteSet from "../note_set.js";
import log from "../../log.js";
import becca from "../../../becca/becca.js";
import type SearchContext from "../search_context.js";
class AncestorExp extends Expression {
private ancestorNoteId: string;
private ancestorDepthComparator;
ancestorDepth?: string;
constructor(ancestorNoteId: string, ancestorDepth?: string) {
super();
this.ancestorNoteId = ancestorNoteId;
this.ancestorDepth = ancestorDepth; // for DEBUG mode
this.ancestorDepthComparator = this.getComparator(ancestorDepth);
}
execute(inputNoteSet: NoteSet, executionContext: {}, searchContext: SearchContext) {
const ancestorNote = becca.notes[this.ancestorNoteId];
if (!ancestorNote) {
log.error(`Subtree note '${this.ancestorNoteId}' was not not found.`);
return new NoteSet([]);
}
const subtree = ancestorNote.getSubtree();
const subTreeNoteSet = new NoteSet(subtree.notes).intersection(inputNoteSet);
if (!this.ancestorDepthComparator) {
return subTreeNoteSet;
}
const depthConformingNoteSet = new NoteSet([]);
for (const note of subTreeNoteSet.notes) {
const distance = note.getDistanceToAncestor(ancestorNote.noteId);
if (this.ancestorDepthComparator(distance)) {
depthConformingNoteSet.add(note);
}
}
return depthConformingNoteSet;
}
getComparator(depthCondition?: string): ((depth: number) => boolean) | null {
if (!depthCondition) {
return null;
}
const comparedDepth = parseInt(depthCondition.substr(2));
if (depthCondition.startsWith("eq")) {
return (depth) => depth === comparedDepth;
} else if (depthCondition.startsWith("gt")) {
return (depth) => depth > comparedDepth;
} else if (depthCondition.startsWith("lt")) {
return (depth) => depth < comparedDepth;
} else {
log.error(`Unrecognized depth condition value ${depthCondition}`);
return null;
}
}
}
export default AncestorExp;

37
apps/server/src/services/search/expressions/and.ts
View File

@@ -1,37 +0,0 @@
"use strict";
import type NoteSet from "../note_set.js";
import type SearchContext from "../search_context.js";
import Expression from "./expression.js";
import TrueExp from "./true.js";
class AndExp extends Expression {
subExpressions: Expression[];
static of(_subExpressions: (Expression | null | undefined)[]) {
const subExpressions = _subExpressions.filter((exp) => !!exp) as Expression[];
if (subExpressions.length === 1) {
return subExpressions[0];
} else if (subExpressions.length > 0) {
return new AndExp(subExpressions);
} else {
return new TrueExp();
}
}
constructor(subExpressions: Expression[]) {
super();
this.subExpressions = subExpressions;
}
execute(inputNoteSet: NoteSet, executionContext: {}, searchContext: SearchContext) {
for (const subExpression of this.subExpressions) {
inputNoteSet = subExpression.execute(inputNoteSet, executionContext, searchContext);
}
return inputNoteSet;
}
}
export default AndExp;

46
apps/server/src/services/search/expressions/attribute_exists.ts
View File

@@ -1,46 +0,0 @@
"use strict";
import NoteSet from "../note_set.js";
import type SearchContext from "../search_context.js";
import becca from "../../../becca/becca.js";
import Expression from "./expression.js";
class AttributeExistsExp extends Expression {
attributeType: string;
attributeName: string;
private isTemplateLabel: boolean;
private prefixMatch: boolean;
constructor(attributeType: string, attributeName: string, prefixMatch: boolean) {
super();
this.attributeType = attributeType;
this.attributeName = attributeName;
// template attr is used as a marker for templates, but it's not meant to be inherited
this.isTemplateLabel = this.attributeType === "label" && (this.attributeName === "template" || this.attributeName === "workspacetemplate");
this.prefixMatch = prefixMatch;
}
execute(inputNoteSet: NoteSet, executionContext: {}, searchContext: SearchContext) {
const attrs = this.prefixMatch ? becca.findAttributesWithPrefix(this.attributeType, this.attributeName) : becca.findAttributes(this.attributeType, this.attributeName);
const resultNoteSet = new NoteSet();
for (const attr of attrs) {
const note = attr.note;
if (attr.isInheritable && !this.isTemplateLabel) {
resultNoteSet.addAll(note.getSubtreeNotesIncludingTemplated());
} else if (note.isInherited() && !this.isTemplateLabel) {
resultNoteSet.addAll(note.getInheritingNotes());
} else {
resultNoteSet.add(note);
}
}
return resultNoteSet.intersection(inputNoteSet);
}
}
export default AttributeExistsExp;

39
apps/server/src/services/search/expressions/child_of.ts
View File

@@ -1,39 +0,0 @@
"use strict";
import Expression from "./expression.js";
import NoteSet from "../note_set.js";
import type SearchContext from "../search_context.js";
class ChildOfExp extends Expression {
private subExpression: Expression;
constructor(subExpression: Expression) {
super();
this.subExpression = subExpression;
}
execute(inputNoteSet: NoteSet, executionContext: {}, searchContext: SearchContext) {
const subInputNoteSet = new NoteSet();
for (const note of inputNoteSet.notes) {
subInputNoteSet.addAll(note.parents);
}
const subResNoteSet = this.subExpression.execute(subInputNoteSet, executionContext, searchContext);
const resNoteSet = new NoteSet();
for (const parentNote of subResNoteSet.notes) {
for (const childNote of parentNote.children) {
if (inputNoteSet.hasNote(childNote)) {
resNoteSet.add(childNote);
}
}
}
return resNoteSet;
}
}
export default ChildOfExp;

31
apps/server/src/services/search/expressions/descendant_of.ts
View File

@@ -1,31 +0,0 @@
"use strict";
import Expression from "./expression.js";
import NoteSet from "../note_set.js";
import becca from "../../../becca/becca.js";
import type SearchContext from "../search_context.js";
class DescendantOfExp extends Expression {
private subExpression: Expression;
constructor(subExpression: Expression) {
super();
this.subExpression = subExpression;
}
execute(inputNoteSet: NoteSet, executionContext: {}, searchContext: SearchContext) {
const subInputNoteSet = new NoteSet(Object.values(becca.notes));
const subResNoteSet = this.subExpression.execute(subInputNoteSet, executionContext, searchContext);
const subTreeNoteSet = new NoteSet();
for (const note of subResNoteSet.notes) {
subTreeNoteSet.addAll(note.getSubtree().notes);
}
return inputNoteSet.intersection(subTreeNoteSet);
}
}
export default DescendantOfExp;

14
apps/server/src/services/search/expressions/expression.ts
View File

@@ -1,14 +0,0 @@
"use strict";
import type NoteSet from "../note_set.js";
import type SearchContext from "../search_context.js";
export default abstract class Expression {
name: string;
constructor() {
this.name = this.constructor.name; // for DEBUG mode to have expression name as part of dumped JSON
}
abstract execute(inputNoteSet: NoteSet, executionContext: {}, searchContext: SearchContext): NoteSet;
}

24
apps/server/src/services/search/expressions/is_hidden.ts
View File

@@ -1,24 +0,0 @@
"use strict";
import Expression from "./expression.js";
import NoteSet from "../note_set.js";
import type SearchContext from "../search_context.js";
/**
* Note is hidden when all its note paths start in hidden subtree (i.e., the note is not cloned into visible tree)
*/
class IsHiddenExp extends Expression {
execute(inputNoteSet: NoteSet, executionContext: {}, searchContext: SearchContext) {
const resultNoteSet = new NoteSet();
for (const note of inputNoteSet.notes) {
if (note.isHiddenCompletely()) {
resultNoteSet.add(note);
}
}
return resultNoteSet;
}
}
export default IsHiddenExp;

46
apps/server/src/services/search/expressions/label_comparison.ts
View File

@@ -1,46 +0,0 @@
"use strict";
import Expression from "./expression.js";
import NoteSet from "../note_set.js";
import becca from "../../../becca/becca.js";
import type SearchContext from "../search_context.js";
type Comparator = (value: string) => boolean;
class LabelComparisonExp extends Expression {
attributeType: string;
attributeName: string;
comparator: Comparator;
constructor(attributeType: string, attributeName: string, comparator: Comparator) {
super();
this.attributeType = attributeType;
this.attributeName = attributeName.toLowerCase();
this.comparator = comparator;
}
execute(inputNoteSet: NoteSet, executionContext: {}, searchContext: SearchContext) {
const attrs = becca.findAttributes(this.attributeType, this.attributeName);
const resultNoteSet = new NoteSet();
for (const attr of attrs) {
const note = attr.note;
const value = attr.value?.toLowerCase();
if (inputNoteSet.hasNoteId(note.noteId) && this.comparator(value)) {
if (attr.isInheritable) {
resultNoteSet.addAll(note.getSubtreeNotesIncludingTemplated());
} else if (note.isInherited()) {
resultNoteSet.addAll(note.getInheritingNotes());
} else {
resultNoteSet.add(note);
}
}
}
return resultNoteSet;
}
}
export default LabelComparisonExp;

23
apps/server/src/services/search/expressions/not.ts
View File

@@ -1,23 +0,0 @@
"use strict";
import type NoteSet from "../note_set.js";
import type SearchContext from "../search_context.js";
import Expression from "./expression.js";
class NotExp extends Expression {
subExpression: Expression;
constructor(subExpression: Expression) {
super();
this.subExpression = subExpression;
}
execute(inputNoteSet: NoteSet, executionContext: {}, searchContext: SearchContext) {
const subNoteSet = this.subExpression.execute(inputNoteSet, executionContext, searchContext);
return inputNoteSet.minus(subNoteSet);
}
}
export default NotExp;

19
apps/server/src/services/search/expressions/note_content_fulltext.spec.ts
View File

@@ -1,19 +0,0 @@
import { describe, expect,it } from "vitest";
import NoteContentFulltextExp from "./note_content_fulltext.js";
describe("Fuzzy Search Operators", () => {
it("~= operator works with typos", () => {
// Test that the ~= operator can handle common typos
const expression = new NoteContentFulltextExp("~=", { tokens: ["hello"] });
expect(expression.tokens).toEqual(["hello"]);
expect(() => new NoteContentFulltextExp("~=", { tokens: ["he"] })).toThrow(); // Too short
});
it("~* operator works with fuzzy contains", () => {
// Test that the ~* operator handles fuzzy substring matching
const expression = new NoteContentFulltextExp("~*", { tokens: ["world"] });
expect(expression.tokens).toEqual(["world"]);
expect(() => new NoteContentFulltextExp("~*", { tokens: ["wo"] })).toThrow(); // Too short
});
});

434
apps/server/src/services/search/expressions/note_content_fulltext.ts
View File

@@ -1,434 +0,0 @@
import type { NoteRow } from "@triliumnext/commons";
import becca from "../../../becca/becca.js";
import log from "../../log.js";
import protectedSessionService from "../../protected_session.js";
import sql from "../../sql.js";
import NoteSet from "../note_set.js";
import type SearchContext from "../search_context.js";
import {
FUZZY_SEARCH_CONFIG,
fuzzyMatchWord,
normalizeSearchText,
validateAndPreprocessContent,
validateFuzzySearchTokens} from "../utils/text_utils.js";
import Expression from "./expression.js";
import preprocessContent from "./note_content_fulltext_preprocessor.js";
const ALLOWED_OPERATORS = new Set(["=", "!=", "*=*", "*=", "=*", "%=", "~=", "~*"]);
// Maximum content size for search processing (2MB)
const MAX_SEARCH_CONTENT_SIZE = 2 * 1024 * 1024;
const cachedRegexes: Record<string, RegExp> = {};
function getRegex(str: string): RegExp {
if (!(str in cachedRegexes)) {
cachedRegexes[str] = new RegExp(str, "ms"); // multiline, dot-all
}
return cachedRegexes[str];
}
interface ConstructorOpts {
tokens: string[];
raw?: boolean;
flatText?: boolean;
}
type SearchRow = Pick<NoteRow, "noteId" | "type" | "mime" | "content" | "isProtected">;
class NoteContentFulltextExp extends Expression {
private operator: string;
tokens: string[];
private raw: boolean;
private flatText: boolean;
constructor(operator: string, { tokens, raw, flatText }: ConstructorOpts) {
super();
if (!operator || !tokens || !Array.isArray(tokens)) {
throw new Error('Invalid parameters: operator and tokens are required');
}
// Validate fuzzy search tokens
const validation = validateFuzzySearchTokens(tokens, operator);
if (!validation.isValid) {
throw new Error(validation.error!);
}
this.operator = operator;
this.tokens = tokens;
this.raw = !!raw;
this.flatText = !!flatText;
}
execute(inputNoteSet: NoteSet, executionContext: {}, searchContext: SearchContext) {
if (!ALLOWED_OPERATORS.has(this.operator)) {
searchContext.addError(`Note content can be searched only with operators: ${Array.from(ALLOWED_OPERATORS).join(", ")}, operator ${this.operator} given.`);
return inputNoteSet;
}
// Add tokens to highlightedTokens so snippet extraction knows what to look for
for (const token of this.tokens) {
if (!searchContext.highlightedTokens.includes(token)) {
searchContext.highlightedTokens.push(token);
}
}
const resultNoteSet = new NoteSet();
// Search through notes with content
for (const row of sql.iterateRows<SearchRow>(`
SELECT noteId, type, mime, content, isProtected
FROM notes JOIN blobs USING (blobId)
WHERE type IN ('text', 'code', 'mermaid', 'canvas', 'mindMap')
AND isDeleted = 0
AND LENGTH(content) < ${MAX_SEARCH_CONTENT_SIZE}`)) {
this.findInText(row, inputNoteSet, resultNoteSet);
}
// For exact match with flatText, also search notes WITHOUT content (they may have matching attributes)
if (this.flatText && (this.operator === "=" || this.operator === "!=")) {
for (const note of inputNoteSet.notes) {
// Skip if already found or doesn't exist
if (resultNoteSet.hasNoteId(note.noteId) || !(note.noteId in becca.notes)) {
continue;
}
const noteFromBecca = becca.notes[note.noteId];
const flatText = noteFromBecca.getFlatText();
// For flatText, only check attribute values (format: #name=value or ~name=value)
// Don't match against noteId, type, mime, or title which are also in flatText
let matches = false;
const phrase = this.tokens.join(" ");
const normalizedPhrase = normalizeSearchText(phrase);
const normalizedFlatText = normalizeSearchText(flatText);
// Check if =phrase appears in flatText (indicates attribute value match)
// For single words, use word-boundary matching to avoid substring matches
if (!normalizedPhrase.includes(' ')) {
// Single word: look for =word with word boundaries
// Split by = to get attribute values, then check each value for exact word match
const parts = normalizedFlatText.split('=');
matches = parts.slice(1).some(part => this.exactWordMatch(normalizedPhrase, part));
} else {
// Multi-word phrase: check for substring match
matches = normalizedFlatText.includes(`=${normalizedPhrase}`);
}
if ((this.operator === "=" && matches) || (this.operator === "!=" && !matches)) {
resultNoteSet.add(noteFromBecca);
}
}
}
return resultNoteSet;
}
/**
* Helper method to check if a single word appears as an exact match in text
* @param wordToFind - The word to search for (should be normalized)
* @param text - The text to search in (should be normalized)
* @returns true if the word is found as an exact match (not substring)
*/
private exactWordMatch(wordToFind: string, text: string): boolean {
const words = text.split(/\s+/);
return words.some(word => word === wordToFind);
}
/**
* Checks if content contains the exact word (with word boundaries) or exact phrase
* This is case-insensitive since content and token are already normalized
*/
private containsExactWord(token: string, content: string): boolean {
// Normalize both for case-insensitive comparison
const normalizedToken = normalizeSearchText(token);
const normalizedContent = normalizeSearchText(content);
// If token contains spaces, it's a multi-word phrase from quotes
// Check for substring match (consecutive phrase)
if (normalizedToken.includes(' ')) {
return normalizedContent.includes(normalizedToken);
}
// For single words, use exact word matching to avoid substring matches
return this.exactWordMatch(normalizedToken, normalizedContent);
}
/**
* Checks if content contains the exact phrase (consecutive words in order)
* This is case-insensitive since content and tokens are already normalized
*/
private containsExactPhrase(tokens: string[], content: string, checkFlatTextAttributes: boolean = false): boolean {
const normalizedTokens = tokens.map(t => normalizeSearchText(t));
const normalizedContent = normalizeSearchText(content);
// Join tokens with single space to form the phrase
const phrase = normalizedTokens.join(" ");
// For single-word phrases, use word-boundary matching to avoid substring matches
// e.g., "asd" should not match "asdfasdf"
if (!phrase.includes(' ')) {
// Single word: use exact word matching to avoid substring matches
return this.exactWordMatch(phrase, normalizedContent);
}
// For multi-word phrases, check if the phrase appears as consecutive words
if (normalizedContent.includes(phrase)) {
return true;
}
// For flatText, also check if the phrase appears in attribute values
// Attributes in flatText appear as "#name=value" or "~name=value"
// So we need to check for "=phrase" to match attribute values
if (checkFlatTextAttributes && normalizedContent.includes(`=${phrase}`)) {
return true;
}
return false;
}
findInText({ noteId, isProtected, content, type, mime }: SearchRow, inputNoteSet: NoteSet, resultNoteSet: NoteSet) {
if (!inputNoteSet.hasNoteId(noteId) || !(noteId in becca.notes)) {
return;
}
if (isProtected) {
if (!protectedSessionService.isProtectedSessionAvailable() || !content || typeof content !== "string") {
return;
}
try {
content = protectedSessionService.decryptString(content) || undefined;
} catch (e) {
log.info(`Cannot decrypt content of note ${noteId}`);
return;
}
}
if (!content) {
return;
}
content = preprocessContent(content, type, mime, this.raw);
// Apply content size validation and preprocessing
const processedContent = validateAndPreprocessContent(content, noteId);
if (!processedContent) {
return; // Content too large or invalid
}
content = processedContent;
if (this.tokens.length === 1) {
const [token] = this.tokens;
let matches = false;
if (this.operator === "=") {
matches = this.containsExactWord(token, content);
// Also check flatText if enabled (includes attributes)
if (!matches && this.flatText) {
const flatText = becca.notes[noteId].getFlatText();
matches = this.containsExactPhrase([token], flatText, true);
}
} else if (this.operator === "!=") {
matches = !this.containsExactWord(token, content);
// For negation, check flatText too
if (matches && this.flatText) {
const flatText = becca.notes[noteId].getFlatText();
matches = !this.containsExactPhrase([token], flatText, true);
}
}
if (
matches ||
(this.operator === "*=" && content.endsWith(token)) ||
(this.operator === "=*" && content.startsWith(token)) ||
(this.operator === "*=*" && content.includes(token)) ||
(this.operator === "%=" && getRegex(token).test(content)) ||
(this.operator === "~=" && this.matchesWithFuzzy(content, noteId)) ||
(this.operator === "~*" && this.fuzzyMatchToken(normalizeSearchText(token), normalizeSearchText(content)))
) {
resultNoteSet.add(becca.notes[noteId]);
}
} else {
// Multi-token matching with fuzzy support and phrase proximity
if (this.operator === "~=" || this.operator === "~*") {
// Fuzzy phrase matching
if (this.matchesWithFuzzy(content, noteId)) {
resultNoteSet.add(becca.notes[noteId]);
}
} else if (this.operator === "=" || this.operator === "!=") {
// Exact phrase matching for = and !=
let matches = this.containsExactPhrase(this.tokens, content, false);
// Also check flatText if enabled (includes attributes)
if (!matches && this.flatText) {
const flatText = becca.notes[noteId].getFlatText();
matches = this.containsExactPhrase(this.tokens, flatText, true);
}
if ((this.operator === "=" && matches) ||
(this.operator === "!=" && !matches)) {
resultNoteSet.add(becca.notes[noteId]);
}
} else {
// Other operators: check all tokens present (any order)
const nonMatchingToken = this.tokens.find(
(token) =>
!this.tokenMatchesContent(token, content, noteId)
);
if (!nonMatchingToken) {
resultNoteSet.add(becca.notes[noteId]);
}
}
}
return content;
}
/**
* Checks if a token matches content with optional fuzzy matching
*/
private tokenMatchesContent(token: string, content: string, noteId: string): boolean {
const normalizedToken = normalizeSearchText(token);
const normalizedContent = normalizeSearchText(content);
if (normalizedContent.includes(normalizedToken)) {
return true;
}
// Check flat text for default fulltext search
if (!this.flatText || !becca.notes[noteId].getFlatText().includes(token)) {
return false;
}
return true;
}
/**
* Performs fuzzy matching with edit distance and phrase proximity
*/
private matchesWithFuzzy(content: string, noteId: string): boolean {
try {
const normalizedContent = normalizeSearchText(content);
const flatText = this.flatText ? normalizeSearchText(becca.notes[noteId].getFlatText()) : "";
// For phrase matching, check if tokens appear within reasonable proximity
if (this.tokens.length > 1) {
return this.matchesPhrase(normalizedContent, flatText);
}
// Single token fuzzy matching
const token = normalizeSearchText(this.tokens[0]);
return this.fuzzyMatchToken(token, normalizedContent) ||
(this.flatText && this.fuzzyMatchToken(token, flatText));
} catch (error) {
log.error(`Error in fuzzy matching for note ${noteId}: ${error}`);
return false;
}
}
/**
* Checks if multiple tokens match as a phrase with proximity consideration
*/
private matchesPhrase(content: string, flatText: string): boolean {
const searchText = this.flatText ? `${content} ${flatText}` : content;
// Apply content size limits for phrase matching
const limitedText = validateAndPreprocessContent(searchText);
if (!limitedText) {
return false;
}
const words = limitedText.toLowerCase().split(/\s+/);
// Only skip phrase matching for truly extreme word counts that could crash the system
if (words.length > FUZZY_SEARCH_CONFIG.ABSOLUTE_MAX_WORD_COUNT) {
console.error(`Phrase matching skipped due to extreme word count that could cause system instability: ${words.length} words`);
return false;
}
// Warn about large word counts but still attempt matching
if (words.length > FUZZY_SEARCH_CONFIG.PERFORMANCE_WARNING_WORDS) {
console.info(`Large word count for phrase matching: ${words.length} words - may take longer but will attempt full matching`);
}
// Find positions of each token
const tokenPositions: number[][] = this.tokens.map(token => {
const normalizedToken = normalizeSearchText(token);
const positions: number[] = [];
words.forEach((word, index) => {
if (this.fuzzyMatchSingle(normalizedToken, word)) {
positions.push(index);
}
});
return positions;
});
// Check if we found all tokens
if (tokenPositions.some(positions => positions.length === 0)) {
return false;
}
// Check for phrase proximity using configurable distance
return this.hasProximityMatch(tokenPositions, FUZZY_SEARCH_CONFIG.MAX_PHRASE_PROXIMITY);
}
/**
* Checks if token positions indicate a phrase match within max distance
*/
private hasProximityMatch(tokenPositions: number[][], maxDistance: number): boolean {
// For 2 tokens, simple proximity check
if (tokenPositions.length === 2) {
const [pos1, pos2] = tokenPositions;
return pos1.some(p1 => pos2.some(p2 => Math.abs(p1 - p2) <= maxDistance));
}
// For more tokens, check if we can find a sequence where all tokens are within range
const findSequence = (remaining: number[][], currentPos: number): boolean => {
if (remaining.length === 0) return true;
const [nextPositions, ...rest] = remaining;
return nextPositions.some(pos =>
Math.abs(pos - currentPos) <= maxDistance &&
findSequence(rest, pos)
);
};
const [firstPositions, ...rest] = tokenPositions;
return firstPositions.some(startPos => findSequence(rest, startPos));
}
/**
* Performs fuzzy matching for a single token against content
*/
private fuzzyMatchToken(token: string, content: string): boolean {
if (token.length < FUZZY_SEARCH_CONFIG.MIN_FUZZY_TOKEN_LENGTH) {
// For short tokens, require exact match to avoid too many false positives
return content.includes(token);
}
const words = content.split(/\s+/);
// Only limit word processing for truly extreme cases to prevent system instability
const limitedWords = words.slice(0, FUZZY_SEARCH_CONFIG.ABSOLUTE_MAX_WORD_COUNT);
return limitedWords.some(word => this.fuzzyMatchSingle(token, word));
}
/**
* Fuzzy matches a single token against a single word
*/
private fuzzyMatchSingle(token: string, word: string): boolean {
// Use shared optimized fuzzy matching logic
return fuzzyMatchWord(token, word, FUZZY_SEARCH_CONFIG.MAX_EDIT_DISTANCE);
}
}
export default NoteContentFulltextExp;

40
apps/server/src/services/search/expressions/note_content_fulltext_preprocessor.spec.ts
View File

@@ -1,40 +0,0 @@
import { NoteType } from "@triliumnext/commons";
import { describe, expect,it } from "vitest";
import preprocessContent from "./note_content_fulltext_preprocessor";
describe("Mind map preprocessing", () => {
const type: NoteType = "mindMap";
const mime = "application/json";
it("supports empty JSON", () => {
expect(preprocessContent("{}", type, mime)).toEqual("");
});
it("supports blank text / invalid JSON", () => {
expect(preprocessContent("", type, mime)).toEqual("");
expect(preprocessContent(`{ "node": " }`, type, mime)).toEqual("");
});
it("reads data", () => {
expect(preprocessContent(`{ "nodedata": { "topic": "Root", "children": [ { "topic": "Child 1" }, { "topic": "Child 2", "children": [ { "topic": "Grandchild" } ] } ] } }`, type, mime)).toEqual("root, child 1, child 2, grandchild");
});
});
describe("Canvas preprocessing", () => {
const type: NoteType = "canvas";
const mime = "application/json";
it("supports empty JSON", () => {
expect(preprocessContent("{}", type, mime)).toEqual("");
});
it("supports blank text / invalid JSON", () => {
expect(preprocessContent("", type, mime)).toEqual("");
});
it("reads elements", () => {
expect(preprocessContent(`{ "elements": [ { "type": "text", "text": "Hello" } ] }`, type, mime)).toEqual("hello");
expect(preprocessContent(`{ "elements": [ { "type": "text" }, { "type": "text", "text": "World" }, { "type": "rectangle", "text": "Ignored" } ] }`, type, mime)).toEqual("world");
});
});

126
apps/server/src/services/search/expressions/note_content_fulltext_preprocessor.ts
View File

@@ -1,126 +0,0 @@
import striptags from "striptags";
import { normalize } from "../../utils.js";
export default function preprocessContent(rawContent: string | Uint8Array, type: string, mime: string, raw?: boolean) {
let content = normalize(rawContent.toString());
if (type === "text" && mime === "text/html") {
if (!raw) {
// Content size already filtered at DB level, safe to process
content = stripTags(content);
}
content = content.replace(/&nbsp;/g, " ");
} else if (type === "mindMap" && mime === "application/json") {
content = processMindmapContent(content);
} else if (type === "canvas" && mime === "application/json") {
content = processCanvasContent(content);
}
return content.trim();
}
function processMindmapContent(content: string) {
let mindMapcontent;
try {
mindMapcontent = JSON.parse(content);
} catch (e) {
return "";
}
// Define interfaces for the JSON structure
interface MindmapNode {
id: string;
topic: string;
children: MindmapNode[]; // Recursive structure
direction?: number;
expanded?: boolean;
}
interface MindmapData {
nodedata: MindmapNode;
arrows: any[]; // If you know the structure, replace `any` with the correct type
summaries: any[];
direction: number;
theme: {
name: string;
type: string;
palette: string[];
cssvar: Record<string, string>; // Object with string keys and string values
};
}
// Recursive function to collect all topics
function collectTopics(node?: MindmapNode): string[] {
if (!node) {
return [];
}
// Collect the current node's topic
let topics = [node.topic];
// If the node has children, collect topics recursively
if (node.children && node.children.length > 0) {
for (const child of node.children) {
topics = topics.concat(collectTopics(child));
}
}
return topics;
}
// Start extracting from the root node
const topicsArray = collectTopics(mindMapcontent.nodedata);
// Combine topics into a single string
const topicsString = topicsArray.join(", ");
return normalize(topicsString.toString());
}
function processCanvasContent(content: string) {
interface Element {
type: string;
text?: string; // Optional since not all objects have a `text` property
id: string;
[key: string]: any; // Other properties that may exist
}
let canvasContent;
try {
canvasContent = JSON.parse(content);
} catch (e) {
return "";
}
const elements = canvasContent.elements;
if (Array.isArray(elements)) {
const texts = elements
.filter((element: Element) => element.type === "text" && element.text) // Filter for 'text' type elements with a 'text' property
.map((element: Element) => element.text!); // Use `!` to assert `text` is defined after filtering
content = normalize(texts.join(" "));
} else {
content = "";
}
return content;
}
function stripTags(content: string) {
// we want to allow link to preserve URLs: https://github.com/zadam/trilium/issues/2412
// we want to insert space in place of block tags (because they imply text separation)
// but we don't want to insert text for typical formatting inline tags which can occur within one word
const linkTag = "a";
const inlineFormattingTags = ["b", "strong", "em", "i", "span", "big", "small", "font", "sub", "sup"];
// replace tags which imply text separation with a space
content = striptags(content, [linkTag, ...inlineFormattingTags], " ");
// replace the inline formatting tags (but not links) without a space
content = striptags(content, [linkTag], "");
// at least the closing link tag can be easily stripped
return content.replace(/<\/a>/gi, "");
}

208
apps/server/src/services/search/expressions/note_flat_text.ts
View File

@@ -1,208 +0,0 @@
import { becca_service } from "@triliumnext/core";
import becca from "../../../becca/becca.js";
import type BNote from "../../../becca/entities/bnote.js";
import { normalize } from "../../utils.js";
import NoteSet from "../note_set.js";
import type SearchContext from "../search_context.js";
import { fuzzyMatchWord, fuzzyMatchWordWithResult,normalizeSearchText } from "../utils/text_utils.js";
import Expression from "./expression.js";
class NoteFlatTextExp extends Expression {
tokens: string[];
constructor(tokens: string[]) {
super();
// Normalize tokens using centralized normalization function
this.tokens = tokens.map(token => normalizeSearchText(token));
}
execute(inputNoteSet: NoteSet, executionContext: any, searchContext: SearchContext) {
const resultNoteSet = new NoteSet();
/**
* @param note
* @param remainingTokens - tokens still needed to be found in the path towards root
* @param takenPath - path so far taken towards from candidate note towards the root.
* It contains the suffix fragment of the full note path.
*/
const searchPathTowardsRoot = (note: BNote, remainingTokens: string[], takenPath: string[]) => {
if (remainingTokens.length === 0) {
// we're done, just build the result
const resultPath = this.getNotePath(note, takenPath);
if (resultPath) {
const noteId = resultPath[resultPath.length - 1];
if (!resultNoteSet.hasNoteId(noteId)) {
// we could get here from multiple paths, the first one wins because the paths
// are sorted by importance
executionContext.noteIdToNotePath[noteId] = resultPath;
resultNoteSet.add(becca.notes[noteId]);
}
}
return;
}
if (note.parents.length === 0 || note.noteId === "root") {
// we've reached root, but there are still remaining tokens -> this candidate note produced no result
return;
}
const foundAttrTokens: string[] = [];
for (const token of remainingTokens) {
// Add defensive checks for undefined properties
const typeMatches = note.type && note.type.includes(token);
const mimeMatches = note.mime && note.mime.includes(token);
if (typeMatches || mimeMatches) {
foundAttrTokens.push(token);
}
}
for (const attribute of note.getOwnedAttributes()) {
const normalizedName = normalizeSearchText(attribute.name);
const normalizedValue = normalizeSearchText(attribute.value);
for (const token of remainingTokens) {
if (normalizedName.includes(token) || normalizedValue.includes(token)) {
foundAttrTokens.push(token);
}
}
}
for (const parentNote of note.parents) {
const title = normalizeSearchText(becca_service.getNoteTitle(note.noteId, parentNote.noteId));
const foundTokens: string[] = foundAttrTokens.slice();
for (const token of remainingTokens) {
if (this.smartMatch(title, token, searchContext)) {
foundTokens.push(token);
}
}
if (foundTokens.length > 0) {
const newRemainingTokens = remainingTokens.filter((token) => !foundTokens.includes(token));
searchPathTowardsRoot(parentNote, newRemainingTokens, [note.noteId, ...takenPath]);
} else {
searchPathTowardsRoot(parentNote, remainingTokens, [note.noteId, ...takenPath]);
}
}
};
const candidateNotes = this.getCandidateNotes(inputNoteSet, searchContext);
for (const note of candidateNotes) {
// autocomplete should be able to find notes by their noteIds as well (only leafs)
if (this.tokens.length === 1 && note.noteId.toLowerCase() === this.tokens[0]) {
searchPathTowardsRoot(note, [], [note.noteId]);
continue;
}
const foundAttrTokens: string[] = [];
for (const token of this.tokens) {
// Add defensive checks for undefined properties
const typeMatches = note.type && note.type.includes(token);
const mimeMatches = note.mime && note.mime.includes(token);
if (typeMatches || mimeMatches) {
foundAttrTokens.push(token);
}
for (const attribute of note.ownedAttributes) {
if (normalizeSearchText(attribute.name).includes(token) || normalizeSearchText(attribute.value).includes(token)) {
foundAttrTokens.push(token);
}
}
}
for (const parentNote of note.parents) {
const title = normalizeSearchText(becca_service.getNoteTitle(note.noteId, parentNote.noteId));
const foundTokens = foundAttrTokens.slice();
for (const token of this.tokens) {
if (this.smartMatch(title, token, searchContext)) {
foundTokens.push(token);
}
}
if (foundTokens.length > 0) {
const remainingTokens = this.tokens.filter((token) => !foundTokens.includes(token));
searchPathTowardsRoot(parentNote, remainingTokens, [note.noteId]);
}
}
}
return resultNoteSet;
}
getNotePath(note: BNote, takenPath: string[]): string[] {
if (takenPath.length === 0) {
throw new Error("Path is not expected to be empty.");
} else if (takenPath.length === 1 && takenPath[0] === note.noteId) {
return note.getBestNotePath();
} else {
// this note is the closest to root containing the last matching token(s), thus completing the requirements
// what's in this note's predecessors does not matter, thus we'll choose the best note path
const topMostMatchingTokenNotePath = becca.getNote(takenPath[0])?.getBestNotePath() || [];
return [...topMostMatchingTokenNotePath, ...takenPath.slice(1)];
}
}
/**
* Returns noteIds which have at least one matching tokens
*/
getCandidateNotes(noteSet: NoteSet, searchContext?: SearchContext): BNote[] {
const candidateNotes: BNote[] = [];
for (const note of noteSet.notes) {
const normalizedFlatText = normalizeSearchText(note.getFlatText());
for (const token of this.tokens) {
if (this.smartMatch(normalizedFlatText, token, searchContext)) {
candidateNotes.push(note);
break;
}
}
}
return candidateNotes;
}
/**
* Smart matching that tries exact match first, then fuzzy fallback
* @param text The text to search in
* @param token The token to search for
* @param searchContext The search context to track matched words for highlighting
* @returns True if match found (exact or fuzzy)
*/
private smartMatch(text: string, token: string, searchContext?: SearchContext): boolean {
// Exact match has priority
if (text.includes(token)) {
return true;
}
// Fuzzy fallback only if enabled and for tokens >= 4 characters
if (searchContext?.enableFuzzyMatching && token.length >= 4) {
const matchedWord = fuzzyMatchWordWithResult(token, text);
if (matchedWord) {
// Track the fuzzy matched word for highlighting
if (!searchContext.highlightedTokens.includes(matchedWord)) {
searchContext.highlightedTokens.push(matchedWord);
}
return true;
}
}
return false;
}
}
export default NoteFlatTextExp;

40
apps/server/src/services/search/expressions/or.ts
View File

@@ -1,40 +0,0 @@
"use strict";
import Expression from "./expression.js";
import NoteSet from "../note_set.js";
import TrueExp from "./true.js";
import type SearchContext from "../search_context.js";
class OrExp extends Expression {
subExpressions: Expression[];
static of(subExpressions: Expression[]) {
subExpressions = subExpressions.filter((exp) => !!exp);
if (subExpressions.length === 1) {
return subExpressions[0];
} else if (subExpressions.length > 0) {
return new OrExp(subExpressions);
} else {
return new TrueExp();
}
}
constructor(subExpressions: Expression[]) {
super();
this.subExpressions = subExpressions;
}
execute(inputNoteSet: NoteSet, executionContext: {}, searchContext: SearchContext) {
const resultNoteSet = new NoteSet();
for (const subExpression of this.subExpressions) {
resultNoteSet.mergeIn(subExpression.execute(inputNoteSet, executionContext, searchContext));
}
return resultNoteSet;
}
}
export default OrExp;

120
apps/server/src/services/search/expressions/order_by_and_limit.ts
View File

@@ -1,120 +0,0 @@
"use strict";
import type BNote from "../../../becca/entities/bnote.js";
import NoteSet from "../note_set.js";
import type SearchContext from "../search_context.js";
import Expression from "./expression.js";
interface ValueExtractor {
extract: (note: BNote) => number | string | null;
}
interface OrderDefinition {
direction?: string;
smaller: number;
larger: number;
valueExtractor: ValueExtractor;
}
class OrderByAndLimitExp extends Expression {
private orderDefinitions: OrderDefinition[];
limit: number;
subExpression: Expression | null;
constructor(orderDefinitions: Pick<OrderDefinition, "direction" | "valueExtractor">[], limit?: number) {
super();
this.orderDefinitions = orderDefinitions as OrderDefinition[];
for (const od of this.orderDefinitions) {
od.smaller = od.direction === "asc" ? -1 : 1;
od.larger = od.direction === "asc" ? 1 : -1;
}
this.limit = limit || 0;
this.subExpression = null; // it's expected to be set after construction
}
execute(inputNoteSet: NoteSet, executionContext: {}, searchContext: SearchContext) {
if (!this.subExpression) {
throw new Error("Missing subexpression");
}
let { notes } = this.subExpression.execute(inputNoteSet, executionContext, searchContext);
notes.sort((a, b) => {
for (const { valueExtractor, smaller, larger } of this.orderDefinitions) {
let valA: string | number | Date | null = valueExtractor.extract(a);
let valB: string | number | Date | null = valueExtractor.extract(b);
if (valA === undefined) {
valA = null;
}
if (valB === undefined) {
valB = null;
}
if (valA === null && valB === null) {
// neither has attribute at all
continue;
} else if (valB === null) {
return smaller;
} else if (valA === null) {
return larger;
}
// if both are dates, then parse them for dates comparison
if (typeof valA === "string" && this.isDate(valA) && typeof valB === "string" && this.isDate(valB)) {
valA = new Date(valA);
valB = new Date(valB);
}
// if both are numbers, then parse them for numerical comparison
else if (typeof valA === "string" && this.isNumber(valA) && typeof valB === "string" && this.isNumber(valB)) {
valA = parseFloat(valA);
valB = parseFloat(valB);
}
if (!valA && !valB) {
// the attribute value is empty/zero in both notes so continue to the next order definition
continue;
} else if (valA < valB) {
return smaller;
} else if (valA > valB) {
return larger;
}
// else the values are equal and continue to next order definition
}
return 0;
});
if (this.limit > 0) {
notes = notes.slice(0, this.limit);
}
const noteSet = new NoteSet(notes);
noteSet.sorted = true;
return noteSet;
}
isDate(date: number | string) {
return !isNaN(new Date(date).getTime());
}
isNumber(x: number | string) {
if (typeof x === "number") {
return true;
} else if (typeof x === "string") {
// isNaN will return false for blank string
return x.trim() !== "" && !isNaN(parseInt(x, 10));
} else {
return false;
}
}
}
export default OrderByAndLimitExp;

39
apps/server/src/services/search/expressions/parent_of.ts
View File

@@ -1,39 +0,0 @@
"use strict";
import Expression from "./expression.js";
import NoteSet from "../note_set.js";
import type SearchContext from "../search_context.js";
class ParentOfExp extends Expression {
private subExpression: Expression;
constructor(subExpression: Expression) {
super();
this.subExpression = subExpression;
}
execute(inputNoteSet: NoteSet, executionContext: {}, searchContext: SearchContext) {
const subInputNoteSet = new NoteSet();
for (const note of inputNoteSet.notes) {
subInputNoteSet.addAll(note.children);
}
const subResNoteSet = this.subExpression.execute(subInputNoteSet, executionContext, searchContext);
const resNoteSet = new NoteSet();
for (const childNote of subResNoteSet.notes) {
for (const parentNote of childNote.parents) {
if (inputNoteSet.hasNote(parentNote)) {
resNoteSet.add(parentNote);
}
}
}
return resNoteSet;
}
}
export default ParentOfExp;

89
apps/server/src/services/search/expressions/property_comparison.ts
View File

@@ -1,89 +0,0 @@
"use strict";
import Expression from "./expression.js";
import NoteSet from "../note_set.js";
import buildComparator from "../services/build_comparator.js";
/**
* Search string is lower cased for case-insensitive comparison. But when retrieving properties,
* we need the case-sensitive form, so we have this translation object.
*/
const PROP_MAPPING: Record<string, string> = {
noteid: "noteId",
title: "title",
type: "type",
mime: "mime",
isprotected: "isProtected",
isarchived: "isArchived",
datecreated: "dateCreated",
datemodified: "dateModified",
utcdatecreated: "utcDateCreated",
utcdatemodified: "utcDateModified",
parentcount: "parentCount",
childrencount: "childrenCount",
attributecount: "attributeCount",
labelcount: "labelCount",
ownedlabelcount: "ownedLabelCount",
relationcount: "relationCount",
ownedrelationcount: "ownedRelationCount",
relationcountincludinglinks: "relationCountIncludingLinks",
ownedrelationcountincludinglinks: "ownedRelationCountIncludingLinks",
targetrelationcount: "targetRelationCount",
targetrelationcountincludinglinks: "targetRelationCountIncludingLinks",
contentsize: "contentSize",
contentandattachmentssize: "contentAndAttachmentsSize",
contentandattachmentsandrevisionssize: "contentAndAttachmentsAndRevisionsSize",
revisioncount: "revisionCount"
};
interface SearchContext {
dbLoadNeeded?: boolean;
}
class PropertyComparisonExp extends Expression {
propertyName: string;
operator: string;
comparedValue: string;
private comparator;
static isProperty(name: string) {
return name in PROP_MAPPING;
}
constructor(searchContext: SearchContext, propertyName: string, operator: string, comparedValue: string) {
super();
this.propertyName = PROP_MAPPING[propertyName];
this.operator = operator; // for DEBUG mode
this.comparedValue = comparedValue; // for DEBUG mode
this.comparator = buildComparator(operator, comparedValue);
if (["contentsize", "contentandattachmentssize", "contentandattachmentsandrevisionssize", "revisioncount"].includes(this.propertyName)) {
searchContext.dbLoadNeeded = true;
}
}
execute(inputNoteSet: NoteSet, executionContext: {}, searchContext: SearchContext) {
const resNoteSet = new NoteSet();
for (const note of inputNoteSet.notes) {
let value = (note as any)[this.propertyName];
if (value !== undefined && value !== null && typeof value !== "string") {
value = value.toString();
}
if (value) {
value = value.toLowerCase();
}
if (this.comparator && this.comparator(value)) {
resNoteSet.add(note);
}
}
return resNoteSet;
}
}
export default PropertyComparisonExp;

45
apps/server/src/services/search/expressions/relation_where.ts
View File

@@ -1,45 +0,0 @@
"use strict";
import Expression from "./expression.js";
import NoteSet from "../note_set.js";
import becca from "../../../becca/becca.js";
import type SearchContext from "../search_context.js";
class RelationWhereExp extends Expression {
private relationName: string;
private subExpression: Expression;
constructor(relationName: string, subExpression: Expression) {
super();
this.relationName = relationName;
this.subExpression = subExpression;
}
execute(inputNoteSet: NoteSet, executionContext: {}, searchContext: SearchContext) {
const candidateNoteSet = new NoteSet();
for (const attr of becca.findAttributes("relation", this.relationName)) {
const note = attr.note;
if (inputNoteSet.hasNoteId(note.noteId) && attr.targetNote) {
const subInputNoteSet = new NoteSet([attr.targetNote]);
const subResNoteSet = this.subExpression.execute(subInputNoteSet, executionContext, searchContext);
if (subResNoteSet.hasNote(attr.targetNote)) {
if (attr.isInheritable) {
candidateNoteSet.addAll(note.getSubtreeNotesIncludingTemplated());
} else if (note.isInherited()) {
candidateNoteSet.addAll(note.getInheritingNotes());
} else {
candidateNoteSet.add(note);
}
}
}
}
return candidateNoteSet.intersection(inputNoteSet);
}
}
export default RelationWhereExp;

14
apps/server/src/services/search/expressions/true.ts
View File

@@ -1,14 +0,0 @@
"use strict";
import type NoteSet from "../note_set.js";
import type SearchContext from "../search_context.js";
import Expression from "./expression.js";
class TrueExp extends Expression {
execute(inputNoteSet: NoteSet, executionContext: {}, searchContext: SearchContext): NoteSet {
return inputNoteSet;
}
}
export default TrueExp;

2
apps/server/src/services/search/note_set.ts
View File

@@ -1,2 +0,0 @@
import { NoteSet } from "@triliumnext/core";
export default NoteSet;

75
apps/server/src/services/search/search_context.ts
View File

@@ -1,75 +0,0 @@
"use strict";
import hoistedNoteService from "../hoisted_note.js";
import type { SearchParams } from "./services/types.js";
class SearchContext {
fastSearch: boolean;
includeArchivedNotes: boolean;
includeHiddenNotes: boolean;
ignoreHoistedNote: boolean;
/** Whether to ignore certain attributes from the search such as ~internalLink. */
ignoreInternalAttributes: boolean;
ancestorNoteId?: string;
ancestorDepth?: string;
orderBy?: string;
orderDirection?: string;
limit?: number | null;
debug?: boolean;
debugInfo: {} | null;
fuzzyAttributeSearch: boolean;
enableFuzzyMatching: boolean; // Controls whether fuzzy matching is enabled for this search phase
highlightedTokens: string[];
originalQuery: string;
fulltextQuery: string;
dbLoadNeeded: boolean;
error: string | null;
constructor(params: SearchParams = {}) {
this.fastSearch = !!params.fastSearch;
this.includeArchivedNotes = !!params.includeArchivedNotes;
this.includeHiddenNotes = !!params.includeHiddenNotes;
this.ignoreHoistedNote = !!params.ignoreHoistedNote;
this.ignoreInternalAttributes = !!params.ignoreInternalAttributes;
this.ancestorNoteId = params.ancestorNoteId;
if (!this.ancestorNoteId && !this.ignoreHoistedNote) {
// hoisting in hidden subtree should not limit autocomplete
// since we want to link (create relations) to the normal non-hidden notes
this.ancestorNoteId = hoistedNoteService.getHoistedNoteId();
}
this.ancestorDepth = params.ancestorDepth;
this.orderBy = params.orderBy;
this.orderDirection = params.orderDirection;
this.limit = params.limit;
this.debug = params.debug;
this.debugInfo = null;
this.fuzzyAttributeSearch = !!params.fuzzyAttributeSearch;
this.enableFuzzyMatching = true; // Default to true for backward compatibility
this.highlightedTokens = [];
this.originalQuery = "";
this.fulltextQuery = ""; // complete fulltext part
// if true, becca does not have (up-to-date) information needed to process the query
// and some extra data needs to be loaded before executing
this.dbLoadNeeded = false;
this.error = null;
}
addError(error: string) {
// we record only the first error, subsequent ones are usually a consequence of the first
if (!this.error) {
this.error = error;
}
}
hasError() {
return !!this.error;
}
getError() {
return this.error;
}
}
export default SearchContext;

168
apps/server/src/services/search/search_result.ts
View File

@@ -1,168 +0,0 @@
import { becca_service } from "@triliumnext/core";
import becca from "../../becca/becca.js";
import {
calculateOptimizedEditDistance,
FUZZY_SEARCH_CONFIG,
normalizeSearchText} from "./utils/text_utils.js";
// Scoring constants for better maintainability
const SCORE_WEIGHTS = {
NOTE_ID_EXACT_MATCH: 1000,
TITLE_EXACT_MATCH: 2000,
TITLE_PREFIX_MATCH: 500,
TITLE_WORD_MATCH: 300,
TOKEN_EXACT_MATCH: 4,
TOKEN_PREFIX_MATCH: 2,
TOKEN_CONTAINS_MATCH: 1,
TOKEN_FUZZY_MATCH: 0.5,
TITLE_FACTOR: 2.0,
PATH_FACTOR: 0.3,
HIDDEN_NOTE_PENALTY: 3,
// Score caps to prevent fuzzy matches from outranking exact matches
MAX_FUZZY_SCORE_PER_TOKEN: 3, // Cap fuzzy token contributions to stay below exact matches
MAX_FUZZY_TOKEN_LENGTH_MULTIPLIER: 3, // Limit token length impact for fuzzy matches
MAX_TOTAL_FUZZY_SCORE: 200 // Total cap on fuzzy scoring per search
} as const;
class SearchResult {
notePathArray: string[];
score: number;
notePathTitle: string;
highlightedNotePathTitle?: string;
contentSnippet?: string;
highlightedContentSnippet?: string;
attributeSnippet?: string;
highlightedAttributeSnippet?: string;
private fuzzyScore: number; // Track fuzzy score separately
constructor(notePathArray: string[]) {
this.notePathArray = notePathArray;
this.notePathTitle = becca_service.getNoteTitleForPath(notePathArray);
this.score = 0;
this.fuzzyScore = 0;
}
get notePath() {
return this.notePathArray.join("/");
}
get noteId() {
return this.notePathArray[this.notePathArray.length - 1];
}
computeScore(fulltextQuery: string, tokens: string[], enableFuzzyMatching: boolean = true) {
this.score = 0;
this.fuzzyScore = 0; // Reset fuzzy score tracking
const note = becca.notes[this.noteId];
const normalizedQuery = normalizeSearchText(fulltextQuery.toLowerCase());
const normalizedTitle = normalizeSearchText(note.title.toLowerCase());
// Note ID exact match, much higher score
if (note.noteId.toLowerCase() === fulltextQuery) {
this.score += SCORE_WEIGHTS.NOTE_ID_EXACT_MATCH;
}
// Title matching scores with fuzzy matching support
if (normalizedTitle === normalizedQuery) {
this.score += SCORE_WEIGHTS.TITLE_EXACT_MATCH;
} else if (normalizedTitle.startsWith(normalizedQuery)) {
this.score += SCORE_WEIGHTS.TITLE_PREFIX_MATCH;
} else if (this.isWordMatch(normalizedTitle, normalizedQuery)) {
this.score += SCORE_WEIGHTS.TITLE_WORD_MATCH;
} else if (enableFuzzyMatching) {
// Try fuzzy matching for typos only if enabled
const fuzzyScore = this.calculateFuzzyTitleScore(normalizedTitle, normalizedQuery);
this.score += fuzzyScore;
this.fuzzyScore += fuzzyScore; // Track fuzzy score contributions
}
// Add scores for token matches
this.addScoreForStrings(tokens, note.title, SCORE_WEIGHTS.TITLE_FACTOR, enableFuzzyMatching);
this.addScoreForStrings(tokens, this.notePathTitle, SCORE_WEIGHTS.PATH_FACTOR, enableFuzzyMatching);
if (note.isInHiddenSubtree()) {
this.score = this.score / SCORE_WEIGHTS.HIDDEN_NOTE_PENALTY;
}
}
addScoreForStrings(tokens: string[], str: string, factor: number, enableFuzzyMatching: boolean = true) {
const normalizedStr = normalizeSearchText(str.toLowerCase());
const chunks = normalizedStr.split(" ");
let tokenScore = 0;
for (const chunk of chunks) {
for (const token of tokens) {
const normalizedToken = normalizeSearchText(token.toLowerCase());
if (chunk === normalizedToken) {
tokenScore += SCORE_WEIGHTS.TOKEN_EXACT_MATCH * token.length * factor;
} else if (chunk.startsWith(normalizedToken)) {
tokenScore += SCORE_WEIGHTS.TOKEN_PREFIX_MATCH * token.length * factor;
} else if (chunk.includes(normalizedToken)) {
tokenScore += SCORE_WEIGHTS.TOKEN_CONTAINS_MATCH * token.length * factor;
} else {
// Try fuzzy matching for individual tokens with caps applied
const editDistance = calculateOptimizedEditDistance(chunk, normalizedToken, FUZZY_SEARCH_CONFIG.MAX_EDIT_DISTANCE);
if (editDistance <= FUZZY_SEARCH_CONFIG.MAX_EDIT_DISTANCE &&
normalizedToken.length >= FUZZY_SEARCH_CONFIG.MIN_FUZZY_TOKEN_LENGTH &&
this.fuzzyScore < SCORE_WEIGHTS.MAX_TOTAL_FUZZY_SCORE) {
const fuzzyWeight = SCORE_WEIGHTS.TOKEN_FUZZY_MATCH * (1 - editDistance / FUZZY_SEARCH_CONFIG.MAX_EDIT_DISTANCE);
// Apply caps: limit token length multiplier and per-token contribution
const cappedTokenLength = Math.min(token.length, SCORE_WEIGHTS.MAX_FUZZY_TOKEN_LENGTH_MULTIPLIER);
const fuzzyTokenScore = Math.min(
fuzzyWeight * cappedTokenLength * factor,
SCORE_WEIGHTS.MAX_FUZZY_SCORE_PER_TOKEN
);
tokenScore += fuzzyTokenScore;
this.fuzzyScore += fuzzyTokenScore;
}
}
}
}
this.score += tokenScore;
}
/**
* Checks if the query matches as a complete word in the text
*/
private isWordMatch(text: string, query: string): boolean {
return text.includes(` ${query} `) ||
text.startsWith(`${query} `) ||
text.endsWith(` ${query}`);
}
/**
* Calculates fuzzy matching score for title matches with caps applied
*/
private calculateFuzzyTitleScore(title: string, query: string): number {
// Check if we've already hit the fuzzy scoring cap
if (this.fuzzyScore >= SCORE_WEIGHTS.MAX_TOTAL_FUZZY_SCORE) {
return 0;
}
const editDistance = calculateOptimizedEditDistance(title, query, FUZZY_SEARCH_CONFIG.MAX_EDIT_DISTANCE);
const maxLen = Math.max(title.length, query.length);
// Only apply fuzzy matching if the query is reasonably long and edit distance is small
if (query.length >= FUZZY_SEARCH_CONFIG.MIN_FUZZY_TOKEN_LENGTH &&
editDistance <= FUZZY_SEARCH_CONFIG.MAX_EDIT_DISTANCE &&
editDistance / maxLen <= 0.3) {
const similarity = 1 - (editDistance / maxLen);
const baseFuzzyScore = SCORE_WEIGHTS.TITLE_WORD_MATCH * similarity * 0.7; // Reduced weight for fuzzy matches
// Apply cap to ensure fuzzy title matches don't exceed reasonable bounds
return Math.min(baseFuzzyScore, SCORE_WEIGHTS.MAX_TOTAL_FUZZY_SCORE * 0.3);
}
return 0;
}
}
export default SearchResult;

114
apps/server/src/services/search/services/build_comparator.ts
View File

@@ -1,114 +0,0 @@
import { normalizeSearchText, fuzzyMatchWord, FUZZY_SEARCH_CONFIG } from "../utils/text_utils.js";
const cachedRegexes: Record<string, RegExp> = {};
function getRegex(str: string) {
if (!(str in cachedRegexes)) {
cachedRegexes[str] = new RegExp(str);
}
return cachedRegexes[str];
}
type Comparator<T> = (comparedValue: T) => (val: string) => boolean;
const stringComparators: Record<string, Comparator<string>> = {
"=": (comparedValue) => (val) => {
// For the = operator, check if the value contains the exact word or phrase
// This is case-insensitive
if (!val) return false;
const normalizedVal = normalizeSearchText(val);
const normalizedCompared = normalizeSearchText(comparedValue);
// If comparedValue has spaces, it's a multi-word phrase
// Check for substring match (consecutive phrase)
if (normalizedCompared.includes(" ")) {
return normalizedVal.includes(normalizedCompared);
}
// For single word, split into words and check for exact word match
const words = normalizedVal.split(/\s+/);
return words.some(word => word === normalizedCompared);
},
"!=": (comparedValue) => (val) => {
// Negation of exact word/phrase match
if (!val) return true;
const normalizedVal = normalizeSearchText(val);
const normalizedCompared = normalizeSearchText(comparedValue);
// If comparedValue has spaces, it's a multi-word phrase
// Check for substring match (consecutive phrase) and negate
if (normalizedCompared.includes(" ")) {
return !normalizedVal.includes(normalizedCompared);
}
// For single word, split into words and check for exact word match, then negate
const words = normalizedVal.split(/\s+/);
return !words.some(word => word === normalizedCompared);
},
">": (comparedValue) => (val) => val > comparedValue,
">=": (comparedValue) => (val) => val >= comparedValue,
"<": (comparedValue) => (val) => val < comparedValue,
"<=": (comparedValue) => (val) => val <= comparedValue,
"*=": (comparedValue) => (val) => !!val && val.endsWith(comparedValue),
"=*": (comparedValue) => (val) => !!val && val.startsWith(comparedValue),
"*=*": (comparedValue) => (val) => !!val && val.includes(comparedValue),
"%=": (comparedValue) => (val) => !!val && !!getRegex(comparedValue).test(val),
"~=": (comparedValue) => (val) => {
if (!val || !comparedValue) return false;
// Validate minimum length for fuzzy search to prevent false positives
if (comparedValue.length < FUZZY_SEARCH_CONFIG.MIN_FUZZY_TOKEN_LENGTH) {
return val.includes(comparedValue);
}
const normalizedVal = normalizeSearchText(val);
const normalizedCompared = normalizeSearchText(comparedValue);
// First try exact substring match
if (normalizedVal.includes(normalizedCompared)) {
return true;
}
// Then try fuzzy word matching
const words = normalizedVal.split(/\s+/);
return words.some(word => fuzzyMatchWord(normalizedCompared, word));
},
"~*": (comparedValue) => (val) => {
if (!val || !comparedValue) return false;
// Validate minimum length for fuzzy search
if (comparedValue.length < FUZZY_SEARCH_CONFIG.MIN_FUZZY_TOKEN_LENGTH) {
return val.includes(comparedValue);
}
const normalizedVal = normalizeSearchText(val);
const normalizedCompared = normalizeSearchText(comparedValue);
// For ~* operator, use fuzzy matching across the entire content
return fuzzyMatchWord(normalizedCompared, normalizedVal);
}
};
const numericComparators: Record<string, Comparator<number>> = {
">": (comparedValue) => (val) => parseFloat(val) > comparedValue,
">=": (comparedValue) => (val) => parseFloat(val) >= comparedValue,
"<": (comparedValue) => (val) => parseFloat(val) < comparedValue,
"<=": (comparedValue) => (val) => parseFloat(val) <= comparedValue
};
function buildComparator(operator: string, comparedValue: string) {
comparedValue = comparedValue.toLowerCase();
if (operator in numericComparators && !isNaN(+comparedValue)) {
return numericComparators[operator](parseFloat(comparedValue));
}
if (operator in stringComparators) {
return stringComparators[operator](comparedValue);
}
}
export default buildComparator;

13
apps/server/src/services/search/services/handle_parens.spec.ts
View File

@@ -1,13 +0,0 @@
import { describe, it, expect } from "vitest";
import handleParens from "./handle_parens.js";
import type { TokenStructure } from "./types.js";
describe("Parens handler", () => {
it("handles parens", () => {
const input = ["(", "hello", ")", "and", "(", "(", "pick", "one", ")", "and", "another", ")"].map((token) => ({ token }));
const actual: TokenStructure = [[{ token: "hello" }], { token: "and" }, [[{ token: "pick" }, { token: "one" }], { token: "and" }, { token: "another" }]];
expect(handleParens(input)).toEqual(actual);
});
});

46
apps/server/src/services/search/services/handle_parens.ts
View File

@@ -1,46 +0,0 @@
import type { TokenData, TokenStructure } from "./types.js";
/**
* This will create a recursive object from a list of tokens - tokens between parenthesis are grouped in a single array
*/
function handleParens(tokens: TokenStructure) {
if (tokens.length === 0) {
return [];
}
while (true) {
const leftIdx = tokens.findIndex((token) => "token" in token && token.token === "(");
if (leftIdx === -1) {
return tokens;
}
let rightIdx;
let parensLevel = 0;
for (rightIdx = leftIdx; rightIdx < tokens.length; rightIdx++) {
const token = tokens[rightIdx];
if (!("token" in token)) {
continue;
}
if (token.token === ")") {
parensLevel--;
if (parensLevel === 0) {
break;
}
} else if (token.token === "(") {
parensLevel++;
}
}
if (rightIdx >= tokens.length) {
throw new Error("Did not find matching right parenthesis.");
}
tokens = [...tokens.slice(0, leftIdx), handleParens(tokens.slice(leftIdx + 1, rightIdx)), ...tokens.slice(rightIdx + 1)] as (TokenData | TokenData[])[];
}
}
export default handleParens;

191
apps/server/src/services/search/services/lex.spec.ts
View File

@@ -1,191 +0,0 @@
import { describe, it, expect } from "vitest";
import lex from "./lex.js";
describe("Lexer fulltext", () => {
it("simple lexing", () => {
expect(lex("hello world").fulltextTokens.map((t) => t.token)).toEqual(["hello", "world"]);
expect(lex("hello, world").fulltextTokens.map((t) => t.token)).toEqual(["hello", "world"]);
});
it("use quotes to keep words together", () => {
expect(lex("'hello world' my friend").fulltextTokens.map((t) => t.token)).toEqual(["hello world", "my", "friend"]);
expect(lex('"hello world" my friend').fulltextTokens.map((t) => t.token)).toEqual(["hello world", "my", "friend"]);
expect(lex("`hello world` my friend").fulltextTokens.map((t) => t.token)).toEqual(["hello world", "my", "friend"]);
});
it("you can use different quotes and other special characters inside quotes", () => {
expect(lex("'i can use \" or ` or #~=*' without problem").fulltextTokens.map((t) => t.token)).toEqual(['i can use " or ` or #~=*', "without", "problem"]);
});
it("I can use backslash to escape quotes", () => {
expect(lex('hello \\"world\\"').fulltextTokens.map((t) => t.token)).toEqual(["hello", '"world"']);
expect(lex("hello \\'world\\'").fulltextTokens.map((t) => t.token)).toEqual(["hello", "'world'"]);
expect(lex("hello \\`world\\`").fulltextTokens.map((t) => t.token)).toEqual(["hello", "`world`"]);
expect(lex('"hello \\"world\\"').fulltextTokens.map((t) => t.token)).toEqual(['hello "world"']);
expect(lex("'hello \\'world\\''").fulltextTokens.map((t) => t.token)).toEqual(["hello 'world'"]);
expect(lex("`hello \\`world\\``").fulltextTokens.map((t) => t.token)).toEqual(["hello `world`"]);
expect(lex("\\#token").fulltextTokens.map((t) => t.token)).toEqual(["#token"]);
});
it("quote inside a word does not have a special meaning", () => {
const lexResult = lex("d'Artagnan is dead #hero = d'Artagnan");
expect(lexResult.fulltextTokens.map((t) => t.token)).toEqual(["d'artagnan", "is", "dead"]);
expect(lexResult.expressionTokens.map((t) => t.token)).toEqual(["#hero", "=", "d'artagnan"]);
});
it("if quote is not ended then it's just one long token", () => {
expect(lex("'unfinished quote").fulltextTokens.map((t) => t.token)).toEqual(["unfinished quote"]);
});
it("parenthesis and symbols in fulltext section are just normal characters", () => {
expect(lex("what's u=p <b(r*t)h>").fulltextTokens.map((t) => t.token)).toEqual(["what's", "u=p", "<b(r*t)h>"]);
});
it("operator characters in expressions are separate tokens", () => {
expect(lex("# abc+=-def**-+d").expressionTokens.map((t) => t.token)).toEqual(["#", "abc", "+=-", "def", "**-+", "d"]);
});
it("escaping special characters", () => {
expect(lex("hello \\#\\~\\'").fulltextTokens.map((t) => t.token)).toEqual(["hello", "#~'"]);
});
it("recognizes leading = operator for exact match", () => {
const result1 = lex("=example");
expect(result1.fulltextTokens.map((t) => t.token)).toEqual(["example"]);
expect(result1.leadingOperator).toBe("=");
const result2 = lex("=hello world");
expect(result2.fulltextTokens.map((t) => t.token)).toEqual(["hello", "world"]);
expect(result2.leadingOperator).toBe("=");
const result3 = lex("='hello world'");
expect(result3.fulltextTokens.map((t) => t.token)).toEqual(["hello world"]);
expect(result3.leadingOperator).toBe("=");
});
it("doesn't treat = as leading operator in other contexts", () => {
const result1 = lex("==example");
expect(result1.fulltextTokens.map((t) => t.token)).toEqual(["==example"]);
expect(result1.leadingOperator).toBe("");
const result2 = lex("= example");
expect(result2.fulltextTokens.map((t) => t.token)).toEqual(["=", "example"]);
expect(result2.leadingOperator).toBe("");
const result3 = lex("example");
expect(result3.fulltextTokens.map((t) => t.token)).toEqual(["example"]);
expect(result3.leadingOperator).toBe("");
});
});
describe("Lexer expression", () => {
it("simple attribute existence", () => {
expect(lex("#label ~relation").expressionTokens.map((t) => t.token)).toEqual(["#label", "~relation"]);
});
it("simple label operators", () => {
expect(lex("#label*=*text").expressionTokens.map((t) => t.token)).toEqual(["#label", "*=*", "text"]);
});
it("simple label operator with in quotes", () => {
expect(lex("#label*=*'text'").expressionTokens).toEqual([
{ token: "#label", inQuotes: false, startIndex: 0, endIndex: 5 },
{ token: "*=*", inQuotes: false, startIndex: 6, endIndex: 8 },
{ token: "text", inQuotes: true, startIndex: 10, endIndex: 13 }
]);
});
it("simple label operator with param without quotes", () => {
expect(lex("#label*=*text").expressionTokens).toEqual([
{ token: "#label", inQuotes: false, startIndex: 0, endIndex: 5 },
{ token: "*=*", inQuotes: false, startIndex: 6, endIndex: 8 },
{ token: "text", inQuotes: false, startIndex: 9, endIndex: 12 }
]);
});
it("simple label operator with empty string param", () => {
expect(lex("#label = ''").expressionTokens).toEqual([
{ token: "#label", inQuotes: false, startIndex: 0, endIndex: 5 },
{ token: "=", inQuotes: false, startIndex: 7, endIndex: 7 },
// weird case for empty strings which ends up with endIndex < startIndex :-(
{ token: "", inQuotes: true, startIndex: 10, endIndex: 9 }
]);
});
it("note. prefix also separates fulltext from expression", () => {
expect(lex(`hello fulltext note.labels.capital = Prague`).expressionTokens.map((t) => t.token)).toEqual(["note", ".", "labels", ".", "capital", "=", "prague"]);
});
it("note. prefix in quotes will note start expression", () => {
expect(lex(`hello fulltext "note.txt"`).expressionTokens.map((t) => t.token)).toEqual([]);
expect(lex(`hello fulltext "note.txt"`).fulltextTokens.map((t) => t.token)).toEqual(["hello", "fulltext", "note.txt"]);
});
it("complex expressions with and, or and parenthesis", () => {
expect(lex(`# (#label=text OR #second=text) AND ~relation`).expressionTokens.map((t) => t.token)).toEqual([
"#",
"(",
"#label",
"=",
"text",
"or",
"#second",
"=",
"text",
")",
"and",
"~relation"
]);
});
it("dot separated properties", () => {
expect(lex(`# ~author.title = 'Hugh Howey' AND note.'book title' = 'Silo'`).expressionTokens.map((t) => t.token)).toEqual([
"#",
"~author",
".",
"title",
"=",
"hugh howey",
"and",
"note",
".",
"book title",
"=",
"silo"
]);
});
it("negation of label and relation", () => {
expect(lex(`#!capital ~!neighbor`).expressionTokens.map((t) => t.token)).toEqual(["#!capital", "~!neighbor"]);
});
it("negation of sub-expression", () => {
expect(lex(`# not(#capital) and note.noteId != "root"`).expressionTokens.map((t) => t.token)).toEqual(["#", "not", "(", "#capital", ")", "and", "note", ".", "noteid", "!=", "root"]);
});
it("order by multiple labels", () => {
expect(lex(`# orderby #a,#b`).expressionTokens.map((t) => t.token)).toEqual(["#", "orderby", "#a", ",", "#b"]);
});
});
describe("Lexer invalid queries and edge cases", () => {
it("concatenated attributes", () => {
expect(lex("#label~relation").expressionTokens.map((t) => t.token)).toEqual(["#label", "~relation"]);
});
it("trailing escape \\", () => {
expect(lex("abc \\").fulltextTokens.map((t) => t.token)).toEqual(["abc", "\\"]);
});
});

144
apps/server/src/services/search/services/lex.ts
View File

@@ -1,144 +0,0 @@
import type { TokenData } from "./types.js";
function lex(str: string) {
str = str.toLowerCase();
let fulltextQuery = "";
const fulltextTokens: TokenData[] = [];
const expressionTokens: TokenData[] = [];
let quotes: boolean | string = false; // otherwise contains used quote - ', " or `
let fulltextEnded = false;
let currentWord = "";
let leadingOperator = "";
function isSymbolAnOperator(chr: string) {
return ["=", "*", ">", "<", "!", "-", "+", "%", ","].includes(chr);
}
// Check if the string starts with an exact match operator
// This allows users to use "=searchterm" for exact matching
if (str.startsWith("=") && str.length > 1 && str[1] !== "=" && str[1] !== " ") {
leadingOperator = "=";
str = str.substring(1); // Remove the leading operator from the string
}
function isPreviousSymbolAnOperator() {
if (currentWord.length === 0) {
return false;
} else {
return isSymbolAnOperator(currentWord[currentWord.length - 1]);
}
}
function finishWord(endIndex: number, createAlsoForEmptyWords = false) {
if (currentWord === "" && !createAlsoForEmptyWords) {
return;
}
const rec: TokenData = {
token: currentWord,
inQuotes: !!quotes,
startIndex: endIndex - currentWord.length + 1,
endIndex
};
if (fulltextEnded) {
expressionTokens.push(rec);
} else {
fulltextTokens.push(rec);
fulltextQuery = str.substr(0, endIndex + 1);
}
currentWord = "";
}
for (let i = 0; i < str.length; i++) {
const chr = str[i];
if (chr === "\\") {
if (i + 1 < str.length) {
i++;
currentWord += str[i];
} else {
currentWord += chr;
}
continue;
} else if (['"', "'", "`"].includes(chr)) {
if (!quotes) {
if (currentWord.length === 0 || isPreviousSymbolAnOperator()) {
finishWord(i - 1);
quotes = chr;
} else {
// quote inside a word does not have special meening and does not break word
// e.g. d'Artagnan is kept as a single token
currentWord += chr;
}
} else if (quotes === chr) {
finishWord(i - 1, true);
quotes = false;
} else {
// it's a quote, but within other kind of quotes, so it's valid as a literal character
currentWord += chr;
}
continue;
} else if (!quotes) {
if (!fulltextEnded && currentWord === "note" && chr === "." && i + 1 < str.length) {
fulltextEnded = true;
}
if (chr === "#" || chr === "~") {
if (!fulltextEnded) {
fulltextEnded = true;
} else {
finishWord(i - 1);
}
currentWord = chr;
continue;
} else if (["#", "~"].includes(currentWord) && chr === "!") {
currentWord += chr;
continue;
} else if (chr === " ") {
finishWord(i - 1);
continue;
} else if (fulltextEnded && ["(", ")", "."].includes(chr)) {
finishWord(i - 1);
currentWord += chr;
finishWord(i);
continue;
} else if (fulltextEnded && !["#!", "~!"].includes(currentWord) && isPreviousSymbolAnOperator() !== isSymbolAnOperator(chr)) {
finishWord(i - 1);
currentWord += chr;
continue;
}
}
if (chr === ",") {
continue;
}
currentWord += chr;
}
finishWord(str.length - 1);
fulltextQuery = fulltextQuery.trim();
return {
fulltextQuery,
fulltextTokens,
expressionTokens,
leadingOperator
};
}
export default lex;

413
apps/server/src/services/search/services/parse.spec.ts
View File

@@ -1,413 +0,0 @@
import { describe, it, expect } from "vitest";
import AndExp from "../../search/expressions/and.js";
import AttributeExistsExp from "../../search/expressions/attribute_exists.js";
import type Expression from "../../search/expressions/expression.js";
import LabelComparisonExp from "../../search/expressions/label_comparison.js";
import NotExp from "../../search/expressions/not.js";
import NoteContentFulltextExp from "../../search/expressions/note_content_fulltext.js";
import NoteFlatTextExp from "../../search/expressions/note_flat_text.js";
import OrExp from "../../search/expressions/or.js";
import OrderByAndLimitExp from "../../search/expressions/order_by_and_limit.js";
import PropertyComparisonExp from "../../search/expressions/property_comparison.js";
import SearchContext from "../../search/search_context.js";
import { default as parseInternal, type ParseOpts } from "./parse.js";
describe("Parser", () => {
it("fulltext parser without content", () => {
const rootExp = parse(
{
fulltextTokens: tokens(["hello", "hi"]),
expressionTokens: [],
searchContext: new SearchContext()
},
AndExp
);
expectExpression(rootExp.subExpressions[0], PropertyComparisonExp);
const orExp = expectExpression(rootExp.subExpressions[2], OrExp);
const flatTextExp = expectExpression(orExp.subExpressions[0], NoteFlatTextExp);
expect(flatTextExp.tokens).toEqual(["hello", "hi"]);
});
it("fulltext parser with content", () => {
const rootExp = parse(
{
fulltextTokens: tokens(["hello", "hi"]),
expressionTokens: [],
searchContext: new SearchContext()
},
AndExp
);
assertIsArchived(rootExp.subExpressions[0]);
const orExp = expectExpression(rootExp.subExpressions[2], OrExp);
const firstSub = expectExpression(orExp.subExpressions[0], NoteFlatTextExp);
expect(firstSub.tokens).toEqual(["hello", "hi"]);
const secondSub = expectExpression(orExp.subExpressions[1], NoteContentFulltextExp);
expect(secondSub.tokens).toEqual(["hello", "hi"]);
});
it("simple label comparison", () => {
const rootExp = parse(
{
fulltextTokens: [],
expressionTokens: tokens(["#mylabel", "=", "text"]),
searchContext: new SearchContext()
},
AndExp
);
assertIsArchived(rootExp.subExpressions[0]);
const labelComparisonExp = expectExpression(rootExp.subExpressions[2], LabelComparisonExp);
expect(labelComparisonExp.attributeType).toEqual("label");
expect(labelComparisonExp.attributeName).toEqual("mylabel");
expect(labelComparisonExp.comparator).toBeTruthy();
});
it("simple attribute negation", () => {
let rootExp = parse(
{
fulltextTokens: [],
expressionTokens: tokens(["#!mylabel"]),
searchContext: new SearchContext()
},
AndExp
);
assertIsArchived(rootExp.subExpressions[0]);
let notExp = expectExpression(rootExp.subExpressions[2], NotExp);
let attributeExistsExp = expectExpression(notExp.subExpression, AttributeExistsExp);
expect(attributeExistsExp.attributeType).toEqual("label");
expect(attributeExistsExp.attributeName).toEqual("mylabel");
rootExp = parse(
{
fulltextTokens: [],
expressionTokens: tokens(["~!myrelation"]),
searchContext: new SearchContext()
},
AndExp
);
assertIsArchived(rootExp.subExpressions[0]);
notExp = expectExpression(rootExp.subExpressions[2], NotExp);
attributeExistsExp = expectExpression(notExp.subExpression, AttributeExistsExp);
expect(attributeExistsExp.attributeType).toEqual("relation");
expect(attributeExistsExp.attributeName).toEqual("myrelation");
});
it("simple label AND", () => {
const rootExp = parse(
{
fulltextTokens: [],
expressionTokens: tokens(["#first", "=", "text", "and", "#second", "=", "text"]),
searchContext: new SearchContext()
},
AndExp
);
assertIsArchived(rootExp.subExpressions[0]);
const andExp = expectExpression(rootExp.subExpressions[2], AndExp);
const [firstSub, secondSub] = expectSubexpressions(andExp, LabelComparisonExp, LabelComparisonExp);
expect(firstSub.attributeName).toEqual("first");
expect(secondSub.attributeName).toEqual("second");
});
it("simple label AND without explicit AND", () => {
const rootExp = parse(
{
fulltextTokens: [],
expressionTokens: tokens(["#first", "=", "text", "#second", "=", "text"]),
searchContext: new SearchContext()
},
AndExp
);
assertIsArchived(rootExp.subExpressions[0]);
const andExp = expectExpression(rootExp.subExpressions[2], AndExp);
const [firstSub, secondSub] = expectSubexpressions(andExp, LabelComparisonExp, LabelComparisonExp);
expect(firstSub.attributeName).toEqual("first");
expect(secondSub.attributeName).toEqual("second");
});
it("simple label OR", () => {
const rootExp = parse(
{
fulltextTokens: [],
expressionTokens: tokens(["#first", "=", "text", "or", "#second", "=", "text"]),
searchContext: new SearchContext()
},
AndExp
);
assertIsArchived(rootExp.subExpressions[0]);
const orExp = expectExpression(rootExp.subExpressions[2], OrExp);
const [firstSub, secondSub] = expectSubexpressions(orExp, LabelComparisonExp, LabelComparisonExp);
expect(firstSub.attributeName).toEqual("first");
expect(secondSub.attributeName).toEqual("second");
});
it("fulltext and simple label", () => {
const rootExp = parse(
{
fulltextTokens: tokens(["hello"]),
expressionTokens: tokens(["#mylabel", "=", "text"]),
searchContext: new SearchContext()
},
AndExp
);
const [firstSub, _, thirdSub, fourth] = expectSubexpressions(rootExp, PropertyComparisonExp, undefined, OrExp, LabelComparisonExp);
expect(firstSub.propertyName).toEqual("isArchived");
const noteFlatTextExp = expectExpression(thirdSub.subExpressions[0], NoteFlatTextExp);
expect(noteFlatTextExp.tokens).toEqual(["hello"]);
expect(fourth.attributeName).toEqual("mylabel");
});
it("label sub-expression", () => {
const rootExp = parse(
{
fulltextTokens: [],
expressionTokens: tokens(["#first", "=", "text", "or", ["#second", "=", "text", "and", "#third", "=", "text"]]),
searchContext: new SearchContext()
},
AndExp
);
assertIsArchived(rootExp.subExpressions[0]);
const orExp = expectExpression(rootExp.subExpressions[2], OrExp);
const [firstSub, secondSub] = expectSubexpressions(orExp, LabelComparisonExp, AndExp);
expect(firstSub.attributeName).toEqual("first");
const [firstSubSub, secondSubSub] = expectSubexpressions(secondSub, LabelComparisonExp, LabelComparisonExp);
expect(firstSubSub.attributeName).toEqual("second");
expect(secondSubSub.attributeName).toEqual("third");
});
it("label sub-expression without explicit operator", () => {
const rootExp = parse(
{
fulltextTokens: [],
expressionTokens: tokens(["#first", ["#second", "or", "#third"], "#fourth"]),
searchContext: new SearchContext()
},
AndExp
);
assertIsArchived(rootExp.subExpressions[0]);
const andExp = expectExpression(rootExp.subExpressions[2], AndExp);
const [firstSub, secondSub, thirdSub] = expectSubexpressions(andExp, AttributeExistsExp, OrExp, AttributeExistsExp);
expect(firstSub.attributeName).toEqual("first");
const [firstSubSub, secondSubSub] = expectSubexpressions(secondSub, AttributeExistsExp, AttributeExistsExp);
expect(firstSubSub.attributeName).toEqual("second");
expect(secondSubSub.attributeName).toEqual("third");
expect(thirdSub.attributeName).toEqual("fourth");
});
it("parses limit without order by", () => {
const rootExp = parse(
{
fulltextTokens: tokens(["hello", "hi"]),
expressionTokens: [],
searchContext: new SearchContext({ limit: 2 })
},
OrderByAndLimitExp
);
expect(rootExp.limit).toBe(2);
expect(rootExp.subExpression).toBeInstanceOf(AndExp);
});
describe("orderBy with level > 0", () => {
it("and grouping parentheses should parse without error", () => {
const searchContext = new SearchContext();
const rootExp = parseInternal(
{
fulltextTokens: [],
expressionTokens: tokens(["#foo", "and" , ["#bar", "or", "#baz", ], "orderby", "#priority", "desc"]),
searchContext
}
);
expect(searchContext.error).toBeNull();
});
it("and not() should parse without error", () => {
const searchContext = new SearchContext();
const rootExp = parseInternal(
{
fulltextTokens: [],
expressionTokens: tokens(["#foo", "and" , "not", [ "#bar", "=", "baz" ], "orderby", "#priority", "desc"]) ,
searchContext
}
);
expect(searchContext.error).toBeNull();
});
});
});
describe("Invalid expressions", () => {
it("incomplete comparison", () => {
const searchContext = new SearchContext();
parseInternal({
fulltextTokens: [],
expressionTokens: tokens(["#first", "="]),
searchContext
});
expect(searchContext.error).toEqual('Misplaced or incomplete expression "="');
});
it("comparison between labels is impossible", () => {
let searchContext = new SearchContext();
searchContext.originalQuery = "#first = #second";
parseInternal({
fulltextTokens: [],
expressionTokens: tokens(["#first", "=", "#second"]),
searchContext
});
expect(searchContext.error).toEqual(`Error near token "#second" in "#first = #second", it's possible to compare with constant only.`);
searchContext = new SearchContext();
searchContext.originalQuery = "#first = note.relations.second";
parseInternal({
fulltextTokens: [],
expressionTokens: tokens(["#first", "=", "note", ".", "relations", "second"]),
searchContext
});
expect(searchContext.error).toEqual(`Error near token "note" in "#first = note.relations.second", it's possible to compare with constant only.`);
const rootExp = parse(
{
fulltextTokens: [],
expressionTokens: [
{ token: "#first", inQuotes: false },
{ token: "=", inQuotes: false },
{ token: "#second", inQuotes: true }
],
searchContext: new SearchContext()
},
AndExp
);
assertIsArchived(rootExp.subExpressions[0]);
const labelComparisonExp = expectExpression(rootExp.subExpressions[2], LabelComparisonExp);
expect(labelComparisonExp.attributeType).toEqual("label");
expect(labelComparisonExp.attributeName).toEqual("first");
expect(labelComparisonExp.comparator).toBeTruthy();
});
it("searching by relation without note property", () => {
const searchContext = new SearchContext();
parseInternal({
fulltextTokens: [],
expressionTokens: tokens(["~first", "=", "text", "-", "abc"]),
searchContext
});
expect(searchContext.error).toEqual('Relation can be compared only with property, e.g. ~relation.title=hello in ""');
});
});
type ClassType<T extends Expression> = new (...args: any[]) => T;
function tokens(toks: (string | string[])[], cur = 0): Array<any> {
return toks.map((arg) => {
if (Array.isArray(arg)) {
return tokens(arg, cur);
} else {
cur += arg.length;
return {
token: arg,
inQuotes: false,
startIndex: cur - arg.length,
endIndex: cur - 1
};
}
});
}
function assertIsArchived(_exp: Expression) {
const exp = expectExpression(_exp, PropertyComparisonExp);
expect(exp.propertyName).toEqual("isArchived");
expect(exp.operator).toEqual("=");
expect(exp.comparedValue).toEqual("false");
}
/**
* Parses the corresponding {@link Expression} from plain text, while also expecting the resulting expression to be of the given type.
*
* @param opts the options for parsing.
* @param type the expected type of the expression.
* @returns the expression typecasted to the expected type.
*/
function parse<T extends Expression>(opts: ParseOpts, type: ClassType<T>) {
return expectExpression(parseInternal(opts), type);
}
/**
* Expects the given {@link Expression} to be of the given type.
*
* @param exp an instance of an {@link Expression}.
* @param type a type class such as {@link AndExp}, {@link OrExp}, etc.
* @returns the same expression typecasted to the expected type.
*/
function expectExpression<T extends Expression>(exp: Expression, type: ClassType<T>) {
expect(exp).toBeInstanceOf(type);
return exp as T;
}
/**
* For an {@link AndExp}, it goes through all its subexpressions (up to fourth) and checks their type and returns them as a typecasted array.
* Each subexpression can have their own type.
*
* @param exp the expression containing one or more subexpressions.
* @param firstType the type of the first subexpression.
* @param secondType the type of the second subexpression.
* @param thirdType the type of the third subexpression.
* @param fourthType the type of the fourth subexpression.
* @returns an array of all the subexpressions (in order) typecasted to their expected type.
*/
function expectSubexpressions<FirstT extends Expression, SecondT extends Expression, ThirdT extends Expression, FourthT extends Expression>(
exp: AndExp,
firstType: ClassType<FirstT>,
secondType?: ClassType<SecondT>,
thirdType?: ClassType<ThirdT>,
fourthType?: ClassType<FourthT>
): [FirstT, SecondT, ThirdT, FourthT] {
expectExpression(exp.subExpressions[0], firstType);
if (secondType) {
expectExpression(exp.subExpressions[1], secondType);
}
if (thirdType) {
expectExpression(exp.subExpressions[2], thirdType);
}
if (fourthType) {
expectExpression(exp.subExpressions[3], fourthType);
}
return [exp.subExpressions[0] as FirstT, exp.subExpressions[1] as SecondT, exp.subExpressions[2] as ThirdT, exp.subExpressions[3] as FourthT];
}

501
apps/server/src/services/search/services/parse.ts
View File

@@ -1,501 +0,0 @@
"use strict";
import { dayjs } from "@triliumnext/commons";
import AndExp from "../expressions/and.js";
import OrExp from "../expressions/or.js";
import NotExp from "../expressions/not.js";
import ChildOfExp from "../expressions/child_of.js";
import DescendantOfExp from "../expressions/descendant_of.js";
import ParentOfExp from "../expressions/parent_of.js";
import RelationWhereExp from "../expressions/relation_where.js";
import PropertyComparisonExp from "../expressions/property_comparison.js";
import AttributeExistsExp from "../expressions/attribute_exists.js";
import LabelComparisonExp from "../expressions/label_comparison.js";
import NoteFlatTextExp from "../expressions/note_flat_text.js";
import NoteContentFulltextExp from "../expressions/note_content_fulltext.js";
import OrderByAndLimitExp from "../expressions/order_by_and_limit.js";
import AncestorExp from "../expressions/ancestor.js";
import buildComparator from "./build_comparator.js";
import ValueExtractor from "../value_extractor.js";
import { removeDiacritic } from "../../utils.js";
import TrueExp from "../expressions/true.js";
import IsHiddenExp from "../expressions/is_hidden.js";
import type SearchContext from "../search_context.js";
import type { TokenData, TokenStructure } from "./types.js";
import type Expression from "../expressions/expression.js";
function getFulltext(_tokens: TokenData[], searchContext: SearchContext, leadingOperator?: string) {
const tokens: string[] = _tokens.map((t) => removeDiacritic(t.token));
searchContext.highlightedTokens.push(...tokens);
if (tokens.length === 0) {
return null;
}
// If user specified "=" at the beginning, they want exact match
const operator = leadingOperator === "=" ? "=" : "*=*";
if (!searchContext.fastSearch) {
// For exact match with "=", we need different behavior
if (leadingOperator === "=" && tokens.length >= 1) {
// Exact match on title OR exact match on content OR exact match in flat text (includes attributes)
// For multi-word, join tokens with space to form exact phrase
const titleSearchValue = tokens.join(" ");
return new OrExp([
new PropertyComparisonExp(searchContext, "title", "=", titleSearchValue),
new NoteContentFulltextExp("=", { tokens, flatText: false }),
new NoteContentFulltextExp("=", { tokens, flatText: true })
]);
}
return new OrExp([new NoteFlatTextExp(tokens), new NoteContentFulltextExp(operator, { tokens, flatText: true })]);
} else {
return new NoteFlatTextExp(tokens);
}
}
const OPERATORS = new Set(["=", "!=", "*=*", "*=", "=*", ">", ">=", "<", "<=", "%=", "~=", "~*"]);
function isOperator(token: TokenData) {
if (Array.isArray(token)) {
return false;
}
return OPERATORS.has(token.token);
}
function getExpression(tokens: TokenData[], searchContext: SearchContext, level = 0) {
if (tokens.length === 0) {
return null;
}
const expressions: Expression[] = [];
let op: string | null = null;
let i: number;
function context(i: number) {
let { startIndex, endIndex } = tokens[i];
startIndex = Math.max(0, (startIndex || 0) - 20);
endIndex = Math.min(searchContext.originalQuery.length, (endIndex || Number.MAX_SAFE_INTEGER) + 20);
return `"${startIndex !== 0 ? "..." : ""}${searchContext.originalQuery.substr(startIndex, endIndex - startIndex)}${endIndex !== searchContext.originalQuery.length ? "..." : ""}"`;
}
const resolveConstantOperand = () => {
const operand = tokens[i];
if (!operand.inQuotes && (operand.token.startsWith("#") || operand.token.startsWith("~") || operand.token === "note")) {
searchContext.addError(`Error near token "${operand.token}" in ${context(i)}, it's possible to compare with constant only.`);
return null;
}
if (operand.inQuotes || !["now", "today", "month", "year"].includes(operand.token)) {
return operand.token;
}
let delta = 0;
if (i + 2 < tokens.length) {
if (tokens[i + 1].token === "+") {
i += 2;
delta += parseInt(tokens[i].token);
} else if (tokens[i + 1].token === "-") {
i += 2;
delta -= parseInt(tokens[i].token);
}
}
let format, date;
if (operand.token === "now") {
date = dayjs().add(delta, "second");
format = "YYYY-MM-DD HH:mm:ss";
} else if (operand.token === "today") {
date = dayjs().add(delta, "day");
format = "YYYY-MM-DD";
} else if (operand.token === "month") {
date = dayjs().add(delta, "month");
format = "YYYY-MM";
} else if (operand.token === "year") {
date = dayjs().add(delta, "year");
format = "YYYY";
} else {
throw new Error(`Unrecognized keyword: ${operand.token}`);
}
return date.format(format);
};
const parseNoteProperty: () => Expression | undefined | null = () => {
if (tokens[i].token !== ".") {
searchContext.addError('Expected "." to separate field path');
return;
}
i++;
if (["content", "rawcontent"].includes(tokens[i].token)) {
const raw = tokens[i].token === "rawcontent";
i += 1;
const operator = tokens[i];
if (!isOperator(operator)) {
searchContext.addError(`After content expected operator, but got "${operator.token}" in ${context(i)}`);
return;
}
i++;
return new NoteContentFulltextExp(operator.token, { tokens: [tokens[i].token], raw });
}
if (tokens[i].token === "parents") {
i += 1;
const expression = parseNoteProperty();
if (!expression) {
return;
}
return new ChildOfExp(expression);
}
if (tokens[i].token === "children") {
i += 1;
const expression = parseNoteProperty();
if (!expression) {
return;
}
return new ParentOfExp(expression);
}
if (tokens[i].token === "ancestors") {
i += 1;
const expression = parseNoteProperty();
if (!expression) {
return;
}
return new DescendantOfExp(expression);
}
if (tokens[i].token === "labels") {
if (tokens[i + 1].token !== ".") {
searchContext.addError(`Expected "." to separate field path, got "${tokens[i + 1].token}" in ${context(i)}`);
return;
}
i += 2;
return parseLabel(tokens[i].token);
}
if (tokens[i].token === "relations") {
if (tokens[i + 1].token !== ".") {
searchContext.addError(`Expected "." to separate field path, got "${tokens[i + 1].token}" in ${context(i)}`);
return;
}
i += 2;
return parseRelation(tokens[i].token);
}
if (tokens[i].token === "text") {
if (tokens[i + 1].token !== "*=*") {
searchContext.addError(`Virtual attribute "note.text" supports only *=* operator, instead given "${tokens[i + 1].token}" in ${context(i)}`);
return;
}
i += 2;
return new OrExp([new PropertyComparisonExp(searchContext, "title", "*=*", tokens[i].token), new NoteContentFulltextExp("*=*", { tokens: [tokens[i].token] })]);
}
if (PropertyComparisonExp.isProperty(tokens[i].token)) {
const propertyName = tokens[i].token;
const operator = tokens[i + 1].token;
i += 2;
const comparedValue = resolveConstantOperand();
if (!comparedValue) {
searchContext.addError(`Unresolved constant operand.`);
return;
}
return new PropertyComparisonExp(searchContext, propertyName, operator, comparedValue);
}
searchContext.addError(`Unrecognized note property "${tokens[i].token}" in ${context(i)}`);
};
function parseAttribute(name: string) {
const isLabel = name.startsWith("#");
name = name.substr(1);
const isNegated = name.startsWith("!");
if (isNegated) {
name = name.substr(1);
}
const subExp = isLabel ? parseLabel(name) : parseRelation(name);
return subExp && isNegated ? new NotExp(subExp) : subExp;
}
function parseLabel(labelName: string) {
searchContext.highlightedTokens.push(labelName);
if (i < tokens.length - 2 && isOperator(tokens[i + 1])) {
let operator = tokens[i + 1].token;
i += 2;
const comparedValue = resolveConstantOperand();
if (comparedValue === null) {
return;
}
searchContext.highlightedTokens.push(comparedValue);
if (searchContext.fuzzyAttributeSearch && operator === "=") {
operator = "*=*";
}
const comparator = buildComparator(operator, comparedValue);
if (!comparator) {
searchContext.addError(`Can't find operator '${operator}' in ${context(i - 1)}`);
} else {
return new LabelComparisonExp("label", labelName, comparator);
}
} else {
return new AttributeExistsExp("label", labelName, searchContext.fuzzyAttributeSearch);
}
}
function parseRelation(relationName: string) {
searchContext.highlightedTokens.push(relationName);
if (i < tokens.length - 2 && tokens[i + 1].token === ".") {
i += 1;
const expression = parseNoteProperty();
if (!expression) {
return;
}
return new RelationWhereExp(relationName, expression);
} else if (i < tokens.length - 2 && isOperator(tokens[i + 1])) {
searchContext.addError(`Relation can be compared only with property, e.g. ~relation.title=hello in ${context(i)}`);
return null;
} else {
return new AttributeExistsExp("relation", relationName, searchContext.fuzzyAttributeSearch);
}
}
function parseOrderByAndLimit() {
const orderDefinitions: {
valueExtractor: ValueExtractor;
direction: string;
}[] = [];
let limit: number | undefined = undefined;
if (tokens[i].token === "orderby") {
do {
const propertyPath: string[] = [];
let direction = "asc";
do {
i++;
propertyPath.push(tokens[i].token);
i++;
} while (i < tokens.length && tokens[i].token === ".");
if (i < tokens.length && ["asc", "desc"].includes(tokens[i].token)) {
direction = tokens[i].token;
i++;
}
const valueExtractor = new ValueExtractor(searchContext, propertyPath);
const validationError = valueExtractor.validate();
if (validationError) {
searchContext.addError(validationError);
}
orderDefinitions.push({
valueExtractor,
direction
});
} while (i < tokens.length && tokens[i].token === ",");
}
if (i < tokens.length && tokens[i].token === "limit") {
limit = parseInt(tokens[i + 1].token);
}
return new OrderByAndLimitExp(orderDefinitions, limit);
}
function getAggregateExpression() {
if (op === null || op === "and") {
return AndExp.of(expressions);
} else if (op === "or") {
return OrExp.of(expressions);
} else {
throw new Error(`Unrecognized op=${op}`);
}
}
for (i = 0; i < tokens.length; i++) {
if (Array.isArray(tokens[i])) {
const expression = getExpression(tokens[i] as unknown as TokenData[], searchContext, level + 1);
if (expression) {
expressions.push(expression);
}
continue;
}
const token = tokens[i].token;
if (token === "#" || token === "~") {
continue;
}
if (token.startsWith("#") || token.startsWith("~")) {
const attribute = parseAttribute(token);
if (attribute) {
expressions.push(attribute);
}
} else if (["orderby", "limit"].includes(token)) {
if (level !== 0) {
searchContext.addError("orderBy can appear only on the top expression level");
continue;
}
const exp = parseOrderByAndLimit();
if (!exp) {
continue;
}
exp.subExpression = getAggregateExpression();
return exp;
} else if (token === "not") {
i += 1;
if (!Array.isArray(tokens[i])) {
searchContext.addError(`not keyword should be followed by sub-expression in parenthesis, got ${tokens[i].token} instead`);
continue;
}
const tokenArray = tokens[i] as unknown as TokenData[];
const expression = getExpression(tokenArray, searchContext, level + 1);
if (!expression) {
return;
}
expressions.push(new NotExp(expression));
} else if (token === "note") {
i++;
const expression = parseNoteProperty();
if (!expression) {
return;
}
expressions.push(expression);
continue;
} else if (["and", "or"].includes(token)) {
if (!op) {
op = token;
} else if (op !== token) {
searchContext.addError("Mixed usage of AND/OR - always use parenthesis to group AND/OR expressions.");
}
} else if (isOperator({ token: token })) {
searchContext.addError(`Misplaced or incomplete expression "${token}"`);
} else {
searchContext.addError(`Unrecognized expression "${token}"`);
}
if (!op && expressions.length > 1) {
op = "and";
}
}
return getAggregateExpression();
}
export interface ParseOpts {
fulltextTokens: TokenData[];
expressionTokens: TokenStructure;
searchContext: SearchContext;
originalQuery?: string;
leadingOperator?: string;
}
function parse({ fulltextTokens, expressionTokens, searchContext, leadingOperator }: ParseOpts) {
let expression: Expression | undefined | null;
try {
expression = getExpression(expressionTokens as TokenData[], searchContext);
} catch (e: any) {
searchContext.addError(e.message);
expression = new TrueExp();
}
let exp = AndExp.of([
searchContext.includeArchivedNotes ? null : new PropertyComparisonExp(searchContext, "isarchived", "=", "false"),
getAncestorExp(searchContext),
getFulltext(fulltextTokens, searchContext, leadingOperator),
expression
]);
if (searchContext.limit && !searchContext.orderBy) {
const filterExp = exp;
exp = new OrderByAndLimitExp([], searchContext.limit || undefined);
(exp as any).subExpression = filterExp;
}
if (searchContext.orderBy && searchContext.orderBy !== "relevancy") {
const filterExp = exp;
exp = new OrderByAndLimitExp(
[
{
valueExtractor: new ValueExtractor(searchContext, ["note", searchContext.orderBy]),
direction: searchContext.orderDirection
}
],
searchContext.limit || undefined
);
(exp as any).subExpression = filterExp;
}
return exp;
}
function getAncestorExp({ ancestorNoteId, ancestorDepth, includeHiddenNotes }: SearchContext) {
if (ancestorNoteId && ancestorNoteId !== "root") {
return new AncestorExp(ancestorNoteId, ancestorDepth);
} else if (!includeHiddenNotes) {
return new NotExp(new IsHiddenExp());
} else {
return null;
}
}
export default parse;

241
apps/server/src/services/search/services/progressive_search.spec.ts
View File

@@ -1,241 +0,0 @@
import { describe, it, expect, beforeEach } from "vitest";
import searchService from "./search.js";
import BNote from "../../../becca/entities/bnote.js";
import BBranch from "../../../becca/entities/bbranch.js";
import SearchContext from "../search_context.js";
import becca from "../../../becca/becca.js";
import { findNoteByTitle, note, NoteBuilder } from "../../../test/becca_mocking.js";
describe("Progressive Search Strategy", () => {
let rootNote: any;
beforeEach(() => {
becca.reset();
rootNote = new NoteBuilder(new BNote({ noteId: "root", title: "root", type: "text" }));
new BBranch({
branchId: "none_root",
noteId: "root",
parentNoteId: "none",
notePosition: 10
});
});
describe("Phase 1: Exact Matches Only", () => {
it("should complete search with exact matches when sufficient results found", () => {
// Create notes with exact matches
rootNote
.child(note("Document Analysis One"))
.child(note("Document Report Two"))
.child(note("Document Review Three"))
.child(note("Document Summary Four"))
.child(note("Document Overview Five"))
.child(note("Documnt Analysis Six")); // This has a typo that should require fuzzy matching
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("document", searchContext);
// Should find 5 exact matches and not need fuzzy matching
expect(searchResults.length).toEqual(5);
// Verify all results have high scores (exact matches)
const highQualityResults = searchResults.filter(result => result.score >= 10);
expect(highQualityResults.length).toEqual(5);
// The typo document should not be in results since we have enough exact matches
expect(findNoteByTitle(searchResults, "Documnt Analysis Six")).toBeFalsy();
});
it("should use exact match scoring only in Phase 1", () => {
rootNote
.child(note("Testing Exact Match"))
.child(note("Test Document"))
.child(note("Another Test"));
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("test", searchContext);
// All results should have scores from exact matching only
for (const result of searchResults) {
expect(result.score).toBeGreaterThan(0);
// Scores should be from exact/prefix/contains matches, not fuzzy
expect(result.score % 0.5).not.toBe(0); // Fuzzy scores are multiples of 0.5
}
});
});
describe("Phase 2: Fuzzy Fallback", () => {
it("should trigger fuzzy matching when insufficient exact matches", () => {
// Create only a few notes, some with typos
rootNote
.child(note("Document One"))
.child(note("Report Two"))
.child(note("Anaylsis Three")) // Typo: "Analysis"
.child(note("Sumary Four")); // Typo: "Summary"
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("analysis", searchContext);
// Should find the typo through fuzzy matching
expect(searchResults.length).toBeGreaterThan(0);
expect(findNoteByTitle(searchResults, "Anaylsis Three")).toBeTruthy();
});
it("should merge exact and fuzzy results with exact matches always ranked higher", () => {
rootNote
.child(note("Analysis Report")) // Exact match
.child(note("Data Analysis")) // Exact match
.child(note("Anaylsis Doc")) // Fuzzy match
.child(note("Statistical Anlaysis")); // Fuzzy match
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("analysis", searchContext);
expect(searchResults.length).toBe(4);
// Get the note titles in result order
const resultTitles = searchResults.map(r => becca.notes[r.noteId].title);
// Find positions of exact and fuzzy matches
const exactPositions = resultTitles.map((title, index) =>
title.toLowerCase().includes("analysis") ? index : -1
).filter(pos => pos !== -1);
const fuzzyPositions = resultTitles.map((title, index) =>
(title.includes("Anaylsis") || title.includes("Anlaysis")) ? index : -1
).filter(pos => pos !== -1);
expect(exactPositions.length).toBe(2);
expect(fuzzyPositions.length).toBe(2);
// CRITICAL: All exact matches must come before all fuzzy matches
const lastExactPosition = Math.max(...exactPositions);
const firstFuzzyPosition = Math.min(...fuzzyPositions);
expect(lastExactPosition).toBeLessThan(firstFuzzyPosition);
});
it("should not duplicate results between phases", () => {
rootNote
.child(note("Test Document")) // Would match in both phases
.child(note("Tset Report")); // Only fuzzy match
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("test", searchContext);
// Should only have unique results
const noteIds = searchResults.map(r => r.noteId);
const uniqueNoteIds = [...new Set(noteIds)];
expect(noteIds.length).toBe(uniqueNoteIds.length);
expect(findNoteByTitle(searchResults, "Test Document")).toBeTruthy();
expect(findNoteByTitle(searchResults, "Tset Report")).toBeTruthy();
});
});
describe("Result Sufficiency Thresholds", () => {
it("should respect minimum result count threshold", () => {
// Create exactly 4 high-quality results (below threshold of 5)
rootNote
.child(note("Test One"))
.child(note("Test Two"))
.child(note("Test Three"))
.child(note("Test Four"))
.child(note("Tset Five")); // Typo that should be found via fuzzy
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("test", searchContext);
// Should proceed to Phase 2 and include fuzzy match
expect(searchResults.length).toBe(5);
expect(findNoteByTitle(searchResults, "Tset Five")).toBeTruthy();
});
it("should respect minimum quality score threshold", () => {
// Create notes that might have low exact match scores
rootNote
.child(note("Testing Document")) // Should have decent score
.child(note("Document with test inside")) // Lower score due to position
.child(note("Another test case"))
.child(note("Test case example"))
.child(note("Tset with typo")); // Fuzzy match
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("test", searchContext);
// Should include fuzzy results if exact results don't meet quality threshold
expect(searchResults.length).toBeGreaterThan(4);
});
});
describe("Fuzzy Score Management", () => {
it("should cap fuzzy token scores to prevent outranking exact matches", () => {
// Create note with exact match
rootNote.child(note("Test Document"));
// Create note that could accumulate high fuzzy scores
rootNote.child(note("Tset Documnt with many fuzzy tockens for testng")); // Multiple typos
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("test document", searchContext);
expect(searchResults.length).toBe(2);
// Find the exact and fuzzy match results
const exactResult = searchResults.find(r => becca.notes[r.noteId].title === "Test Document");
const fuzzyResult = searchResults.find(r => becca.notes[r.noteId].title.includes("Tset"));
expect(exactResult).toBeTruthy();
expect(fuzzyResult).toBeTruthy();
// Exact match should always score higher than fuzzy, even with multiple fuzzy matches
expect(exactResult!.score).toBeGreaterThan(fuzzyResult!.score);
});
it("should enforce maximum total fuzzy score per search", () => {
// Create note with many potential fuzzy matches
rootNote.child(note("Tset Documnt Anaylsis Sumary Reportng")); // Many typos
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("test document analysis summary reporting", searchContext);
expect(searchResults.length).toBe(1);
// Total score should be bounded despite many fuzzy matches
expect(searchResults[0].score).toBeLessThan(500); // Should not exceed reasonable bounds due to caps
});
});
describe("SearchContext Integration", () => {
it("should respect enableFuzzyMatching flag", () => {
rootNote
.child(note("Test Document"))
.child(note("Tset Report")); // Typo
// Test with fuzzy matching disabled
const exactOnlyContext = new SearchContext();
exactOnlyContext.enableFuzzyMatching = false;
const exactResults = searchService.findResultsWithQuery("test", exactOnlyContext);
expect(exactResults.length).toBe(1);
expect(findNoteByTitle(exactResults, "Test Document")).toBeTruthy();
expect(findNoteByTitle(exactResults, "Tset Report")).toBeFalsy();
// Test with fuzzy matching enabled (default)
const fuzzyContext = new SearchContext();
const fuzzyResults = searchService.findResultsWithQuery("test", fuzzyContext);
expect(fuzzyResults.length).toBe(2);
expect(findNoteByTitle(fuzzyResults, "Tset Report")).toBeTruthy();
});
});
describe("Edge Cases", () => {
it("should handle empty search results gracefully", () => {
rootNote.child(note("Unrelated Content"));
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("nonexistent", searchContext);
expect(searchResults.length).toBe(0);
});
});
});

796
apps/server/src/services/search/services/search.spec.ts
View File

@@ -1,796 +0,0 @@
import { describe, it, expect, beforeEach, } from "vitest";
import searchService from "./search.js";
import BNote from "../../../becca/entities/bnote.js";
import BBranch from "../../../becca/entities/bbranch.js";
import SearchContext from "../search_context.js";
import dateUtils from "../../date_utils.js";
import becca from "../../../becca/becca.js";
import { findNoteByTitle, note, NoteBuilder } from "../../../test/becca_mocking.js";
describe("Search", () => {
let rootNote: any;
beforeEach(() => {
becca.reset();
rootNote = new NoteBuilder(new BNote({ noteId: "root", title: "root", type: "text" }));
new BBranch({
branchId: "none_root",
noteId: "root",
parentNoteId: "none",
notePosition: 10
});
});
it("simple path match", () => {
rootNote.child(note("Europe").child(note("Austria")));
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("europe austria", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy();
});
it("normal search looks also at attributes", () => {
const austria = note("Austria");
const vienna = note("Vienna");
rootNote.child(austria.relation("capital", vienna.note)).child(vienna.label("inhabitants", "1888776"));
const searchContext = new SearchContext();
let searchResults = searchService.findResultsWithQuery("capital", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy();
searchResults = searchService.findResultsWithQuery("inhabitants", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Vienna")).toBeTruthy();
});
it("normal search looks also at type and mime", () => {
rootNote.child(note("Effective Java", { type: "book", mime: "" })).child(note("Hello World.java", { type: "code", mime: "text/x-java" }));
const searchContext = new SearchContext();
let searchResults = searchService.findResultsWithQuery("book", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Effective Java")).toBeTruthy();
searchResults = searchService.findResultsWithQuery("text", searchContext); // should match mime
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Hello World.java")).toBeTruthy();
searchResults = searchService.findResultsWithQuery("java", searchContext);
expect(searchResults.length).toEqual(2);
});
it("only end leafs are results", () => {
rootNote.child(note("Europe").child(note("Austria")));
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("europe", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Europe")).toBeTruthy();
});
it("only end leafs are results", () => {
rootNote.child(note("Europe").child(note("Austria").label("capital", "Vienna")));
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("Vienna", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy();
});
it("label comparison with short syntax", () => {
rootNote.child(note("Europe").child(note("Austria").label("capital", "Vienna")).child(note("Czech Republic").label("capital", "Prague")));
const searchContext = new SearchContext();
let searchResults = searchService.findResultsWithQuery("#capital=Vienna", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy();
// case sensitivity:
searchResults = searchService.findResultsWithQuery("#CAPITAL=VIENNA", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy();
searchResults = searchService.findResultsWithQuery("#caPItal=vienNa", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy();
});
it("label comparison with full syntax", () => {
rootNote.child(note("Europe").child(note("Austria").label("capital", "Vienna")).child(note("Czech Republic").label("capital", "Prague")));
const searchContext = new SearchContext();
let searchResults = searchService.findResultsWithQuery("# note.labels.capital=Prague", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy();
});
it("numeric label comparison", () => {
rootNote.child(note("Europe")
.label("country", "", true)
.child(note("Austria").label("population", "8859000"))
.child(note("Czech Republic").label("population", "10650000"))
);
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("#country #population >= 10000000", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy();
});
it("inherited label comparison", () => {
rootNote.child(note("Europe").label("country", "", true).child(note("Austria")).child(note("Czech Republic")));
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("austria #country", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy();
});
it("numeric label comparison fallback to string comparison", () => {
// dates should not be coerced into numbers which would then give wrong numbers
rootNote.child(note("Europe")
.label("country", "", true)
.child(note("Austria").label("established", "1955-07-27"))
.child(note("Czech Republic").label("established", "1993-01-01"))
.child(note("Hungary").label("established", "1920-06-04"))
);
const searchContext = new SearchContext();
let searchResults = searchService.findResultsWithQuery('#established <= "1955-01-01"', searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Hungary")).toBeTruthy();
searchResults = searchService.findResultsWithQuery('#established > "1955-01-01"', searchContext);
expect(searchResults.length).toEqual(2);
expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy();
expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy();
});
it("smart date comparisons", () => {
// dates should not be coerced into numbers which would then give wrong numbers
rootNote.child(note("My note", { dateCreated: dateUtils.localNowDateTime() })
.label("year", new Date().getFullYear().toString())
.label("month", dateUtils.localNowDate().substr(0, 7))
.label("date", dateUtils.localNowDate())
.label("dateTime", dateUtils.localNowDateTime())
);
const searchContext = new SearchContext();
function test(query: string, expectedResultCount: number) {
const searchResults = searchService.findResultsWithQuery(query, searchContext);
expect(searchResults.length, `Searching for '${query}' unexpectedly returned ${Number(searchResults?.length)} instead of ${expectedResultCount} results. SearchResult: '${JSON.stringify(searchResults)}'`)
.toEqual(expectedResultCount);
if (expectedResultCount === 1) {
expect(findNoteByTitle(searchResults, "My note")).toBeTruthy();
}
}
test("#year = YEAR", 1);
test("#year = 'YEAR'", 0);
test("#year >= YEAR", 1);
test("#year <= YEAR", 1);
test("#year < YEAR+1", 1);
test("#year < YEAR + 1", 1);
test("#year < year + 1", 1);
test("#year > YEAR+1", 0);
test("#month = MONTH", 1);
test("#month = month", 1);
test("#month = 'MONTH'", 0);
test("note.dateCreated =* month", 2);
test("#date = TODAY", 1);
test("#date = today", 1);
test("#date = 'today'", 0);
test("#date > TODAY", 0);
test("#date > TODAY-1", 1);
test("#date > TODAY - 1", 1);
test("#date < TODAY+1", 1);
test("#date < TODAY + 1", 1);
test("#date < 'TODAY + 1'", 1);
test("#dateTime <= NOW+10", 1);
test("#dateTime <= NOW + 10", 1);
test("#dateTime < NOW-10", 0);
test("#dateTime >= NOW-10", 1);
test("#dateTime < NOW-10", 0);
});
it("logical or", () => {
rootNote.child(note("Europe")
.label("country", "", true)
.child(note("Austria").label("languageFamily", "germanic"))
.child(note("Czech Republic").label("languageFamily", "slavic"))
.child(note("Hungary").label("languageFamily", "finnougric"))
);
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("#languageFamily = slavic OR #languageFamily = germanic", searchContext);
expect(searchResults.length).toEqual(2);
expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy();
expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy();
});
it("leading = operator for exact match", () => {
rootNote
.child(note("Example Note").label("type", "document"))
.child(note("Examples of Usage").label("type", "tutorial"))
.child(note("Sample").label("type", "example"));
const searchContext = new SearchContext();
// Using leading = for exact word match - should find notes containing the exact word "example"
let searchResults = searchService.findResultsWithQuery("=example", searchContext);
expect(searchResults.length).toEqual(2); // "Example Note" and "Sample" (has label "example")
expect(findNoteByTitle(searchResults, "Example Note")).toBeTruthy();
expect(findNoteByTitle(searchResults, "Sample")).toBeTruthy();
// Without =, it should find all notes containing "example" (substring match)
searchResults = searchService.findResultsWithQuery("example", searchContext);
expect(searchResults.length).toEqual(3); // All notes
// = operator should not match partial words
searchResults = searchService.findResultsWithQuery("=examples", searchContext);
expect(searchResults.length).toEqual(1); // Only "Examples of Usage"
expect(findNoteByTitle(searchResults, "Examples of Usage")).toBeTruthy();
});
it("leading = operator for exact match - comprehensive title tests", () => {
// Create notes with varying titles to test exact vs contains matching
rootNote
.child(note("testing"))
.child(note("testing123"))
.child(note("My testing notes"))
.child(note("123testing"))
.child(note("test"));
const searchContext = new SearchContext();
// Test 1: Exact word match with leading = should find notes containing the exact word "testing"
let searchResults = searchService.findResultsWithQuery("=testing", searchContext);
expect(searchResults.length).toEqual(2); // "testing" and "My testing notes" (word boundary)
expect(findNoteByTitle(searchResults, "testing")).toBeTruthy();
expect(findNoteByTitle(searchResults, "My testing notes")).toBeTruthy();
// Test 2: Without =, it should find all notes containing "testing" (substring contains behavior)
searchResults = searchService.findResultsWithQuery("testing", searchContext);
expect(searchResults.length).toEqual(4); // All notes with "testing" substring
// Test 3: Exact match should only find the exact composite word
searchResults = searchService.findResultsWithQuery("=testing123", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "testing123")).toBeTruthy();
// Test 4: Exact match should only find the exact composite word
searchResults = searchService.findResultsWithQuery("=123testing", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "123testing")).toBeTruthy();
// Test 5: Verify that "test" doesn't match "testing" with exact search
searchResults = searchService.findResultsWithQuery("=test", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "test")).toBeTruthy();
});
it("leading = operator with quoted phrases", () => {
rootNote
.child(note("exact phrase"))
.child(note("exact phrase match"))
.child(note("this exact phrase here"))
.child(note("phrase exact"));
const searchContext = new SearchContext();
// Test 1: With = and quotes, treat as exact phrase match (consecutive words in order)
let searchResults = searchService.findResultsWithQuery("='exact phrase'", searchContext);
// Should match only notes containing the exact phrase "exact phrase"
expect(searchResults.length).toEqual(3); // Only notes with consecutive "exact phrase"
expect(findNoteByTitle(searchResults, "exact phrase")).toBeTruthy();
expect(findNoteByTitle(searchResults, "exact phrase match")).toBeTruthy();
expect(findNoteByTitle(searchResults, "this exact phrase here")).toBeTruthy();
// Test 2: Without =, quoted phrase should find substring/contains matches
searchResults = searchService.findResultsWithQuery("'exact phrase'", searchContext);
expect(searchResults.length).toEqual(3); // All notes containing the phrase substring
expect(findNoteByTitle(searchResults, "exact phrase")).toBeTruthy();
expect(findNoteByTitle(searchResults, "exact phrase match")).toBeTruthy();
expect(findNoteByTitle(searchResults, "this exact phrase here")).toBeTruthy();
// Test 3: Verify word order matters with exact phrase matching
searchResults = searchService.findResultsWithQuery("='phrase exact'", searchContext);
expect(searchResults.length).toEqual(1); // Only "phrase exact" matches
expect(findNoteByTitle(searchResults, "phrase exact")).toBeTruthy();
});
it("leading = operator case sensitivity", () => {
rootNote
.child(note("TESTING"))
.child(note("testing"))
.child(note("Testing"))
.child(note("TeStiNg"));
const searchContext = new SearchContext();
// Exact match should be case-insensitive (based on lex.ts line 4: str.toLowerCase())
let searchResults = searchService.findResultsWithQuery("=testing", searchContext);
expect(searchResults.length).toEqual(4); // All variants of "testing"
searchResults = searchService.findResultsWithQuery("=TESTING", searchContext);
expect(searchResults.length).toEqual(4); // All variants
searchResults = searchService.findResultsWithQuery("=Testing", searchContext);
expect(searchResults.length).toEqual(4); // All variants
searchResults = searchService.findResultsWithQuery("=TeStiNg", searchContext);
expect(searchResults.length).toEqual(4); // All variants
});
it("leading = operator with special characters", () => {
rootNote
.child(note("test-note"))
.child(note("test_note"))
.child(note("test.note"))
.child(note("test note"))
.child(note("testnote"));
const searchContext = new SearchContext();
// Each exact match should only find its specific variant (compound words are treated as single words)
let searchResults = searchService.findResultsWithQuery("=test-note", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "test-note")).toBeTruthy();
searchResults = searchService.findResultsWithQuery("=test_note", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "test_note")).toBeTruthy();
searchResults = searchService.findResultsWithQuery("=test.note", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "test.note")).toBeTruthy();
// For phrases with spaces, use quotes to keep them together
// With exact phrase matching, this finds notes with the consecutive phrase
searchResults = searchService.findResultsWithQuery("='test note'", searchContext);
expect(searchResults.length).toEqual(1); // Only "test note" has the exact phrase
expect(findNoteByTitle(searchResults, "test note")).toBeTruthy();
// Without quotes, "test note" is tokenized as two separate tokens
// and will be treated as an exact phrase search with = operator
searchResults = searchService.findResultsWithQuery("=test note", searchContext);
expect(searchResults.length).toEqual(1); // Only "test note" has the exact phrase
// Without =, should find all matches containing "test" substring
searchResults = searchService.findResultsWithQuery("test", searchContext);
expect(searchResults.length).toEqual(5);
});
it("fuzzy attribute search", () => {
rootNote.child(note("Europe")
.label("country", "", true)
.child(note("Austria").label("languageFamily", "germanic"))
.child(note("Czech Republic").label("languageFamily", "slavic"))
);
let searchContext = new SearchContext({ fuzzyAttributeSearch: false });
let searchResults = searchService.findResultsWithQuery("#language", searchContext);
expect(searchResults.length).toEqual(0);
searchResults = searchService.findResultsWithQuery("#languageFamily=ger", searchContext);
expect(searchResults.length).toEqual(0);
searchContext = new SearchContext({ fuzzyAttributeSearch: true });
searchResults = searchService.findResultsWithQuery("#language", searchContext);
expect(searchResults.length).toEqual(2);
searchResults = searchService.findResultsWithQuery("#languageFamily=ger", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy();
});
it("filter by note property", () => {
rootNote.child(note("Europe").child(note("Austria")).child(note("Czech Republic")));
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("# note.title =* czech", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy();
});
it("filter by note's parent", () => {
rootNote
.child(note("Europe")
.child(note("Austria"))
.child(note("Czech Republic").child(note("Prague")))
)
.child(note("Asia").child(note("Taiwan")));
const searchContext = new SearchContext();
let searchResults = searchService.findResultsWithQuery("# note.parents.title = Europe", searchContext);
expect(searchResults.length).toEqual(2);
expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy();
expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy();
searchResults = searchService.findResultsWithQuery("# note.parents.title = Asia", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Taiwan")).toBeTruthy();
searchResults = searchService.findResultsWithQuery("# note.parents.parents.title = Europe", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Prague")).toBeTruthy();
});
it("filter by note's ancestor", () => {
rootNote
.child(note("Europe")
.child(note("Austria"))
.child(note("Czech Republic").child(note("Prague").label("city")))
)
.child(note("Asia").child(note("Taiwan").child(note("Taipei").label("city"))));
const searchContext = new SearchContext();
let searchResults = searchService.findResultsWithQuery("#city AND note.ancestors.title = Europe", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Prague")).toBeTruthy();
searchResults = searchService.findResultsWithQuery("#city AND note.ancestors.title = Asia", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Taipei")).toBeTruthy();
});
it("filter by note's child", () => {
rootNote
.child(note("Europe")
.child(note("Austria").child(note("Vienna")))
.child(note("Czech Republic").child(note("Prague")))
)
.child(note("Oceania").child(note("Australia")));
const searchContext = new SearchContext();
let searchResults = searchService.findResultsWithQuery("# note.children.title =* Aust", searchContext);
expect(searchResults.length).toEqual(2);
expect(findNoteByTitle(searchResults, "Europe")).toBeTruthy();
expect(findNoteByTitle(searchResults, "Oceania")).toBeTruthy();
searchResults = searchService.findResultsWithQuery("# note.children.title =* Aust AND note.children.title *= republic", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Europe")).toBeTruthy();
searchResults = searchService.findResultsWithQuery("# note.children.children.title = Prague", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Europe")).toBeTruthy();
});
it("filter by relation's note properties using short syntax", () => {
const austria = note("Austria");
const portugal = note("Portugal");
rootNote.child(note("Europe")
.child(austria)
.child(note("Czech Republic").relation("neighbor", austria.note))
.child(portugal)
.child(note("Spain").relation("neighbor", portugal.note))
);
const searchContext = new SearchContext();
let searchResults = searchService.findResultsWithQuery("# ~neighbor.title = Austria", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy();
searchResults = searchService.findResultsWithQuery("# ~neighbor.title = Portugal", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Spain")).toBeTruthy();
});
it("filter by relation's note properties using long syntax", () => {
const austria = note("Austria");
const portugal = note("Portugal");
rootNote.child(note("Europe")
.child(austria)
.child(note("Czech Republic").relation("neighbor", austria.note))
.child(portugal)
.child(note("Spain").relation("neighbor", portugal.note))
);
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("# note.relations.neighbor.title = Austria", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy();
});
it("filter by multiple level relation", () => {
const austria = note("Austria");
const slovakia = note("Slovakia");
const italy = note("Italy");
const ukraine = note("Ukraine");
rootNote.child(note("Europe")
.child(austria.relation("neighbor", italy.note).relation("neighbor", slovakia.note))
.child(note("Czech Republic").relation("neighbor", austria.note).relation("neighbor", slovakia.note))
.child(slovakia.relation("neighbor", ukraine.note))
.child(ukraine)
);
const searchContext = new SearchContext();
let searchResults = searchService.findResultsWithQuery("# note.relations.neighbor.relations.neighbor.title = Italy", searchContext);
expect(searchResults.length).toEqual(1);
expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy();
searchResults = searchService.findResultsWithQuery("# note.relations.neighbor.relations.neighbor.title = Ukraine", searchContext);
expect(searchResults.length).toEqual(2);
expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy();
expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy();
});
it("test note properties", () => {
const austria = note("Austria");
austria.relation("myself", austria.note);
austria.label("capital", "Vienna");
austria.label("population", "8859000");
rootNote
.child(note("Asia"))
.child(note("Europe").child(austria.child(note("Vienna")).child(note("Sebastian Kurz"))))
.child(note("Mozart").child(austria));
austria.note.isProtected = false;
austria.note.dateCreated = "2020-05-14 12:11:42.001+0200";
austria.note.dateModified = "2020-05-14 13:11:42.001+0200";
austria.note.utcDateCreated = "2020-05-14 10:11:42.001Z";
austria.note.utcDateModified = "2020-05-14 11:11:42.001Z";
// austria.note.contentLength = 1001;
const searchContext = new SearchContext();
function test(propertyName: string, value: string, expectedResultCount: number) {
const searchResults = searchService.findResultsWithQuery(`# note.${propertyName} = ${value}`, searchContext);
expect(searchResults.length).toEqual(expectedResultCount);
}
test("type", "text", 7);
test("TYPE", "TEXT", 7);
test("type", "code", 0);
test("mime", "text/html", 6);
test("mime", "application/json", 0);
test("isProtected", "false", 7);
test("isProtected", "FALSE", 7);
test("isProtected", "true", 0);
test("isProtected", "TRUE", 0);
test("dateCreated", "'2020-05-14 12:11:42.001+0200'", 1);
test("dateCreated", "wrong", 0);
test("dateModified", "'2020-05-14 13:11:42.001+0200'", 1);
test("dateModified", "wrong", 0);
test("utcDateCreated", "'2020-05-14 10:11:42.001Z'", 1);
test("utcDateCreated", "wrong", 0);
test("utcDateModified", "'2020-05-14 11:11:42.001Z'", 1);
test("utcDateModified", "wrong", 0);
test("parentCount", "2", 1);
test("parentCount", "3", 0);
test("childrenCount", "2", 1);
test("childrenCount", "10", 0);
test("attributeCount", "3", 1);
test("attributeCount", "4", 0);
test("labelCount", "2", 1);
test("labelCount", "3", 0);
test("relationCount", "1", 1);
test("relationCount", "2", 0);
});
it("test order by", () => {
const italy = note("Italy").label("capital", "Rome");
const slovakia = note("Slovakia").label("capital", "Bratislava");
const austria = note("Austria").label("capital", "Vienna");
const ukraine = note("Ukraine").label("capital", "Kiev");
rootNote.child(note("Europe").child(ukraine).child(slovakia).child(austria).child(italy));
const searchContext = new SearchContext();
let searchResults = searchService.findResultsWithQuery("# note.parents.title = Europe orderBy note.title", searchContext);
expect(searchResults.length).toEqual(4);
expect(becca.notes[searchResults[0].noteId].title).toEqual("Austria");
expect(becca.notes[searchResults[1].noteId].title).toEqual("Italy");
expect(becca.notes[searchResults[2].noteId].title).toEqual("Slovakia");
expect(becca.notes[searchResults[3].noteId].title).toEqual("Ukraine");
searchResults = searchService.findResultsWithQuery("# note.parents.title = Europe orderBy note.labels.capital", searchContext);
expect(searchResults.length).toEqual(4);
expect(becca.notes[searchResults[0].noteId].title).toEqual("Slovakia");
expect(becca.notes[searchResults[1].noteId].title).toEqual("Ukraine");
expect(becca.notes[searchResults[2].noteId].title).toEqual("Italy");
expect(becca.notes[searchResults[3].noteId].title).toEqual("Austria");
searchResults = searchService.findResultsWithQuery("# note.parents.title = Europe orderBy note.labels.capital DESC", searchContext);
expect(searchResults.length).toEqual(4);
expect(becca.notes[searchResults[0].noteId].title).toEqual("Austria");
expect(becca.notes[searchResults[1].noteId].title).toEqual("Italy");
expect(becca.notes[searchResults[2].noteId].title).toEqual("Ukraine");
expect(becca.notes[searchResults[3].noteId].title).toEqual("Slovakia");
searchResults = searchService.findResultsWithQuery("# note.parents.title = Europe orderBy note.labels.capital DESC limit 2", searchContext);
expect(searchResults.length).toEqual(2);
expect(becca.notes[searchResults[0].noteId].title).toEqual("Austria");
expect(becca.notes[searchResults[1].noteId].title).toEqual("Italy");
searchResults = searchService.findResultsWithQuery("# note.parents.title = Europe orderBy #capital DESC limit 1", searchContext);
expect(searchResults.length).toEqual(1);
searchResults = searchService.findResultsWithQuery("# note.parents.title = Europe orderBy #capital DESC limit 1000", searchContext);
expect(searchResults.length).toEqual(4);
});
it("test not(...)", () => {
const italy = note("Italy").label("capital", "Rome");
const slovakia = note("Slovakia").label("capital", "Bratislava");
rootNote.child(note("Europe").child(slovakia).child(italy));
const searchContext = new SearchContext();
let searchResults = searchService.findResultsWithQuery("# not(#capital) and note.noteId != root", searchContext);
expect(searchResults.length).toEqual(1);
expect(becca.notes[searchResults[0].noteId].title).toEqual("Europe");
searchResults = searchService.findResultsWithQuery("#!capital and note.noteId != root", searchContext);
expect(searchResults.length).toEqual(1);
expect(becca.notes[searchResults[0].noteId].title).toEqual("Europe");
});
it("test note.text *=* something", () => {
const italy = note("Italy").label("capital", "Rome");
const slovakia = note("Slovakia").label("capital", "Bratislava");
rootNote.child(note("Europe").child(slovakia).child(italy));
const searchContext = new SearchContext();
let searchResults = searchService.findResultsWithQuery("# note.text *=* vaki and note.noteId != root", searchContext);
expect(searchResults.length).toEqual(1);
expect(becca.notes[searchResults[0].noteId].title).toEqual("Slovakia");
});
it("test that fulltext does not match archived notes", () => {
const italy = note("Italy").label("capital", "Rome");
const slovakia = note("Slovakia").label("capital", "Bratislava");
rootNote.child(note("Reddit").label("archived", "", true).child(note("Post X")).child(note("Post Y"))).child(note("Reddit is bad"));
const searchContext = new SearchContext({ includeArchivedNotes: false });
let searchResults = searchService.findResultsWithQuery("reddit", searchContext);
expect(searchResults.length).toEqual(1);
expect(becca.notes[searchResults[0].noteId].title).toEqual("Reddit is bad");
});
it("search completes in reasonable time", () => {
// Create a moderate-sized dataset to test performance
const countries = ["Austria", "Belgium", "Croatia", "Denmark", "Estonia", "Finland", "Germany", "Hungary", "Ireland", "Japan"];
const europeanCountries = note("Europe");
countries.forEach(country => {
europeanCountries.child(note(country).label("type", "country").label("continent", "Europe"));
});
rootNote.child(europeanCountries);
const searchContext = new SearchContext();
const startTime = Date.now();
// Perform a search that exercises multiple features
const searchResults = searchService.findResultsWithQuery("#type=country AND continent", searchContext);
const endTime = Date.now();
const duration = endTime - startTime;
// Search should complete in under 1 second for reasonable dataset
expect(duration).toBeLessThan(1000);
expect(searchResults.length).toEqual(10);
});
it("progressive search always puts exact matches before fuzzy matches", () => {
rootNote
.child(note("Analysis Report")) // Exact match
.child(note("Data Analysis")) // Exact match
.child(note("Test Analysis")) // Exact match
.child(note("Advanced Anaylsis")) // Fuzzy match (typo)
.child(note("Quick Anlaysis")); // Fuzzy match (typo)
const searchContext = new SearchContext();
const searchResults = searchService.findResultsWithQuery("analysis", searchContext);
// With only 3 exact matches (below threshold), fuzzy should be triggered
// Should find all 5 matches but exact ones should come first
expect(searchResults.length).toEqual(5);
// Get note titles in result order
const resultTitles = searchResults.map(r => becca.notes[r.noteId].title);
// Find all exact matches (contain "analysis")
const exactMatchIndices = resultTitles.map((title, index) =>
title.toLowerCase().includes("analysis") ? index : -1
).filter(index => index !== -1);
// Find all fuzzy matches (contain typos)
const fuzzyMatchIndices = resultTitles.map((title, index) =>
(title.includes("Anaylsis") || title.includes("Anlaysis")) ? index : -1
).filter(index => index !== -1);
expect(exactMatchIndices.length).toEqual(3);
expect(fuzzyMatchIndices.length).toEqual(2);
// CRITICAL: All exact matches must appear before all fuzzy matches
const lastExactIndex = Math.max(...exactMatchIndices);
const firstFuzzyIndex = Math.min(...fuzzyMatchIndices);
expect(lastExactIndex).toBeLessThan(firstFuzzyIndex);
});
// FIXME: test what happens when we order without any filter criteria
// it("comparison between labels", () => {
// rootNote
// .child(note("Europe")
// .child(note("Austria")
// .label('capital', 'Vienna')
// .label('largestCity', 'Vienna'))
// .child(note("Canada")
// .label('capital', 'Ottawa')
// .label('largestCity', 'Toronto'))
// .child(note("Czech Republic")
// .label('capital', 'Prague')
// .label('largestCity', 'Prague'))
// );
//
// const searchContext = new SearchContext();
//
// const searchResults = searchService.findResultsWithQuery('#capital = #largestCity', searchContext);
// expect(searchResults.length).toEqual(2);
// expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy();
// expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy();
// })
});

794
apps/server/src/services/search/services/search.ts
View File

@@ -1,794 +0,0 @@
import { becca_service } from "@triliumnext/core";
import normalizeString from "normalize-strings";
import striptags from "striptags";
import becca from "../../../becca/becca.js";
import type BNote from "../../../becca/entities/bnote.js";
import hoistedNoteService from "../../hoisted_note.js";
import log from "../../log.js";
import protectedSessionService from "../../protected_session.js";
import scriptService from "../../script.js";
import sql from "../../sql.js";
import { escapeHtml, escapeRegExp } from "../../utils.js";
import type Expression from "../expressions/expression.js";
import SearchContext from "../search_context.js";
import SearchResult from "../search_result.js";
import handleParens from "./handle_parens.js";
import lex from "./lex.js";
import parse from "./parse.js";
import type { SearchParams, TokenStructure } from "./types.js";
export interface SearchNoteResult {
searchResultNoteIds: string[];
highlightedTokens: string[];
error: string | null;
}
export const EMPTY_RESULT: SearchNoteResult = {
searchResultNoteIds: [],
highlightedTokens: [],
error: null
};
function searchFromNote(note: BNote): SearchNoteResult {
let searchResultNoteIds;
let highlightedTokens: string[];
const searchScript = note.getRelationValue("searchScript");
const searchString = note.getLabelValue("searchString") || "";
let error: string | null = null;
if (searchScript) {
searchResultNoteIds = searchFromRelation(note, "searchScript");
highlightedTokens = [];
} else {
const searchContext = new SearchContext({
fastSearch: note.hasLabel("fastSearch"),
ancestorNoteId: note.getRelationValue("ancestor") || undefined,
ancestorDepth: note.getLabelValue("ancestorDepth") || undefined,
includeArchivedNotes: note.hasLabel("includeArchivedNotes"),
orderBy: note.getLabelValue("orderBy") || undefined,
orderDirection: note.getLabelValue("orderDirection") || undefined,
limit: parseInt(note.getLabelValue("limit") || "0", 10),
debug: note.hasLabel("debug"),
fuzzyAttributeSearch: false
});
searchResultNoteIds = findResultsWithQuery(searchString, searchContext).map((sr) => sr.noteId);
highlightedTokens = searchContext.highlightedTokens;
error = searchContext.getError();
}
// we won't return search note's own noteId
// also don't allow root since that would force infinite cycle
return {
searchResultNoteIds: searchResultNoteIds.filter((resultNoteId) => !["root", note.noteId].includes(resultNoteId)),
highlightedTokens,
error
};
}
function searchFromRelation(note: BNote, relationName: string) {
const scriptNote = note.getRelationTarget(relationName);
if (!scriptNote) {
log.info(`Search note's relation ${relationName} has not been found.`);
return [];
}
if (!scriptNote.isJavaScript() || scriptNote.getScriptEnv() !== "backend") {
log.info(`Note ${scriptNote.noteId} is not executable.`);
return [];
}
if (!note.isContentAvailable()) {
log.info(`Note ${scriptNote.noteId} is not available outside of protected session.`);
return [];
}
const result = scriptService.executeNote(scriptNote, { originEntity: note });
if (!Array.isArray(result)) {
log.info(`Result from ${scriptNote.noteId} is not an array.`);
return [];
}
if (result.length === 0) {
return [];
}
// we expect either array of noteIds (strings) or notes, in that case we extract noteIds ourselves
return typeof result[0] === "string" ? result : result.map((item) => item.noteId);
}
function loadNeededInfoFromDatabase() {
/**
* This complex structure is needed to calculate total occupied space by a note. Several object instances
* (note, revisions, attachments) can point to a single blobId, and thus the blob size should count towards the total
* only once.
*
* noteId => { blobId => blobSize }
*/
const noteBlobs: Record<string, Record<string, number>> = {};
type NoteContentLengthsRow = {
noteId: string;
blobId: string;
length: number;
};
const noteContentLengths = sql.getRows<NoteContentLengthsRow>(`
SELECT
noteId,
blobId,
LENGTH(content) AS length
FROM notes
JOIN blobs USING(blobId)
WHERE notes.isDeleted = 0`);
for (const { noteId, blobId, length } of noteContentLengths) {
if (!(noteId in becca.notes)) {
log.error(`Note '${noteId}' not found in becca.`);
continue;
}
becca.notes[noteId].contentSize = length;
becca.notes[noteId].revisionCount = 0;
noteBlobs[noteId] = { [blobId]: length };
}
type AttachmentContentLengthsRow = {
noteId: string;
blobId: string;
length: number;
};
const attachmentContentLengths = sql.getRows<AttachmentContentLengthsRow>(`
SELECT
ownerId AS noteId,
attachments.blobId,
LENGTH(content) AS length
FROM attachments
JOIN notes ON attachments.ownerId = notes.noteId
JOIN blobs ON attachments.blobId = blobs.blobId
WHERE attachments.isDeleted = 0
AND notes.isDeleted = 0`);
for (const { noteId, blobId, length } of attachmentContentLengths) {
if (!(noteId in becca.notes)) {
log.error(`Note '${noteId}' not found in becca.`);
continue;
}
if (!(noteId in noteBlobs)) {
log.error(`Did not find a '${noteId}' in the noteBlobs.`);
continue;
}
noteBlobs[noteId][blobId] = length;
}
for (const noteId in noteBlobs) {
becca.notes[noteId].contentAndAttachmentsSize = Object.values(noteBlobs[noteId]).reduce((acc, size) => acc + size, 0);
}
type RevisionRow = {
noteId: string;
blobId: string;
length: number;
isNoteRevision: true;
};
const revisionContentLengths = sql.getRows<RevisionRow>(`
SELECT
noteId,
revisions.blobId,
LENGTH(content) AS length,
1 AS isNoteRevision
FROM notes
JOIN revisions USING(noteId)
JOIN blobs ON revisions.blobId = blobs.blobId
WHERE notes.isDeleted = 0
UNION ALL
SELECT
noteId,
revisions.blobId,
LENGTH(content) AS length,
0 AS isNoteRevision -- it's attachment not counting towards revision count
FROM notes
JOIN revisions USING(noteId)
JOIN attachments ON attachments.ownerId = revisions.revisionId
JOIN blobs ON attachments.blobId = blobs.blobId
WHERE notes.isDeleted = 0`);
for (const { noteId, blobId, length, isNoteRevision } of revisionContentLengths) {
if (!(noteId in becca.notes)) {
log.error(`Note '${noteId}' not found in becca.`);
continue;
}
if (!(noteId in noteBlobs)) {
log.error(`Did not find a '${noteId}' in the noteBlobs.`);
continue;
}
noteBlobs[noteId][blobId] = length;
if (isNoteRevision) {
const noteRevision = becca.notes[noteId];
if (noteRevision && noteRevision.revisionCount) {
noteRevision.revisionCount++;
}
}
}
for (const noteId in noteBlobs) {
becca.notes[noteId].contentAndAttachmentsAndRevisionsSize = Object.values(noteBlobs[noteId]).reduce((acc, size) => acc + size, 0);
}
}
function findResultsWithExpression(expression: Expression, searchContext: SearchContext): SearchResult[] {
if (searchContext.dbLoadNeeded) {
loadNeededInfoFromDatabase();
}
// If there's an explicit orderBy clause, skip progressive search
// as it would interfere with the ordering
if (searchContext.orderBy) {
// For ordered queries, don't use progressive search but respect
// the original fuzzy matching setting
return performSearch(expression, searchContext, searchContext.enableFuzzyMatching);
}
// If fuzzy matching is explicitly disabled, skip progressive search
if (!searchContext.enableFuzzyMatching) {
return performSearch(expression, searchContext, false);
}
// Phase 1: Try exact matches first (without fuzzy matching)
const exactResults = performSearch(expression, searchContext, false);
// Check if we have sufficient high-quality results
const minResultThreshold = 5;
const minScoreForQuality = 10; // Minimum score to consider a result "high quality"
const highQualityResults = exactResults.filter(result => result.score >= minScoreForQuality);
// If we have enough high-quality exact matches, return them
if (highQualityResults.length >= minResultThreshold) {
return exactResults;
}
// Phase 2: Add fuzzy matching as fallback when exact matches are insufficient
const fuzzyResults = performSearch(expression, searchContext, true);
// Merge results, ensuring exact matches always rank higher than fuzzy matches
return mergeExactAndFuzzyResults(exactResults, fuzzyResults);
}
function performSearch(expression: Expression, searchContext: SearchContext, enableFuzzyMatching: boolean): SearchResult[] {
const allNoteSet = becca.getAllNoteSet();
const noteIdToNotePath: Record<string, string[]> = {};
const executionContext = {
noteIdToNotePath
};
// Store original fuzzy setting and temporarily override it
const originalFuzzyMatching = searchContext.enableFuzzyMatching;
searchContext.enableFuzzyMatching = enableFuzzyMatching;
const noteSet = expression.execute(allNoteSet, executionContext, searchContext);
const searchResults = noteSet.notes.map((note) => {
const notePathArray = executionContext.noteIdToNotePath[note.noteId] || note.getBestNotePath();
if (!notePathArray) {
throw new Error(`Can't find note path for note ${JSON.stringify(note.getPojo())}`);
}
return new SearchResult(notePathArray);
});
for (const res of searchResults) {
res.computeScore(searchContext.fulltextQuery, searchContext.highlightedTokens, enableFuzzyMatching);
}
// Restore original fuzzy setting
searchContext.enableFuzzyMatching = originalFuzzyMatching;
if (!noteSet.sorted) {
searchResults.sort((a, b) => {
if (a.score > b.score) {
return -1;
} else if (a.score < b.score) {
return 1;
}
// if score does not decide then sort results by depth of the note.
// This is based on the assumption that more important results are closer to the note root.
if (a.notePathArray.length === b.notePathArray.length) {
return a.notePathTitle < b.notePathTitle ? -1 : 1;
}
return a.notePathArray.length < b.notePathArray.length ? -1 : 1;
});
}
return searchResults;
}
function mergeExactAndFuzzyResults(exactResults: SearchResult[], fuzzyResults: SearchResult[]): SearchResult[] {
// Create a map of exact result note IDs for deduplication
const exactNoteIds = new Set(exactResults.map(result => result.noteId));
// Add fuzzy results that aren't already in exact results
const additionalFuzzyResults = fuzzyResults.filter(result => !exactNoteIds.has(result.noteId));
// Sort exact results by score (best exact matches first)
exactResults.sort((a, b) => {
if (a.score > b.score) {
return -1;
} else if (a.score < b.score) {
return 1;
}
// if score does not decide then sort results by depth of the note.
if (a.notePathArray.length === b.notePathArray.length) {
return a.notePathTitle < b.notePathTitle ? -1 : 1;
}
return a.notePathArray.length < b.notePathArray.length ? -1 : 1;
});
// Sort fuzzy results by score (best fuzzy matches first)
additionalFuzzyResults.sort((a, b) => {
if (a.score > b.score) {
return -1;
} else if (a.score < b.score) {
return 1;
}
// if score does not decide then sort results by depth of the note.
if (a.notePathArray.length === b.notePathArray.length) {
return a.notePathTitle < b.notePathTitle ? -1 : 1;
}
return a.notePathArray.length < b.notePathArray.length ? -1 : 1;
});
// CRITICAL: Always put exact matches before fuzzy matches, regardless of scores
return [...exactResults, ...additionalFuzzyResults];
}
function parseQueryToExpression(query: string, searchContext: SearchContext) {
const { fulltextQuery, fulltextTokens, expressionTokens, leadingOperator } = lex(query);
searchContext.fulltextQuery = fulltextQuery;
let structuredExpressionTokens: TokenStructure;
try {
structuredExpressionTokens = handleParens(expressionTokens);
} catch (e: any) {
structuredExpressionTokens = [];
searchContext.addError(e.message);
}
const expression = parse({
fulltextTokens,
expressionTokens: structuredExpressionTokens,
searchContext,
originalQuery: query,
leadingOperator
});
if (searchContext.debug) {
searchContext.debugInfo = {
fulltextTokens,
structuredExpressionTokens,
expression
};
log.info(`Search debug: ${JSON.stringify(searchContext.debugInfo, null, 4)}`);
}
return expression;
}
function searchNotes(query: string, params: SearchParams = {}): BNote[] {
const searchResults = findResultsWithQuery(query, new SearchContext(params));
return searchResults.map((sr) => becca.notes[sr.noteId]);
}
function findResultsWithQuery(query: string, searchContext: SearchContext): SearchResult[] {
query = query || "";
searchContext.originalQuery = query;
const expression = parseQueryToExpression(query, searchContext);
if (!expression) {
return [];
}
// If the query starts with '#', it's a pure expression query.
// Don't use progressive search for these as they may have complex
// ordering or other logic that shouldn't be interfered with.
const isPureExpressionQuery = query.trim().startsWith('#');
if (isPureExpressionQuery) {
// For pure expression queries, use standard search without progressive phases
return performSearch(expression, searchContext, searchContext.enableFuzzyMatching);
}
return findResultsWithExpression(expression, searchContext);
}
function findFirstNoteWithQuery(query: string, searchContext: SearchContext): BNote | null {
const searchResults = findResultsWithQuery(query, searchContext);
return searchResults.length > 0 ? becca.notes[searchResults[0].noteId] : null;
}
function extractContentSnippet(noteId: string, searchTokens: string[], maxLength: number = 200): string {
const note = becca.notes[noteId];
if (!note) {
return "";
}
// Only extract content for text-based notes
if (!["text", "code", "mermaid", "canvas", "mindMap"].includes(note.type)) {
return "";
}
try {
let content = note.getContent();
if (!content || typeof content !== "string") {
return "";
}
// Handle protected notes
if (note.isProtected && protectedSessionService.isProtectedSessionAvailable()) {
try {
content = protectedSessionService.decryptString(content) || "";
} catch (e) {
return ""; // Can't decrypt, don't show content
}
} else if (note.isProtected) {
return ""; // Protected but no session available
}
// Strip HTML tags for text notes
if (note.type === "text") {
content = striptags(content);
}
// Normalize whitespace while preserving paragraph breaks
// First, normalize multiple newlines to double newlines (paragraph breaks)
content = content.replace(/\n\s*\n/g, "\n\n");
// Then normalize spaces within lines
content = content.split('\n').map(line => line.replace(/\s+/g, " ").trim()).join('\n');
// Finally trim the whole content
content = content.trim();
if (!content) {
return "";
}
// Try to find a snippet around the first matching token
const normalizedContent = normalizeString(content.toLowerCase());
let snippetStart = 0;
let matchFound = false;
for (const token of searchTokens) {
const normalizedToken = normalizeString(token.toLowerCase());
const matchIndex = normalizedContent.indexOf(normalizedToken);
if (matchIndex !== -1) {
// Center the snippet around the match
snippetStart = Math.max(0, matchIndex - maxLength / 2);
matchFound = true;
break;
}
}
// Extract snippet
let snippet = content.substring(snippetStart, snippetStart + maxLength);
// If snippet contains linebreaks, limit to max 4 lines and override character limit
const lines = snippet.split('\n');
if (lines.length > 4) {
// Find which lines contain the search tokens to ensure they're included
const normalizedLines = lines.map(line => normalizeString(line.toLowerCase()));
const normalizedTokens = searchTokens.map(token => normalizeString(token.toLowerCase()));
// Find the first line that contains a search token
let firstMatchLine = -1;
for (let i = 0; i < normalizedLines.length; i++) {
if (normalizedTokens.some(token => normalizedLines[i].includes(token))) {
firstMatchLine = i;
break;
}
}
if (firstMatchLine !== -1) {
// Center the 4-line window around the first match
// Try to show 1 line before and 2 lines after the match
const startLine = Math.max(0, firstMatchLine - 1);
const endLine = Math.min(lines.length, startLine + 4);
snippet = lines.slice(startLine, endLine).join('\n');
} else {
// No match found in lines (shouldn't happen), just take first 4
snippet = lines.slice(0, 4).join('\n');
}
// Add ellipsis if we truncated lines
snippet = `${snippet }...`;
} else if (lines.length > 1) {
// For multi-line snippets that are 4 or fewer lines, keep them as-is
// No need to truncate
} else {
// Single line content - apply original word boundary logic
// Try to start/end at word boundaries
if (snippetStart > 0) {
const firstSpace = snippet.search(/\s/);
if (firstSpace > 0 && firstSpace < 20) {
snippet = snippet.substring(firstSpace + 1);
}
snippet = `...${ snippet}`;
}
if (snippetStart + maxLength < content.length) {
const lastSpace = snippet.search(/\s[^\s]*$/);
if (lastSpace > snippet.length - 20 && lastSpace > 0) {
snippet = snippet.substring(0, lastSpace);
}
snippet = `${snippet }...`;
}
}
return snippet;
} catch (e) {
log.error(`Error extracting content snippet for note ${noteId}: ${e}`);
return "";
}
}
function extractAttributeSnippet(noteId: string, searchTokens: string[], maxLength: number = 200): string {
const note = becca.notes[noteId];
if (!note) {
return "";
}
try {
// Get all attributes for this note
const attributes = note.getAttributes();
if (!attributes || attributes.length === 0) {
return "";
}
const matchingAttributes: Array<{name: string, value: string, type: string}> = [];
// Look for attributes that match the search tokens
for (const attr of attributes) {
const attrName = attr.name?.toLowerCase() || "";
const attrValue = attr.value?.toLowerCase() || "";
const attrType = attr.type || "";
// Check if any search token matches the attribute name or value
const hasMatch = searchTokens.some(token => {
const normalizedToken = normalizeString(token.toLowerCase());
return attrName.includes(normalizedToken) || attrValue.includes(normalizedToken);
});
if (hasMatch) {
matchingAttributes.push({
name: attr.name || "",
value: attr.value || "",
type: attrType
});
}
}
if (matchingAttributes.length === 0) {
return "";
}
// Limit to 4 lines maximum, similar to content snippet logic
const lines: string[] = [];
for (const attr of matchingAttributes.slice(0, 4)) {
let line = "";
if (attr.type === "label") {
line = attr.value ? `#${attr.name}="${attr.value}"` : `#${attr.name}`;
} else if (attr.type === "relation") {
// For relations, show the target note title if possible
const targetNote = attr.value ? becca.notes[attr.value] : null;
const targetTitle = targetNote ? targetNote.title : attr.value;
line = `~${attr.name}="${targetTitle}"`;
}
if (line) {
lines.push(line);
}
}
let snippet = lines.join('\n');
// Apply length limit while preserving line structure
if (snippet.length > maxLength) {
// Try to truncate at word boundaries but keep lines intact
const truncated = snippet.substring(0, maxLength);
const lastNewline = truncated.lastIndexOf('\n');
if (lastNewline > maxLength / 2) {
// If we can keep most content by truncating to last complete line
snippet = truncated.substring(0, lastNewline);
} else {
// Otherwise just truncate and add ellipsis
const lastSpace = truncated.lastIndexOf(' ');
snippet = truncated.substring(0, lastSpace > maxLength / 2 ? lastSpace : maxLength - 3);
snippet = `${snippet }...`;
}
}
return snippet;
} catch (e) {
log.error(`Error extracting attribute snippet for note ${noteId}: ${e}`);
return "";
}
}
function searchNotesForAutocomplete(query: string, fastSearch: boolean = true) {
const searchContext = new SearchContext({
fastSearch,
includeArchivedNotes: false,
includeHiddenNotes: true,
fuzzyAttributeSearch: true,
ignoreInternalAttributes: true,
ancestorNoteId: hoistedNoteService.isHoistedInHiddenSubtree() ? "root" : hoistedNoteService.getHoistedNoteId()
});
const allSearchResults = findResultsWithQuery(query, searchContext);
const trimmed = allSearchResults.slice(0, 200);
// Extract content and attribute snippets
for (const result of trimmed) {
result.contentSnippet = extractContentSnippet(result.noteId, searchContext.highlightedTokens);
result.attributeSnippet = extractAttributeSnippet(result.noteId, searchContext.highlightedTokens);
}
highlightSearchResults(trimmed, searchContext.highlightedTokens, searchContext.ignoreInternalAttributes);
return trimmed.map((result) => {
const { title, icon } = becca_service.getNoteTitleAndIcon(result.noteId);
return {
notePath: result.notePath,
noteTitle: title,
notePathTitle: result.notePathTitle,
highlightedNotePathTitle: result.highlightedNotePathTitle,
contentSnippet: result.contentSnippet,
highlightedContentSnippet: result.highlightedContentSnippet,
attributeSnippet: result.attributeSnippet,
highlightedAttributeSnippet: result.highlightedAttributeSnippet,
icon: icon ?? "bx bx-note"
};
});
}
/**
* @param ignoreInternalAttributes whether to ignore certain attributes from the search such as ~internalLink.
*/
function highlightSearchResults(searchResults: SearchResult[], highlightedTokens: string[], ignoreInternalAttributes = false) {
highlightedTokens = Array.from(new Set(highlightedTokens));
// we remove < signs because they can cause trouble in matching and overwriting existing highlighted chunks
// which would make the resulting HTML string invalid.
// { and } are used for marking <b> and </b> tag (to avoid matches on single 'b' character)
// < and > are used for marking <small> and </small>
highlightedTokens = highlightedTokens.map((token) => token.replace("/[<\{\}]/g", "")).filter((token) => !!token?.trim());
// sort by the longest, so we first highlight the longest matches
highlightedTokens.sort((a, b) => (a.length > b.length ? -1 : 1));
for (const result of searchResults) {
result.highlightedNotePathTitle = result.notePathTitle.replace(/[<{}]/g, "");
// Initialize highlighted content snippet
if (result.contentSnippet) {
// Escape HTML but preserve newlines for later conversion to <br>
result.highlightedContentSnippet = escapeHtml(result.contentSnippet);
// Remove any stray < { } that might interfere with our highlighting markers
result.highlightedContentSnippet = result.highlightedContentSnippet.replace(/[<{}]/g, "");
}
// Initialize highlighted attribute snippet
if (result.attributeSnippet) {
// Escape HTML but preserve newlines for later conversion to <br>
result.highlightedAttributeSnippet = escapeHtml(result.attributeSnippet);
// Remove any stray < { } that might interfere with our highlighting markers
result.highlightedAttributeSnippet = result.highlightedAttributeSnippet.replace(/[<{}]/g, "");
}
}
function wrapText(text: string, start: number, length: number, prefix: string, suffix: string) {
return text.substring(0, start) + prefix + text.substr(start, length) + suffix + text.substring(start + length);
}
for (const token of highlightedTokens) {
if (!token) {
// Avoid empty tokens, which might cause an infinite loop.
continue;
}
for (const result of searchResults) {
// Reset token
const tokenRegex = new RegExp(escapeRegExp(token), "gi");
let match;
// Highlight in note path title
if (result.highlightedNotePathTitle) {
const titleRegex = new RegExp(escapeRegExp(token), "gi");
while ((match = titleRegex.exec(normalizeString(result.highlightedNotePathTitle))) !== null) {
result.highlightedNotePathTitle = wrapText(result.highlightedNotePathTitle, match.index, token.length, "{", "}");
// 2 characters are added, so we need to adjust the index
titleRegex.lastIndex += 2;
}
}
// Highlight in content snippet
if (result.highlightedContentSnippet) {
const contentRegex = new RegExp(escapeRegExp(token), "gi");
while ((match = contentRegex.exec(normalizeString(result.highlightedContentSnippet))) !== null) {
result.highlightedContentSnippet = wrapText(result.highlightedContentSnippet, match.index, token.length, "{", "}");
// 2 characters are added, so we need to adjust the index
contentRegex.lastIndex += 2;
}
}
// Highlight in attribute snippet
if (result.highlightedAttributeSnippet) {
const attributeRegex = new RegExp(escapeRegExp(token), "gi");
while ((match = attributeRegex.exec(normalizeString(result.highlightedAttributeSnippet))) !== null) {
result.highlightedAttributeSnippet = wrapText(result.highlightedAttributeSnippet, match.index, token.length, "{", "}");
// 2 characters are added, so we need to adjust the index
attributeRegex.lastIndex += 2;
}
}
}
}
for (const result of searchResults) {
if (result.highlightedNotePathTitle) {
result.highlightedNotePathTitle = result.highlightedNotePathTitle.replace(/{/g, "<b>").replace(/}/g, "</b>");
}
if (result.highlightedContentSnippet) {
// Replace highlighting markers with HTML tags
result.highlightedContentSnippet = result.highlightedContentSnippet.replace(/{/g, "<b>").replace(/}/g, "</b>");
// Convert newlines to <br> tags for HTML display
result.highlightedContentSnippet = result.highlightedContentSnippet.replace(/\n/g, "<br>");
}
if (result.highlightedAttributeSnippet) {
// Replace highlighting markers with HTML tags
result.highlightedAttributeSnippet = result.highlightedAttributeSnippet.replace(/{/g, "<b>").replace(/}/g, "</b>");
// Convert newlines to <br> tags for HTML display
result.highlightedAttributeSnippet = result.highlightedAttributeSnippet.replace(/\n/g, "<br>");
}
}
}
export default {
searchFromNote,
searchNotesForAutocomplete,
findResultsWithQuery,
findFirstNoteWithQuery,
searchNotes,
extractContentSnippet,
extractAttributeSnippet,
highlightSearchResults
};

24
apps/server/src/services/search/services/types.ts
View File

@@ -1,24 +0,0 @@
export type TokenStructure = (TokenData | TokenStructure)[];
export interface TokenData {
token: string;
inQuotes?: boolean;
startIndex?: number;
endIndex?: number;
}
export interface SearchParams {
fastSearch?: boolean;
includeArchivedNotes?: boolean;
includeHiddenNotes?: boolean;
ignoreHoistedNote?: boolean;
/** Whether to ignore certain attributes from the search such as ~internalLink. */
ignoreInternalAttributes?: boolean;
ancestorNoteId?: string;
ancestorDepth?: string;
orderBy?: string;
orderDirection?: string;
limit?: number | null;
debug?: boolean;
fuzzyAttributeSearch?: boolean;
}

65
apps/server/src/services/search/utils/text_utils.spec.ts
View File

@@ -1,65 +0,0 @@
import { describe, it, expect } from "vitest";
import { calculateOptimizedEditDistance, validateFuzzySearchTokens, fuzzyMatchWord } from './text_utils.js';
describe('Fuzzy Search Core', () => {
describe('calculateOptimizedEditDistance', () => {
it('calculates edit distance for common typos', () => {
expect(calculateOptimizedEditDistance('hello', 'helo')).toBe(1);
expect(calculateOptimizedEditDistance('world', 'wrold')).toBe(2);
expect(calculateOptimizedEditDistance('cafe', 'café')).toBe(1);
expect(calculateOptimizedEditDistance('identical', 'identical')).toBe(0);
});
it('handles performance safety with oversized input', () => {
const longString = 'a'.repeat(2000);
const result = calculateOptimizedEditDistance(longString, 'short');
expect(result).toBeGreaterThan(2); // Should use fallback heuristic
});
});
describe('validateFuzzySearchTokens', () => {
it('validates minimum length requirements for fuzzy operators', () => {
const result1 = validateFuzzySearchTokens(['ab'], '~=');
expect(result1.isValid).toBe(false);
expect(result1.error).toContain('at least 3 characters');
const result2 = validateFuzzySearchTokens(['hello'], '~=');
expect(result2.isValid).toBe(true);
const result3 = validateFuzzySearchTokens(['ok'], '=');
expect(result3.isValid).toBe(true); // Non-fuzzy operators allow short tokens
});
it('validates token types and empty arrays', () => {
expect(validateFuzzySearchTokens([], '=')).toEqual({
isValid: false,
error: 'Invalid tokens: at least one token is required'
});
expect(validateFuzzySearchTokens([''], '=')).toEqual({
isValid: false,
error: 'Invalid tokens: empty or whitespace-only tokens are not allowed'
});
});
});
describe('fuzzyMatchWord', () => {
it('matches words with diacritics normalization', () => {
expect(fuzzyMatchWord('cafe', 'café')).toBe(true);
expect(fuzzyMatchWord('naive', 'naïve')).toBe(true);
});
it('matches with typos within distance threshold', () => {
expect(fuzzyMatchWord('hello', 'helo')).toBe(true);
expect(fuzzyMatchWord('world', 'wrold')).toBe(true);
expect(fuzzyMatchWord('test', 'tset')).toBe(true);
expect(fuzzyMatchWord('test', 'xyz')).toBe(false);
});
it('handles edge cases safely', () => {
expect(fuzzyMatchWord('', 'test')).toBe(false);
expect(fuzzyMatchWord('test', '')).toBe(false);
expect(fuzzyMatchWord('a', 'b')).toBe(false); // Very short tokens
});
});
});

334
apps/server/src/services/search/utils/text_utils.ts
View File

@@ -1,334 +0,0 @@
"use strict";
import { normalize } from "../../utils.js";
/**
* Shared text processing utilities for search functionality
*/
// Configuration constants for fuzzy matching
export const FUZZY_SEARCH_CONFIG = {
// Minimum token length for fuzzy operators to prevent false positives
MIN_FUZZY_TOKEN_LENGTH: 3,
// Maximum edit distance for fuzzy matching
MAX_EDIT_DISTANCE: 2,
// Maximum proximity distance for phrase matching (in words)
MAX_PHRASE_PROXIMITY: 10,
// Absolute hard limits for extreme cases - only to prevent system crashes
ABSOLUTE_MAX_CONTENT_SIZE: 100 * 1024 * 1024, // 100MB - extreme upper limit to prevent OOM
ABSOLUTE_MAX_WORD_COUNT: 2000000, // 2M words - extreme upper limit for word processing
// Performance warning thresholds - inform user but still attempt search
PERFORMANCE_WARNING_SIZE: 5 * 1024 * 1024, // 5MB - warn about potential performance impact
PERFORMANCE_WARNING_WORDS: 100000, // 100K words - warn about word count impact
// Progressive processing thresholds for very large content
PROGRESSIVE_PROCESSING_SIZE: 10 * 1024 * 1024, // 10MB - use progressive processing
PROGRESSIVE_PROCESSING_WORDS: 500000, // 500K words - use progressive processing
// Performance thresholds
EARLY_TERMINATION_THRESHOLD: 3,
} as const;
/**
* Normalizes text by removing diacritics and converting to lowercase.
* This is the centralized text normalization function used across all search components.
* Uses the shared normalize function from utils for consistency.
*
* Examples:
* - "café" -> "cafe"
* - "naïve" -> "naive"
* - "HELLO WORLD" -> "hello world"
*
* @param text The text to normalize
* @returns The normalized text
*/
export function normalizeSearchText(text: string): string {
if (!text || typeof text !== 'string') {
return '';
}
// Use shared normalize function for consistency across the codebase
return normalize(text);
}
/**
* Optimized edit distance calculation using single array and early termination.
* This is significantly more memory efficient than the 2D matrix approach and includes
* early termination optimizations for better performance.
*
* @param str1 First string
* @param str2 Second string
* @param maxDistance Maximum allowed distance (for early termination)
* @returns The edit distance between the strings, or maxDistance + 1 if exceeded
*/
export function calculateOptimizedEditDistance(str1: string, str2: string, maxDistance: number = FUZZY_SEARCH_CONFIG.MAX_EDIT_DISTANCE): number {
// Input validation
if (typeof str1 !== 'string' || typeof str2 !== 'string') {
throw new Error('Both arguments must be strings');
}
if (maxDistance < 0 || !Number.isInteger(maxDistance)) {
throw new Error('maxDistance must be a non-negative integer');
}
const len1 = str1.length;
const len2 = str2.length;
// Performance guard: if strings are too long, limit processing
const maxStringLength = 1000;
if (len1 > maxStringLength || len2 > maxStringLength) {
// For very long strings, fall back to simple length-based heuristic
return Math.abs(len1 - len2) <= maxDistance ? Math.abs(len1 - len2) : maxDistance + 1;
}
// Early termination: if length difference exceeds max distance
if (Math.abs(len1 - len2) > maxDistance) {
return maxDistance + 1;
}
// Handle edge cases
if (len1 === 0) return len2 <= maxDistance ? len2 : maxDistance + 1;
if (len2 === 0) return len1 <= maxDistance ? len1 : maxDistance + 1;
// Use single array optimization for better memory usage
let previousRow = Array.from({ length: len2 + 1 }, (_, i) => i);
let currentRow = new Array(len2 + 1);
for (let i = 1; i <= len1; i++) {
currentRow[0] = i;
let minInRow = i;
for (let j = 1; j <= len2; j++) {
const cost = str1[i - 1] === str2[j - 1] ? 0 : 1;
currentRow[j] = Math.min(
previousRow[j] + 1, // deletion
currentRow[j - 1] + 1, // insertion
previousRow[j - 1] + cost // substitution
);
// Track minimum value in current row for early termination
if (currentRow[j] < minInRow) {
minInRow = currentRow[j];
}
}
// Early termination: if minimum distance in row exceeds threshold
if (minInRow > maxDistance) {
return maxDistance + 1;
}
// Swap arrays for next iteration
[previousRow, currentRow] = [currentRow, previousRow];
}
const result = previousRow[len2];
return result <= maxDistance ? result : maxDistance + 1;
}
/**
* Validates that tokens meet minimum requirements for fuzzy operators.
*
* @param tokens Array of search tokens
* @param operator The search operator being used
* @returns Validation result with success status and error message
*/
export function validateFuzzySearchTokens(tokens: string[], operator: string): { isValid: boolean; error?: string } {
if (!operator || typeof operator !== 'string') {
return {
isValid: false,
error: 'Invalid operator: operator must be a non-empty string'
};
}
if (!Array.isArray(tokens)) {
return {
isValid: false,
error: 'Invalid tokens: tokens must be an array'
};
}
if (tokens.length === 0) {
return {
isValid: false,
error: 'Invalid tokens: at least one token is required'
};
}
// Check for null, undefined, or non-string tokens
const invalidTypeTokens = tokens.filter(token =>
token == null || typeof token !== 'string'
);
if (invalidTypeTokens.length > 0) {
return {
isValid: false,
error: 'Invalid tokens: all tokens must be non-null strings'
};
}
// Check for empty string tokens
const emptyTokens = tokens.filter(token => token.trim().length === 0);
if (emptyTokens.length > 0) {
return {
isValid: false,
error: 'Invalid tokens: empty or whitespace-only tokens are not allowed'
};
}
if (operator !== '~=' && operator !== '~*') {
return { isValid: true };
}
// Check minimum token length for fuzzy operators
const shortTokens = tokens.filter(token => token.length < FUZZY_SEARCH_CONFIG.MIN_FUZZY_TOKEN_LENGTH);
if (shortTokens.length > 0) {
return {
isValid: false,
error: `Fuzzy search operators (~=, ~*) require tokens of at least ${FUZZY_SEARCH_CONFIG.MIN_FUZZY_TOKEN_LENGTH} characters. Invalid tokens: ${shortTokens.join(', ')}`
};
}
// Check for excessively long tokens that could cause performance issues
const maxTokenLength = 100; // Reasonable limit for search tokens
const longTokens = tokens.filter(token => token.length > maxTokenLength);
if (longTokens.length > 0) {
return {
isValid: false,
error: `Tokens are too long (max ${maxTokenLength} characters). Long tokens: ${longTokens.map(t => t.substring(0, 20) + '...').join(', ')}`
};
}
return { isValid: true };
}
/**
* Validates and preprocesses content for search operations.
* Philosophy: Try to search everything! Only block truly extreme cases that could crash the system.
*
* @param content The content to validate and preprocess
* @param noteId The note ID (for logging purposes)
* @returns Processed content, only null for truly extreme cases that could cause system instability
*/
export function validateAndPreprocessContent(content: string, noteId?: string): string | null {
if (!content || typeof content !== 'string') {
return null;
}
// Only block content that could actually crash the system (100MB+)
if (content.length > FUZZY_SEARCH_CONFIG.ABSOLUTE_MAX_CONTENT_SIZE) {
console.error(`Content size exceeds absolute system limit for note ${noteId || 'unknown'}: ${content.length} bytes - this could cause system instability`);
// Only in truly extreme cases, truncate to prevent system crash
return content.substring(0, FUZZY_SEARCH_CONFIG.ABSOLUTE_MAX_CONTENT_SIZE);
}
// Warn about very large content but still process it
if (content.length > FUZZY_SEARCH_CONFIG.PERFORMANCE_WARNING_SIZE) {
console.info(`Large content for note ${noteId || 'unknown'}: ${content.length} bytes - processing may take time but will attempt full search`);
}
// For word count, be even more permissive - only block truly extreme cases
const wordCount = content.split(/\s+/).length;
if (wordCount > FUZZY_SEARCH_CONFIG.ABSOLUTE_MAX_WORD_COUNT) {
console.error(`Word count exceeds absolute system limit for note ${noteId || 'unknown'}: ${wordCount} words - this could cause system instability`);
// Only in truly extreme cases, truncate to prevent system crash
return content.split(/\s+/).slice(0, FUZZY_SEARCH_CONFIG.ABSOLUTE_MAX_WORD_COUNT).join(' ');
}
// Warn about high word counts but still process them
if (wordCount > FUZZY_SEARCH_CONFIG.PERFORMANCE_WARNING_WORDS) {
console.info(`High word count for note ${noteId || 'unknown'}: ${wordCount} words - phrase matching may take time but will attempt full search`);
}
// Progressive processing warning for very large content
if (content.length > FUZZY_SEARCH_CONFIG.PROGRESSIVE_PROCESSING_SIZE || wordCount > FUZZY_SEARCH_CONFIG.PROGRESSIVE_PROCESSING_WORDS) {
console.info(`Very large content for note ${noteId || 'unknown'} - using progressive processing to maintain responsiveness`);
}
return content;
}
/**
* Escapes special regex characters in a string for use in RegExp constructor
*/
function escapeRegExp(string: string): string {
return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
/**
* Checks if a word matches a token with fuzzy matching and returns the matched word.
* Optimized for common case where distances are small.
*
* @param token The search token (should be normalized)
* @param text The text to match against (should be normalized)
* @param maxDistance Maximum allowed edit distance
* @returns The matched word if found, null otherwise
*/
export function fuzzyMatchWordWithResult(token: string, text: string, maxDistance: number = FUZZY_SEARCH_CONFIG.MAX_EDIT_DISTANCE): string | null {
// Input validation
if (typeof token !== 'string' || typeof text !== 'string') {
return null;
}
if (token.length === 0 || text.length === 0) {
return null;
}
try {
// Normalize both strings for comparison
const normalizedToken = token.toLowerCase();
const normalizedText = text.toLowerCase();
// Exact match check first (most common case)
if (normalizedText.includes(normalizedToken)) {
// Find the exact match in the original text to preserve case
const exactMatch = text.match(new RegExp(escapeRegExp(token), 'i'));
return exactMatch ? exactMatch[0] : token;
}
// For fuzzy matching, we need to check individual words in the text
// Split the text into words and check each word against the token
const words = normalizedText.split(/\s+/).filter(word => word.length > 0);
const originalWords = text.split(/\s+/).filter(word => word.length > 0);
for (let i = 0; i < words.length; i++) {
const word = words[i];
const originalWord = originalWords[i];
// Skip if word is too different in length for fuzzy matching
if (Math.abs(word.length - normalizedToken.length) > maxDistance) {
continue;
}
// For very short tokens or very different lengths, be more strict
if (normalizedToken.length < 4 || Math.abs(word.length - normalizedToken.length) > 2) {
continue;
}
// Use optimized edit distance calculation
const distance = calculateOptimizedEditDistance(normalizedToken, word, maxDistance);
if (distance <= maxDistance) {
return originalWord; // Return the original word with case preserved
}
}
return null;
} catch (error) {
// Log error and return null for safety
console.warn('Error in fuzzy word matching:', error);
return null;
}
}
/**
* Checks if a word matches a token with fuzzy matching.
* Optimized for common case where distances are small.
*
* @param token The search token (should be normalized)
* @param word The word to match against (should be normalized)
* @param maxDistance Maximum allowed edit distance
* @returns True if the word matches the token within the distance threshold
*/
export function fuzzyMatchWord(token: string, text: string, maxDistance: number = FUZZY_SEARCH_CONFIG.MAX_EDIT_DISTANCE): boolean {
return fuzzyMatchWordWithResult(token, text, maxDistance) !== null;
}

79
apps/server/src/services/search/value_extractor.spec.ts
View File

@@ -1,79 +0,0 @@
import { describe, it, expect, beforeEach } from "vitest";
import ValueExtractor from "./value_extractor.js";
import becca from "../../becca/becca.js";
import SearchContext from "./search_context.js";
import { note } from "../../test/becca_mocking.js";
const dsc = new SearchContext();
describe("Value extractor", () => {
beforeEach(() => {
becca.reset();
});
it("simple title extraction", async () => {
const europe = note("Europe").note;
const valueExtractor = new ValueExtractor(dsc, ["note", "title"]);
expect(valueExtractor.validate()).toBeFalsy();
expect(valueExtractor.extract(europe)).toEqual("Europe");
});
it("label extraction", async () => {
const austria = note("Austria").label("Capital", "Vienna").note;
let valueExtractor = new ValueExtractor(dsc, ["note", "labels", "capital"]);
expect(valueExtractor.validate()).toBeFalsy();
expect(valueExtractor.extract(austria)).toEqual("Vienna");
valueExtractor = new ValueExtractor(dsc, ["#capital"]);
expect(valueExtractor.validate()).toBeFalsy();
expect(valueExtractor.extract(austria)).toEqual("Vienna");
});
it("parent/child property extraction", async () => {
const vienna = note("Vienna");
const europe = note("Europe").child(note("Austria").child(vienna));
let valueExtractor = new ValueExtractor(dsc, ["note", "children", "children", "title"]);
expect(valueExtractor.validate()).toBeFalsy();
expect(valueExtractor.extract(europe.note)).toEqual("Vienna");
valueExtractor = new ValueExtractor(dsc, ["note", "parents", "parents", "title"]);
expect(valueExtractor.validate()).toBeFalsy();
expect(valueExtractor.extract(vienna.note)).toEqual("Europe");
});
it("extract through relation", async () => {
const czechRepublic = note("Czech Republic").label("capital", "Prague");
const slovakia = note("Slovakia").label("capital", "Bratislava");
const austria = note("Austria").relation("neighbor", czechRepublic.note).relation("neighbor", slovakia.note);
let valueExtractor = new ValueExtractor(dsc, ["note", "relations", "neighbor", "labels", "capital"]);
expect(valueExtractor.validate()).toBeFalsy();
expect(valueExtractor.extract(austria.note)).toEqual("Prague");
valueExtractor = new ValueExtractor(dsc, ["~neighbor", "labels", "capital"]);
expect(valueExtractor.validate()).toBeFalsy();
expect(valueExtractor.extract(austria.note)).toEqual("Prague");
});
});
describe("Invalid value extractor property path", () => {
it('each path must start with "note" (or label/relation)', () => expect(new ValueExtractor(dsc, ["neighbor"]).validate()).toBeTruthy());
it("extra path element after terminal label", () => expect(new ValueExtractor(dsc, ["~neighbor", "labels", "capital", "noteId"]).validate()).toBeTruthy());
it("extra path element after terminal title", () => expect(new ValueExtractor(dsc, ["note", "title", "isProtected"]).validate()).toBeTruthy());
it("relation name and note property is missing", () => expect(new ValueExtractor(dsc, ["note", "relations"]).validate()).toBeTruthy());
it("relation is specified but target note property is not specified", () => expect(new ValueExtractor(dsc, ["note", "relations", "myrel"]).validate()).toBeTruthy());
});

128
apps/server/src/services/search/value_extractor.ts
View File

@@ -1,128 +0,0 @@
"use strict";
import type BNote from "../../becca/entities/bnote.js";
/**
* Search string is lower cased for case-insensitive comparison. But when retrieving properties,
* we need a case-sensitive form, so we have this translation object.
*/
const PROP_MAPPING: Record<string, string> = {
noteid: "noteId",
title: "title",
type: "type",
mime: "mime",
isprotected: "isProtected",
isarchived: "isArchived",
datecreated: "dateCreated",
datemodified: "dateModified",
utcdatecreated: "utcDateCreated",
utcdatemodified: "utcDateModified",
parentcount: "parentCount",
childrencount: "childrenCount",
attributecount: "attributeCount",
labelcount: "labelCount",
ownedlabelcount: "ownedLabelCount",
relationcount: "relationCount",
ownedrelationcount: "ownedRelationCount",
relationcountincludinglinks: "relationCountIncludingLinks",
ownedrelationcountincludinglinks: "ownedRelationCountIncludingLinks",
targetrelationcount: "targetRelationCount",
targetrelationcountincludinglinks: "targetRelationCountIncludingLinks",
contentsize: "contentSize",
contentandattachmentssize: "contentAndAttachmentsSize",
contentandattachmentsandrevisionssize: "contentAndAttachmentsAndRevisionsSize",
revisioncount: "revisionCount"
};
interface SearchContext {
dbLoadNeeded: boolean;
}
class ValueExtractor {
private propertyPath: string[];
constructor(searchContext: SearchContext, propertyPath: string[]) {
this.propertyPath = propertyPath.map((pathEl) => pathEl.toLowerCase());
if (this.propertyPath[0].startsWith("#")) {
this.propertyPath = ["note", "labels", this.propertyPath[0].substr(1), ...this.propertyPath.slice(1, this.propertyPath.length)];
} else if (this.propertyPath[0].startsWith("~")) {
this.propertyPath = ["note", "relations", this.propertyPath[0].substr(1), ...this.propertyPath.slice(1, this.propertyPath.length)];
}
if (["contentsize", "contentandattachmentssize", "contentandattachmentsandrevisionssize", "revisioncount"].includes(this.propertyPath[this.propertyPath.length - 1])) {
searchContext.dbLoadNeeded = true;
}
}
validate() {
if (this.propertyPath[0] !== "note") {
return `property specifier must start with 'note', but starts with '${this.propertyPath[0]}'`;
}
for (let i = 1; i < this.propertyPath.length; i++) {
const pathEl = this.propertyPath[i];
if (pathEl === "labels") {
if (i !== this.propertyPath.length - 2) {
return `label is a terminal property specifier and must be at the end`;
}
i++;
} else if (pathEl === "relations") {
if (i >= this.propertyPath.length - 2) {
return `relation name or property name is missing`;
}
i++;
} else if (pathEl in PROP_MAPPING || pathEl === "random") {
if (i !== this.propertyPath.length - 1) {
return `${pathEl} is a terminal property specifier and must be at the end`;
}
} else if (!["parents", "children"].includes(pathEl)) {
return `Unrecognized property specifier ${pathEl}`;
}
}
}
extract(note: BNote) {
let cursor: BNote | null = note;
let i: number = 0;
const cur = () => this.propertyPath[i];
for (i = 0; i < this.propertyPath.length; i++) {
if (!cursor) {
return cursor;
}
if (cur() === "labels") {
i++;
const attr = cursor.getAttributeCaseInsensitive("label", cur());
return attr ? attr.value : null;
}
if (cur() === "relations") {
i++;
const attr = cursor.getAttributeCaseInsensitive("relation", cur());
cursor = attr?.targetNote || null;
} else if (cur() === "parents") {
cursor = cursor.parents[0];
} else if (cur() === "children") {
cursor = cursor.children[0];
} else if (cur() === "random") {
return Math.random().toString(); // string is expected for comparison
} else if (cur() in PROP_MAPPING) {
return (cursor as any)[PROP_MAPPING[cur()]];
} else {
// FIXME
}
}
}
}
export default ValueExtractor;

6
apps/server/src/services/utils.ts
View File

@@ -105,10 +105,6 @@ export function stripTags(text: string) {
return text.replace(/<(?:.|\n)*?>/gm, "");
}
export function escapeRegExp(str: string) {
return str.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1");
}
export async function crash(message: string) {
if (isElectron) {
const electron = await import("electron");
@@ -450,6 +446,8 @@ function slugify(text: string) {
/** @deprecated */
export const escapeHtml = coreUtils.escapeHtml;
/** @deprecated */
export const escapeRegExp = coreUtils.escapeRegExp;
/** @deprecated */
export const unescapeHtml = coreUtils.unescapeHtml;
/** @deprecated */
export const randomSecureToken = coreUtils.randomSecureToken;