scheduled erasure of attachments WIP

db/migrations/0217__attachments.sql

@@ -10,9 +10,12 @@ CREATE TABLE IF NOT EXISTS "attachments"
     blobId    TEXT DEFAULT null,
     dateModified TEXT NOT NULL,
     utcDateModified TEXT not null,
-    utcDateScheduledForDeletionSince TEXT DEFAULT NULL,
+    utcDateScheduledForErasureSince TEXT DEFAULT NULL,
     isDeleted    INT  not null,
     deleteId    TEXT DEFAULT NULL);
 
 CREATE INDEX IDX_attachments_parentId_role
     on attachments (parentId, role);
+
+CREATE INDEX IDX_attachments_utcDateScheduledForErasureSince
+    on attachments (utcDateScheduledForErasureSince);

db/schema.sql

@@ -121,7 +121,7 @@ CREATE TABLE IF NOT EXISTS "attachments"
     blobId    TEXT DEFAULT null,
     dateModified TEXT NOT NULL,
     utcDateModified TEXT not null,
-    utcDateScheduledForDeletionSince TEXT DEFAULT NULL,
+    utcDateScheduledForErasureSince TEXT DEFAULT NULL,
     isDeleted    INT  not null,
     deleteId    TEXT DEFAULT NULL);
 CREATE INDEX IDX_attachments_parentId_role

package.json

@@ -33,10 +33,10 @@
   "dependencies": {
     "@braintree/sanitize-url": "6.0.2",
     "@electron/remote": "2.0.9",
-    "@excalidraw/excalidraw": "0.14.2",
+    "@excalidraw/excalidraw": "0.15.2",
     "archiver": "5.3.1",
     "async-mutex": "0.4.0",
-    "axios": "1.3.5",
+    "axios": "1.3.6",
     "better-sqlite3": "7.4.5",
     "chokidar": "3.5.3",
     "cls-hooked": "4.2.2",
@@ -51,13 +51,13 @@
     "electron-debug": "3.2.0",
     "electron-dl": "3.5.0",
     "electron-window-state": "5.0.3",
-    "escape-html": "^1.0.3",
+    "escape-html": "1.0.3",
     "express": "4.18.2",
     "express-partial-content": "1.0.2",
     "express-rate-limit": "6.7.0",
     "express-session": "1.17.3",
     "fs-extra": "11.1.1",
-    "helmet": "6.1.2",
+    "helmet": "6.1.5",
     "html": "1.0.0",
     "html2plaintext": "2.1.4",
     "http-proxy-agent": "5.0.0",
@@ -71,7 +71,7 @@
     "jsdom": "21.1.1",
     "mime-types": "2.1.35",
     "multer": "1.4.5-lts.1",
-    "node-abi": "3.35.0",
+    "node-abi": "3.40.0",
     "normalize-strings": "1.1.1",
     "open": "8.4.1",
     "rand-token": "1.0.1",
@@ -83,7 +83,7 @@
     "sanitize-filename": "1.6.3",
     "sanitize-html": "2.10.0",
     "sax": "1.2.4",
-    "semver": "7.3.8",
+    "semver": "7.5.0",
     "serve-favicon": "2.5.0",
     "session-file-store": "1.5.0",
     "stream-throttle": "0.1.3",
@@ -117,7 +117,7 @@
     "prettier": "2.8.7",
     "nodemon": "^2.0.22",
     "rcedit": "3.0.1",
-    "webpack": "5.78.0",
+    "webpack": "5.80.0",
     "webpack-cli": "5.0.1"
   },
   "optionalDependencies": {

src/becca/entities/battachment.js

@@ -20,7 +20,7 @@ class BAttachment extends AbstractBeccaEntity {
     static get entityName() { return "attachments"; }
     static get primaryKeyName() { return "attachmentId"; }
     static get hashedProperties() { return ["attachmentId", "parentId", "role", "mime", "title", "blobId",
-                                            "utcDateScheduledForDeletionSince", "utcDateModified"]; }
+                                            "utcDateScheduledForErasureSince", "utcDateModified"]; }
 
     constructor(row) {
         super();
@@ -56,7 +56,7 @@ class BAttachment extends AbstractBeccaEntity {
         /** @type {string} */
         this.utcDateModified = row.utcDateModified;
         /** @type {string} */
-        this.utcDateScheduledForDeletionSince = row.utcDateScheduledForDeletionSince;
+        this.utcDateScheduledForErasureSince = row.utcDateScheduledForErasureSince;
     }
 
     /** @returns {BAttachment} */
@@ -68,7 +68,7 @@ class BAttachment extends AbstractBeccaEntity {
             title: this.title,
             blobId: this.blobId,
             isProtected: this.isProtected,
-            utcDateScheduledForDeletionSince: this.utcDateScheduledForDeletionSince
+            utcDateScheduledForErasureSince: this.utcDateScheduledForErasureSince
         });
     }
 
@@ -171,7 +171,7 @@ class BAttachment extends AbstractBeccaEntity {
             isDeleted: false,
             dateModified: this.dateModified,
             utcDateModified: this.utcDateModified,
-            utcDateScheduledForDeletionSince: this.utcDateScheduledForDeletionSince
+            utcDateScheduledForErasureSince: this.utcDateScheduledForErasureSince
         };
     }
 

src/public/app/entities/fattachment.js

@@ -21,7 +21,7 @@ class FAttachment {
         /** @type {string} */
         this.utcDateModified = row.utcDateModified;
         /** @type {string} */
-        this.utcDateScheduledForDeletionSince = row.utcDateScheduledForDeletionSince;
+        this.utcDateScheduledForErasureSince = row.utcDateScheduledForErasureSince;
 
         this.froca.attachments[this.attachmentId] = this;
     }

src/public/app/services/utils.js

@@ -27,6 +27,39 @@ function formatTimeWithSeconds(date) {
     return `${padNum(date.getHours())}:${padNum(date.getMinutes())}:${padNum(date.getSeconds())}`;
 }
 
+function formatTimeInterval(ms) {
+    const seconds = Math.round(ms / 1000);
+    const minutes = Math.floor(seconds / 60);
+    const hours = Math.floor(minutes / 60);
+    const days = Math.floor(hours / 24);
+    const plural = (count, name) => `${count} ${name}${count > 1 ? 's' : ''}`;
+    const segments = [];
+
+    if (days > 0) {
+        segments.push(plural(days, 'day'));
+    }
+
+    if (days < 2) {
+        if (hours % 24 > 0) {
+            segments.push(plural(hours % 24, 'hour'));
+        }
+
+        if (hours < 4) {
+            if (minutes % 60 > 0) {
+                segments.push(plural(minutes % 60, 'minute'));
+            }
+
+            if (minutes < 5) {
+                if (seconds % 60 > 0) {
+                    segments.push(plural(seconds % 60, 'second'));
+                }
+            }
+        }
+    }
+
+    return segments.join(", ");
+}
+
 // this is producing local time!
 function formatDate(date) {
     //    return padNum(date.getDate()) + ". " + padNum(date.getMonth() + 1) + ". " + date.getFullYear();
@@ -489,6 +522,7 @@ export default {
     formatDate,
     formatDateISO,
     formatDateTime,
+    formatTimeInterval,
     formatSize,
     localNowDateTime,
     now,

src/public/app/widgets/attachment_detail.js

@@ -2,6 +2,7 @@ import utils from "../services/utils.js";
 import AttachmentActionsWidget from "./buttons/attachments_actions.js";
 import BasicWidget from "./basic_widget.js";
 import server from "../services/server.js";
+import options from "../services/options.js";
 
 const TPL = `
 <div class="attachment-detail">
@@ -44,6 +45,10 @@ const TPL = `
             max-width: 90%; 
             object-fit: contain;
         }
+        
+        .attachment-detail-wrapper.scheduled-for-deletion .attachment-content img {
+            filter: contrast(10%);
+        }
     </style>
 
     <div class="attachment-detail-wrapper">
@@ -54,6 +59,8 @@ const TPL = `
             <div class="attachment-actions-container"></div>
         </div>
         
+        <div class="attachment-deletion-warning alert alert-info"></div>
+        
         <div class="attachment-content"></div>
     </div>
 </div>`;
@@ -100,15 +107,29 @@ export default class AttachmentDetailWidget extends BasicWidget {
                 .text(this.attachment.title);
         }
 
-        const {utcDateScheduledForDeletionSince} = this.attachment;
+        const $deletionWarning = this.$wrapper.find('.attachment-deletion-warning');
+        const {utcDateScheduledForErasureSince} = this.attachment;
 
-        if (utcDateScheduledForDeletionSince) {
-            const scheduledSinceTimestamp = utils.parseDate(utcDateScheduledForDeletionSince)?.getTime();
-            const interval = 3600 * 1000;
-            const deletionTimestamp = scheduledSinceTimestamp + interval;
-            const willBeDeletedInSeconds = Math.round((deletionTimestamp - Date.now()) / 1000);
+        if (utcDateScheduledForErasureSince) {
+            this.$wrapper.addClass("scheduled-for-deletion");
 
-            this.$wrapper.find('.attachment-title').append(`Will be deleted in ${willBeDeletedInSeconds} seconds.`);
+            const scheduledSinceTimestamp = utils.parseDate(utcDateScheduledForErasureSince)?.getTime();
+            const intervalMs = options.getInt('eraseUnusedImageAttachmentsAfterSeconds') * 1000;
+            const deletionTimestamp = scheduledSinceTimestamp + intervalMs;
+            const willBeDeletedInMs = deletionTimestamp - Date.now();
+
+            $deletionWarning.show();
+
+            if (willBeDeletedInMs >= 60000) {
+                $deletionWarning.text(`This attachment will be deleted in ${utils.formatTimeInterval(willBeDeletedInMs)}`);
+            } else {
+                $deletionWarning.text(`This attachment will be deleted soon`);
+            }
+
+            $deletionWarning.append(", because the image attachment is not used. To prevent deletion, add the image back into the note.");
+        } else {
+            this.$wrapper.removeClass("scheduled-for-deletion");
+            $deletionWarning.hide();
         }
 
         this.$wrapper.find('.attachment-details')

src/routes/api/options.js

@@ -61,7 +61,8 @@ const ALLOWED_OPTIONS = new Set([
     'downloadImagesAutomatically',
     'minTocHeadings',
     'checkForUpdates',
-    'disableTray'
+    'disableTray',
+    'eraseUnusedImageAttachmentsAfterSeconds'
 ]);
 
 function getOptions() {

src/services/notes.js

@@ -21,6 +21,7 @@ const htmlSanitizer = require("./html_sanitizer");
 const ValidationError = require("../errors/validation_error");
 const noteTypesService = require("./note_types");
 const fs = require("fs");
+const BAttachment = require("../becca/entities/battachment");
 
 /** @param {BNote} parentNote */
 function getNewNotePosition(parentNote) {
@@ -342,17 +343,17 @@ function checkImageAttachments(note, content) {
     let match;
 
     while (match = re.exec(content)) {
-        foundAttachmentIds.push(match[1]);
+        foundAttachmentIds.add(match[1]);
     }
 
     for (const attachment of note.getAttachmentByRole('image')) {
         const imageInContent = foundAttachmentIds.has(attachment.attachmentId);
 
-        if (attachment.utcDateScheduledForDeletionSince && imageInContent) {
-            attachment.utcDateScheduledForDeletionSince = null;
+        if (attachment.utcDateScheduledForErasureSince && imageInContent) {
+            attachment.utcDateScheduledForErasureSince = null;
             attachment.save();
-        } else if (!attachment.utcDateScheduledForDeletionSince && !imageInContent) {
-            attachment.utcDateScheduledForDeletionSince = dateUtils.utcNowDateTime();
+        } else if (!attachment.utcDateScheduledForErasureSince && !imageInContent) {
+            attachment.utcDateScheduledForErasureSince = dateUtils.utcNowDateTime();
             attachment.save();
         }
     }
@@ -841,6 +842,33 @@ function eraseAttributes(attributeIdsToErase) {
     log.info(`Erased attributes: ${JSON.stringify(attributeIdsToErase)}`);
 }
 
+function eraseAttachments(attachmentIdsToErase) {
+    if (attachmentIdsToErase.length === 0) {
+        return;
+    }
+
+    sql.executeMany(`DELETE FROM attachments WHERE attachmentId IN (???)`, attachmentIdsToErase);
+
+    setEntityChangesAsErased(sql.getManyRows(`SELECT * FROM entity_changes WHERE entityName = 'attachments' AND entityId IN (???)`, attachmentIdsToErase));
+
+    log.info(`Erased attachments: ${JSON.stringify(attachmentIdsToErase)}`);
+}
+
+function eraseUnusedBlobs() {
+    const unusedBlobIds = sql.getColumn(`
+        SELECT blobId
+        FROM blobs
+        LEFT JOIN notes ON notes.blobId = blobs.blobId
+        LEFT JOIN attachments ON attachments.blobId = blobs.blobId
+        WHERE notes.noteId IS NULL AND attachments.attachmentId IS NULL`);
+
+    sql.executeMany(`DELETE FROM blobs WHERE blobId IN (???)`, unusedBlobIds);
+
+    setEntityChangesAsErased(sql.getManyRows(`SELECT * FROM entity_changes WHERE entityName = 'blobs' AND entityId IN (???)`, unusedBlobIds));
+
+    log.info(`Erased unused blobs: ${JSON.stringify(unusedBlobIds)}`);
+}
+
 function eraseDeletedEntities(eraseEntitiesAfterTimeInSeconds = null) {
     // this is important also so that the erased entity changes are sent to the connected clients
     sql.transactional(() => {
@@ -861,6 +889,12 @@ function eraseDeletedEntities(eraseEntitiesAfterTimeInSeconds = null) {
         const attributeIdsToErase = sql.getColumn("SELECT attributeId FROM attributes WHERE isDeleted = 1 AND utcDateModified <= ?", [dateUtils.utcDateTimeStr(cutoffDate)]);
 
         eraseAttributes(attributeIdsToErase);
+
+        const attachmentIdsToErase = sql.getColumn("SELECT attachmentId FROM attachments WHERE isDeleted = 1 AND utcDateModified <= ?", [dateUtils.utcDateTimeStr(cutoffDate)]);
+
+        eraseAttachments(attachmentIdsToErase);
+
+        eraseUnusedBlobs();
     });
 }
 
@@ -1013,11 +1047,21 @@ function getNoteIdMapping(origNote) {
     return noteIdMapping;
 }
 
+function eraseScheduledAttachments() {
+    const eraseIntervalSeconds = optionService.getOptionInt('eraseUnusedImageAttachmentsAfterSeconds');
+    const cutOffDate = dateUtils.utcDateTimeStr(new Date(Date.now() - (eraseIntervalSeconds * 1000)));
+    const attachmentIdsToErase = sql.getColumn('SELECT attachmentId FROM attachments WHERE utcDateScheduledForErasureSince < ?', [cutOffDate]);
+
+    eraseAttachments(attachmentIdsToErase);
+}
+
 sqlInit.dbReady.then(() => {
     // first cleanup kickoff 5 minutes after startup
     setTimeout(cls.wrap(() => eraseDeletedEntities()), 5 * 60 * 1000);
+    setTimeout(cls.wrap(() => eraseScheduledAttachments()), 6 * 60 * 1000);
 
     setInterval(cls.wrap(() => eraseDeletedEntities()), 4 * 3600 * 1000);
+    setInterval(cls.wrap(() => eraseScheduledAttachments()), 3600 * 1000);
 });
 
 module.exports = {

src/services/options_init.js

@@ -88,6 +88,7 @@ const defaultOptions = [
     { name: 'minTocHeadings', value: '5', isSynced: true },
     { name: 'checkForUpdates', value: 'true', isSynced: true },
     { name: 'disableTray', value: 'false', isSynced: false },
+    { name: 'eraseUnusedImageAttachmentsAfterSeconds', value: '86400', isSynced: false },
 ];
 
 function initStartupOptions() {