diff --git a/install/package.json b/install/package.json index 43d32072b5..bd522d6551 100644 --- a/install/package.json +++ b/install/package.json @@ -65,6 +65,7 @@ "cronstrue": "3.14.0", "cropperjs": "1.6.2", "csrf-sync": "4.2.1", + "csv-parse": "6.2.1", "daemon": "1.1.0", "diff": "8.0.4", "esbuild": "0.27.4", diff --git a/public/language/en-GB/admin/settings/activitypub.json b/public/language/en-GB/admin/settings/activitypub.json index 677af75cd7..6c65323ce5 100644 --- a/public/language/en-GB/admin/settings/activitypub.json +++ b/public/language/en-GB/admin/settings/activitypub.json @@ -41,6 +41,17 @@ "relays.state-2": "Active", "relays.errors.invalid-url": "Please enter a valid URL", + "blocklists": "Third-party Blocklists", + "blocklists-help": "For the safety of you and your users, maintaining a blocklist is an essential part of any trust & safety effort when working with content from outside the local moderation scope. NodeBB ships with some recommended defaults, and these can be customized here.", + "blocklists-default": "NodeBB ships with two blocklists by default, the IFTAS Do Not Interact Denylist and the IFTAS Abandoned and Unmanaged Domain Denylist.", + "blocklists.url": "URL", + "blocklists.count": "Domains", + "blocklists.add": "Add New Blocklist", + "blocklists.add-help": "Enter the URL of the blocklist you would like to add. NodeBB understands blocklists that match the format used by IFTAS.", + "blocklists.refreshed": "Blocklist refreshed — it now contains %1 entries", + "blocklists.view.title": "View Blocklist", + "blocklists.view.intro": "These are the %1 domain(s) blocked by this blocklist:", + "server-filtering": "Filtering", "count": "This NodeBB is currently aware of %1 server(s)", "server.filter-help": "Specify servers you would like to bar from federating with your NodeBB. Alternatively, you may opt to selectively allow federation with specific servers, instead. Both options are supported, although they are mutually exclusive.", diff --git a/public/openapi/components/schemas/admin/blocklists.yaml b/public/openapi/components/schemas/admin/blocklists.yaml new file mode 100644 index 0000000000..75c4555432 --- /dev/null +++ b/public/openapi/components/schemas/admin/blocklists.yaml @@ -0,0 +1,14 @@ +BlocklistObject: + type: object + properties: + url: + type: string + description: The blocklist URL + example: https://example.org/blocklist + count: + type: number + description: The number of blocked domains on the list +BlocklistArray: + type: array + items: + $ref: '#/BlocklistObject' \ No newline at end of file diff --git a/public/openapi/read/admin/federation/safety.yaml b/public/openapi/read/admin/federation/safety.yaml index 73015d3f8c..2ddd916eb7 100644 --- a/public/openapi/read/admin/federation/safety.yaml +++ b/public/openapi/read/admin/federation/safety.yaml @@ -13,6 +13,8 @@ get: properties: title: type: string + blocklists: + $ref: ../../../components/schemas/admin/blocklists.yaml#/BlocklistArray instanceCount: type: number - $ref: ../../../components/schemas/CommonProps.yaml#/CommonProps \ No newline at end of file diff --git a/public/openapi/write.yaml b/public/openapi/write.yaml index 0741c799ed..e03ce4f122 100644 --- a/public/openapi/write.yaml +++ b/public/openapi/write.yaml @@ -286,6 +286,12 @@ paths: $ref: 'write/admin/activitypub/relays.yaml' /admin/activitypub/relays/{url}: $ref: 'write/admin/activitypub/relays/url.yaml' + /admin/activitypub/blocklists: + $ref: 'write/admin/activitypub/blocklists.yaml' + /admin/activitypub/blocklists/{url}: + $ref: 'write/admin/activitypub/blocklists/url.yaml' + /admin/activitypub/blocklists/{url}/refresh: + $ref: 'write/admin/activitypub/blocklists/url/refresh.yaml' /files/: $ref: 'write/files.yaml' /files/folder: diff --git a/public/openapi/write/admin/activitypub/blocklists.yaml b/public/openapi/write/admin/activitypub/blocklists.yaml new file mode 100644 index 0000000000..00bee2d995 --- /dev/null +++ b/public/openapi/write/admin/activitypub/blocklists.yaml @@ -0,0 +1,28 @@ +post: + tags: + - admin + summary: add blocklist + description: This operation add a new blocklist + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + url: + type: string + description: The blocklist url + example: https://example.org/blocklist + responses: + '200': + description: blocklist successfully added + content: + application/json: + schema: + type: object + properties: + status: + $ref: ../../../components/schemas/Status.yaml#/Status + response: + $ref: ../../../components/schemas/admin/blocklists.yaml#/BlocklistArray diff --git a/public/openapi/write/admin/activitypub/blocklists/url.yaml b/public/openapi/write/admin/activitypub/blocklists/url.yaml new file mode 100644 index 0000000000..59b6bab90e --- /dev/null +++ b/public/openapi/write/admin/activitypub/blocklists/url.yaml @@ -0,0 +1,55 @@ +get: + tags: + - admin + summary: view blocklist + description: This operation retrieves the domains in a specific blocklist + parameters: + - name: url + in: path + required: true + schema: + type: string + example: https%3A%2F%2Fexample.org%2Fblocklist + responses: + '200': + description: blocklist successfully retrieved + content: + application/json: + schema: + type: object + properties: + status: + $ref: ../../../../components/schemas/Status.yaml#/Status + response: + type: object + properties: + domains: + type: array + items: + type: string + count: + type: number +delete: + tags: + - admin + summary: delete blocklist + description: This operation removes a previously-added blocklist + parameters: + - in: path + name: url + schema: + type: string + required: true + example: https%3A%2F%2Fexample.org%2Fblocklist + responses: + '200': + description: blocklist successfully deleted + content: + application/json: + schema: + type: object + properties: + status: + $ref: ../../../../components/schemas/Status.yaml#/Status + response: + $ref: ../../../../components/schemas/admin/blocklists.yaml#/BlocklistArray \ No newline at end of file diff --git a/public/openapi/write/admin/activitypub/blocklists/url/refresh.yaml b/public/openapi/write/admin/activitypub/blocklists/url/refresh.yaml new file mode 100644 index 0000000000..205fa5b3b1 --- /dev/null +++ b/public/openapi/write/admin/activitypub/blocklists/url/refresh.yaml @@ -0,0 +1,30 @@ +post: + tags: + - admin + summary: refresh blocklist + description: This operation refreshes the blocklist domains + parameters: + - name: url + in: path + required: true + schema: + type: string + example: https%3A%2F%2Fexample.org%2Fblocklist + responses: + '200': + description: blocklist successfully refreshed + content: + application/json: + schema: + type: object + properties: + status: + $ref: ../../../../../components/schemas/Status.yaml#/Status + response: + type: object + properties: + blocklists: + $ref: ../../../../../components/schemas/admin/blocklists.yaml#/BlocklistArray + count: + type: number + description: The number of domains managed by this blocklist diff --git a/public/src/admin/federation/safety.js b/public/src/admin/federation/safety.js new file mode 100644 index 0000000000..f32da91904 --- /dev/null +++ b/public/src/admin/federation/safety.js @@ -0,0 +1,113 @@ +'use strict'; + +import { get, post, del } from 'api'; +import { error } from 'alerts'; +import { render } from 'benchpress'; +import { alert, dialog } from 'bootbox'; + +export function init() { + setupBlocklists(); +}; + +function setupBlocklists() { + const blocklistsEl = document.getElementById('blocklists'); + if (blocklistsEl) { + blocklistsEl.addEventListener('click', (e) => { + const subselector = e.target.closest('[data-action]'); + if (subselector) { + const url = subselector.closest('tr').getAttribute('data-url'); + const action = subselector.getAttribute('data-action'); + switch (action) { + case 'blocklists.add': { + throwModal(); + break; + } + + case 'blocklists.refresh': { + refresh(url); + break; + } + + case 'blocklists.view': { + view(url); + break; + } + + case 'blocklists.remove': { + del(`/admin/activitypub/blocklists/${encodeURIComponent(url)}`, {}) + .then(renderList) + .catch(error); + } + } + } + }); + } +} + +async function renderList(blocklists) { + const html = await app.parseAndTranslate('admin/federation/safety', 'blocklists', { blocklists }); + const tbodyEl = document.querySelector('#blocklists tbody'); + if (tbodyEl) { + $(tbodyEl).html(html); + } +} + +function throwModal() { + render('admin/partials/activitypub/blocklists', {}).then(function (html) { + const submit = function () { + const formEl = modal.find('form').get(0); + if (!formEl.reportValidity()) { + return false; + } + + const payload = Object.fromEntries(new FormData(formEl)); + post('/admin/activitypub/blocklists', payload).then(async (data) => { + const html = await app.parseAndTranslate('admin/federation/safety', 'blocklists', { blocklists: data }); + const tbodyEl = document.querySelector('#blocklists tbody'); + if (tbodyEl) { + $(tbodyEl).html(html); + } + }).catch(error); + }; + const modal = bootbox.dialog({ + title: '[[admin/settings/activitypub:blocklists.add]]', + message: html, + buttons: { + save: { + label: '[[global:save]]', + className: 'btn-primary', + callback: submit, + }, + }, + }); + + modal.on('shown.bs.modal', function () { + modal.find('input').focus(); + }); + }); +} + +async function refresh(url) { + try { + const data = await post(`/admin/activitypub/blocklists/${encodeURIComponent(url)}/refresh`); + alert(`[[admin/settings/activitypub:blocklists.refreshed, ${data.count}]]`); + renderList(data.blocklists); + } catch (e) { + error(e); + } +} + +async function view(url) { + console.log('ehjre'); + try { + const { domains, count } = await get(`/admin/activitypub/blocklists/${encodeURIComponent(url)}`); + dialog({ + title: '[[admin/settings/activitypub:blocklists.view.title]]', + message: `\ +

[[admin/settings/activitypub:blocklists.view.intro, ${count}]]

\ + ', + }); + } catch (e) { + error(e); + } +} \ No newline at end of file diff --git a/src/activitypub/blocklists.js b/src/activitypub/blocklists.js new file mode 100644 index 0000000000..86424d7856 --- /dev/null +++ b/src/activitypub/blocklists.js @@ -0,0 +1,78 @@ +'use strict'; + +const { parse } = require('csv-parse/sync'); + +const db = require('../database'); +const request = require('../request'); +const activitypub = module.parent.exports; + +const Blocklists = module.exports; + +Blocklists.list = async () => { + const blocklists = await db.getSortedSetMembers('blocklists'); + const counts = await db.sortedSetsCard(blocklists.map(blocklist => `blocklist:${blocklist}`)); + + return blocklists.map((url, idx) => { + return { url, count: counts[idx] }; + }); +}; + +Blocklists.get = async (url) => { + const domains = await db.getSortedSetMembers(`blocklist:${url}`); + + return { + domains, + count: domains.length, + }; +}; + +Blocklists.add = async (url) => { + const now = Date.now(); + + await Promise.all([ + db.sortedSetAdd('blocklists', now, url), + Blocklists.refresh(url), + ]); +}; + +Blocklists.remove = async (url) => { + await Promise.all([ + db.sortedSetRemove('blocklists', url), + db.delete(`blocklist:${url}`), + ]); +}; + +Blocklists.refresh = async (url) => { + activitypub.helpers.log(`[blocklists/refresh] Processing ${url}`); + + const { body: csv } = await request.get(url); + let records; + try { + records = parse(csv, { + columns: true, + skip_empty_lines: true, + trim: true, + }); + if (!records.length) { + return 0; + } + } catch (e) { + return 0; + } + + await db.sortedSetAdd( + `blocklist:${url}`, + records.map(entry => entry['#severity'] === 'silence' ? 2 : 1), + records.map(entry => entry['#domain']) + ); + + return records.length; +}; + +Blocklists.check = async (domain) => { + const blocklists = await Blocklists.list(); + let present = await db.isMemberOfSortedSets(blocklists.map(({ url }) => `blocklist:${url}`), domain); + present = present.reduce((memo, present) => memo || present, false); + + return !present; +}; \ No newline at end of file diff --git a/src/activitypub/index.js b/src/activitypub/index.js index 37750664fe..d2f299ecfc 100644 --- a/src/activitypub/index.js +++ b/src/activitypub/index.js @@ -64,6 +64,7 @@ ActivityPub.notes = require('./notes'); ActivityPub.contexts = require('./contexts'); ActivityPub.actors = require('./actors'); ActivityPub.instances = require('./instances'); +ActivityPub.blocklists = require('./blocklists'); ActivityPub.feps = require('./feps'); ActivityPub.rules = require('./rules'); ActivityPub.relays = require('./relays'); @@ -134,11 +135,11 @@ ActivityPub.resolveInboxes = async (ids) => { // Filter out blocked instances const blocked = []; - inboxArr = inboxArr.filter((inbox) => { + const allowed = await Promise.all(inboxArr.map(async (inbox) => { let allowed = false; try { const { hostname } = new URL(inbox); - allowed = ActivityPub.instances.isAllowed(hostname); + allowed = await ActivityPub.instances.isAllowed(hostname); if (!allowed) { blocked.push(inbox); } @@ -147,7 +148,8 @@ ActivityPub.resolveInboxes = async (ids) => { } return allowed; - }); + })); + inboxArr = inboxArr.filter((_, idx) => allowed[idx]); if (blocked.length) { ActivityPub.helpers.log(`[activitypub/resolveInboxes] Not delivering to blocked instances: ${blocked.join(', ')}`); } @@ -306,7 +308,7 @@ ActivityPub.get = async (type, id, uri, options) => { } const { hostname } = new URL(uri); - const allowed = ActivityPub.instances.isAllowed(hostname); + const allowed = await ActivityPub.instances.isAllowed(hostname); if (!allowed) { ActivityPub.helpers.log(`[activitypub/get] Not retrieving ${uri}, domain is blocked.`); const e = new Error(`[[error:activitypub.get-failed]]`); diff --git a/src/activitypub/instances.js b/src/activitypub/instances.js index 64502c0998..4302c17f48 100644 --- a/src/activitypub/instances.js +++ b/src/activitypub/instances.js @@ -2,6 +2,7 @@ const meta = require('../meta'); const db = require('../database'); +const activitypub = module.parent.exports; const Instances = module.exports; @@ -11,9 +12,15 @@ Instances.log = async (domain) => { Instances.getCount = async () => db.sortedSetCard('instances:lastSeen'); -Instances.isAllowed = (domain) => { +Instances.isAllowed = async (domain) => { + const allowed = await activitypub.blocklists.check(domain); let { activitypubFilter: type, activitypubFilterList: list } = meta.config; + + if (!allowed && !type) { + return allowed; + } + list = new Set(String(list).split('\n')); // eslint-disable-next-line no-bitwise - return list.has(domain) ^ !type; + return allowed || (list.has(domain) ^ !type); }; diff --git a/src/activitypub/jobs.js b/src/activitypub/jobs.js index ca403a8045..a8544f4c02 100644 --- a/src/activitypub/jobs.js +++ b/src/activitypub/jobs.js @@ -50,6 +50,18 @@ Jobs.start = async () => { runOnInit: false, onTick: async () => await tryCronJob(backfill), }); + + await cron.addJob({ + name: 'ap:blocklist:refresh', + cronTime: '15 0 * * *', + runOnInit: false, + onTick: async () => { + const lists = await activitypub.blocklists.list(); + await Promise.all(lists.map(({ url }) => { + return activitypub.blocklists.refresh(url); + })); + }, + }); }; async function retryFailedMessages() { diff --git a/src/controllers/admin/federation.js b/src/controllers/admin/federation.js index 21ed25885f..d089d6d509 100644 --- a/src/controllers/admin/federation.js +++ b/src/controllers/admin/federation.js @@ -44,9 +44,11 @@ federationController.pruning = function (req, res) { federationController.safety = async function (req, res) { const instanceCount = await activitypub.instances.getCount(); + const blocklists = await activitypub.blocklists.list(); res.render(`admin/federation/safety`, { title: '[[admin/menu:federation/safety]]', + blocklists, instanceCount, }); }; diff --git a/src/controllers/write/admin.js b/src/controllers/write/admin.js index 65621bc853..105b7f5247 100644 --- a/src/controllers/write/admin.js +++ b/src/controllers/write/admin.js @@ -126,3 +126,36 @@ Admin.activitypub.removeRelay = async (req, res) => { await activitypub.relays.remove(url); helpers.formatApiResponse(200, res, await activitypub.relays.list()); }; + + +Admin.activitypub.addBlocklist = async (req, res, next) => { + const { url } = req.body; + if (!url) { + return next(); + } + + await activitypub.blocklists.add(url); + helpers.formatApiResponse(200, res, await activitypub.blocklists.list()); +}; + +Admin.activitypub.viewBlocklist = async (req, res) => { + const { url } = req.params; + + helpers.formatApiResponse(200, res, await activitypub.blocklists.get(url)); +}; + +Admin.activitypub.removeBlocklist = async (req, res) => { + const { url } = req.params; + + await activitypub.blocklists.remove(url); + helpers.formatApiResponse(200, res, await activitypub.blocklists.list()); +}; + +Admin.activitypub.refreshBlocklist = async (req, res) => { + const { url } = req.params; + + const count = await activitypub.blocklists.refresh(url); + const blocklists = await activitypub.blocklists.list(); + + helpers.formatApiResponse(200, res, { blocklists, count }); +}; diff --git a/src/install.js b/src/install.js index c360868521..9724063df4 100644 --- a/src/install.js +++ b/src/install.js @@ -244,6 +244,12 @@ async function setupDefaultConfigs() { await meta.configs.setOnEmpty(defaults); await meta.configs.init(); + + const db = require('./database'); + await db.sortedSetAdd('blocklists', [Date.now(), Date.now()], [ + 'https://about.iftas.org/wp-content/uploads/2025/10/iftas-dni-latest.csv', + 'https://about.iftas.org/wp-content/uploads/2025/10/iftas-abandoned-unmanaged-latest.csv', + ]); } async function enableDefaultTheme() { diff --git a/src/middleware/activitypub.js b/src/middleware/activitypub.js index 9ea81c0035..0a8a89324d 100644 --- a/src/middleware/activitypub.js +++ b/src/middleware/activitypub.js @@ -105,7 +105,7 @@ middleware.assertPayload = helpers.try(async function (req, res, next) { // Domain check const { hostname } = new URL(actor); - const allowed = activitypub.instances.isAllowed(hostname); + const allowed = await activitypub.instances.isAllowed(hostname); if (!allowed) { activitypub.helpers.log(`[middleware/activitypub] Blocked incoming activity from ${hostname}.`); return res.sendStatus(403); diff --git a/src/routes/write/admin.js b/src/routes/write/admin.js index 2121953efe..caac9e9b3e 100644 --- a/src/routes/write/admin.js +++ b/src/routes/write/admin.js @@ -30,6 +30,10 @@ module.exports = function () { setupApiRoute(router, 'put', '/activitypub/rules/order', [...middlewares, middleware.checkRequired.bind(null, ['rids'])], controllers.write.admin.activitypub.reorderRules); setupApiRoute(router, 'post', '/activitypub/relays', [...middlewares, middleware.checkRequired.bind(null, ['url'])], controllers.write.admin.activitypub.addRelay); setupApiRoute(router, 'delete', '/activitypub/relays/:url', [...middlewares], controllers.write.admin.activitypub.removeRelay); + setupApiRoute(router, 'post', '/activitypub/blocklists', [...middlewares, middleware.checkRequired.bind(null, ['url'])], controllers.write.admin.activitypub.addBlocklist); + setupApiRoute(router, 'get', '/activitypub/blocklists/:url', [...middlewares], controllers.write.admin.activitypub.viewBlocklist); + setupApiRoute(router, 'delete', '/activitypub/blocklists/:url', [...middlewares], controllers.write.admin.activitypub.removeBlocklist); + setupApiRoute(router, 'post', '/activitypub/blocklists/:url/refresh', [...middlewares], controllers.write.admin.activitypub.refreshBlocklist); return router; }; diff --git a/src/upgrades/4.11.0/subscribe-to-default-blocklists.js b/src/upgrades/4.11.0/subscribe-to-default-blocklists.js new file mode 100644 index 0000000000..feb0af76c7 --- /dev/null +++ b/src/upgrades/4.11.0/subscribe-to-default-blocklists.js @@ -0,0 +1,14 @@ +'use strict'; + +const activitypub = require('../../activitypub'); + +module.exports = { + name: 'Subscribe to IFTAS DNI and AUD denylists', + timestamp: Date.UTC(2026, 2, 23), + method: async () => { + await Promise.all([ + activitypub.blocklists.add('https://about.iftas.org/wp-content/uploads/2025/10/iftas-dni-latest.csv'), + activitypub.blocklists.add('https://about.iftas.org/wp-content/uploads/2025/10/iftas-abandoned-unmanaged-latest.csv'), + ]); + }, +}; diff --git a/src/views/admin/federation/safety.tpl b/src/views/admin/federation/safety.tpl index 9c92085732..b7fe38c49d 100644 --- a/src/views/admin/federation/safety.tpl +++ b/src/views/admin/federation/safety.tpl @@ -3,6 +3,45 @@
+
+
[[admin/settings/activitypub:blocklists]]
+
+

[[admin/settings/activitypub:blocklists-help]]

+

[[admin/settings/activitypub:blocklists-default]]

+
+ + + + + + + + {{{ each blocklists }}} + + + + + + {{{ end }}} + + + + + + +
[[admin/settings/activitypub:blocklists.url]][[admin/settings/activitypub:blocklists.count]]
{./url}{./count} +
+ + + +
+
+ +
+
+
+
+
[[admin/settings/activitypub:server-filtering]]
diff --git a/src/views/admin/partials/activitypub/blocklists.tpl b/src/views/admin/partials/activitypub/blocklists.tpl new file mode 100644 index 0000000000..76ad526397 --- /dev/null +++ b/src/views/admin/partials/activitypub/blocklists.tpl @@ -0,0 +1,10 @@ +

[[admin/settings/activitypub:blocklists.add-help]]

+ +
+ + +
+ + +
+
\ No newline at end of file