diff --git a/public/src/client/category.js b/public/src/client/category.js index a2a5d579ad..515a871262 100644 --- a/public/src/client/category.js +++ b/public/src/client/category.js @@ -118,7 +118,7 @@ define('forum/category', [ }; Category.toBottom = async () => { - const { count } = await api.get(`/categories/${ajaxify.data.category.cid}/count`); + const { count } = await api.get(`/categories/${encodeURIComponent(ajaxify.data.category.cid)}/count`); navigator.scrollBottom(count - 1); }; @@ -127,7 +127,7 @@ define('forum/category', [ hooks.fire('action:topics.loading'); const params = utils.params(); - infinitescroll.loadMore(`/categories/${ajaxify.data.cid}/topics`, { + infinitescroll.loadMore(`/categories/${encodeURIComponent(ajaxify.data.cid)}/topics`, { after: after, direction: direction, query: params, diff --git a/src/activitypub/actors.js b/src/activitypub/actors.js index 594e4e5da8..576fd241af 100644 --- a/src/activitypub/actors.js +++ b/src/activitypub/actors.js @@ -7,6 +7,7 @@ const _ = require('lodash'); const db = require('../database'); const meta = require('../meta'); const batch = require('../batch'); +const categories = require('../categories'); const user = require('../user'); const utils = require('../utils'); const TTLCache = require('../cache/ttl'); @@ -20,15 +21,12 @@ const activitypub = module.parent.exports; const Actors = module.exports; -Actors.assert = async (ids, options = {}) => { +Actors.qualify = async (ids, options = {}) => { /** - * Ensures that the passed in ids or webfinger handles are stored in database. - * Options: - * - update: boolean, forces re-fetch/process of the resolved id - * Return one of: - * - An array of newly processed ids - * - false: if input incorrect (or webfinger handle cannot resolve) - * - true: no new IDs processed; all passed-in IDs present. + * Sanity-checks, cache handling, webfinger translations, so that only + * an array of actor uris are handled by assert/assertGroup. + * + * This method is only called by assert/assertGroup (at least in core.) */ // Handle single values @@ -85,6 +83,21 @@ Actors.assert = async (ids, options = {}) => { }); } + return ids; +}; + +Actors.assert = async (ids, options = {}) => { + /** + * Ensures that the passed in ids or webfinger handles are stored in database. + * Options: + * - update: boolean, forces re-fetch/process of the resolved id + * Return one of: + * - An array of newly processed ids + * - false: if input incorrect (or webfinger handle cannot resolve) + * - true: no new IDs processed; all passed-in IDs present. + */ + + ids = await Actors.qualify(ids, options); if (!ids.length) { return true; } @@ -96,6 +109,7 @@ Actors.assert = async (ids, options = {}) => { const urlMap = new Map(); const followersUrlMap = new Map(); const pubKeysMap = new Map(); + const categories = new Set(); let actors = await Promise.all(ids.map(async (id) => { try { activitypub.helpers.log(`[activitypub/actors] Processing ${id}`); @@ -104,8 +118,14 @@ Actors.assert = async (ids, options = {}) => { let typeOk = false; if (Array.isArray(actor.type)) { typeOk = actor.type.some(type => activitypub._constants.acceptableActorTypes.has(type)); + if (!typeOk && actor.type.some(type => activitypub._constants.acceptableGroupTypes.has(type))) { + categories.add(actor.id); + } } else { typeOk = activitypub._constants.acceptableActorTypes.has(actor.type); + if (!typeOk && activitypub._constants.acceptableGroupTypes.has(actor.type)) { + categories.add(actor.id); + } } if ( @@ -220,6 +240,142 @@ Actors.assert = async (ids, options = {}) => { return actors; }; +Actors.assertGroup = async (ids, options = {}) => { + /** + * Ensures that the passed in ids or webfinger handles are stored in database. + * Options: + * - update: boolean, forces re-fetch/process of the resolved id + * Return one of: + * - An array of newly processed ids + * - false: if input incorrect (or webfinger handle cannot resolve) + * - true: no new IDs processed; all passed-in IDs present. + */ + + ids = await Actors.qualify(ids, options); + if (!ids.length) { + return true; + } + + activitypub.helpers.log(`[activitypub/actors] Asserting ${ids.length} group(s)`); + + // NOTE: MAKE SURE EVERY DB ADDITION HAS A CORRESPONDING REMOVAL IN ACTORS.REMOVE! + + const urlMap = new Map(); + const followersUrlMap = new Map(); + const pubKeysMap = new Map(); + const users = new Set(); + let groups = await Promise.all(ids.map(async (id) => { + try { + activitypub.helpers.log(`[activitypub/actors] Processing group ${id}`); + const actor = (typeof id === 'object' && id.hasOwnProperty('id')) ? id : await activitypub.get('uid', 0, id, { cache: process.env.CI === 'true' }); + + let typeOk = false; + if (Array.isArray(actor.type)) { + typeOk = actor.type.some(type => activitypub._constants.acceptableGroupTypes.has(type)); + if (!typeOk && actor.type.some(type => activitypub._constants.acceptableActorTypes.has(type))) { + users.add(actor.id); + } + } else { + typeOk = activitypub._constants.acceptableGroupTypes.has(actor.type); + if (!typeOk && activitypub._constants.acceptableActorTypes.has(actor.type)) { + users.add(actor.id); + } + } + + if ( + !typeOk || + !activitypub._constants.requiredActorProps.every(prop => actor.hasOwnProperty(prop)) + ) { + return null; + } + + // Save url for backreference + const url = Array.isArray(actor.url) ? actor.url.shift() : actor.url; + if (url && url !== actor.id) { + urlMap.set(url, actor.id); + } + + // Save followers url for backreference + if (actor.hasOwnProperty('followers') && activitypub.helpers.isUri(actor.followers)) { + followersUrlMap.set(actor.followers, actor.id); + } + + // Public keys + pubKeysMap.set(actor.id, actor.publicKey); + + return actor; + } catch (e) { + if (e.code === 'ap_get_410') { + // const exists = await user.exists(id); + // if (exists) { + // await user.deleteAccount(id); + // } + } + + return null; + } + })); + groups = groups.filter(Boolean); // remove unresolvable actors + + // Build userData object for storage + const categoryObjs = (await activitypub.mocks.category(groups)).filter(Boolean); + const now = Date.now(); + + const bulkSet = categoryObjs.reduce((memo, category) => { + const key = `categoryRemote:${category.cid}`; + memo.push([key, category], [`${key}:keys`, pubKeysMap.get(category.cid)]); + return memo; + }, []); + if (urlMap.size) { + bulkSet.push(['remoteUrl:cid', Object.fromEntries(urlMap)]); + } + if (followersUrlMap.size) { + bulkSet.push(['followersUrl:cid', Object.fromEntries(followersUrlMap)]); + } + + const exists = await db.isSortedSetMembers('usersRemote:lastCrawled', categoryObjs.map(p => p.cid)); + const cidsForCurrent = categoryObjs.map((p, idx) => (exists[idx] ? p.cid : 0)); + const current = await categories.getCategoriesFields(cidsForCurrent, ['slug']); + const queries = categoryObjs.reduce((memo, profile, idx) => { + const { slug } = current[idx]; + + if (options.update || slug !== profile.slug) { + if (cidsForCurrent[idx] !== 0 && slug) { + // memo.searchRemove.push(['ap.preferredUsername:sorted', `${slug.toLowerCase()}:${profile.uid}`]); + memo.handleRemove.push(slug.toLowerCase()); + } + + // memo.searchAdd.push(['ap.preferredUsername:sorted', 0, `${profile.slug.toLowerCase()}:${profile.uid}`]); + memo.handleAdd[profile.slug.toLowerCase()] = profile.cid; + } + + // if (options.update || (profile.fullname && fullname !== profile.fullname)) { + // if (fullname && cidsForCurrent[idx] !== 0) { + // memo.searchRemove.push(['ap.name:sorted', `${fullname.toLowerCase()}:${profile.uid}`]); + // } + + // memo.searchAdd.push(['ap.name:sorted', 0, `${profile.fullname.toLowerCase()}:${profile.uid}`]); + // } + + return memo; + }, { /* searchRemove: [], searchAdd: [], */ handleRemove: [], handleAdd: {} }); + + // Removals + await Promise.all([ + // db.sortedSetRemoveBulk(queries.searchRemove), + db.deleteObjectFields('handle:cid', queries.handleRemove), + ]); + + await Promise.all([ + db.setObjectBulk(bulkSet), + db.sortedSetAdd('usersRemote:lastCrawled', groups.map(() => now), groups.map(p => p.uid)), + // db.sortedSetAddBulk(queries.searchAdd), + db.setObject('handle:cid', queries.handleAdd), + ]); + + return categoryObjs; +}; + Actors.getLocalFollowers = async (id) => { const response = { uids: new Set(), diff --git a/src/activitypub/index.js b/src/activitypub/index.js index 96fcbeac47..273e443221 100644 --- a/src/activitypub/index.js +++ b/src/activitypub/index.js @@ -39,7 +39,8 @@ ActivityPub._constants = Object.freeze({ acceptedPostTypes: [ 'Note', 'Page', 'Article', 'Question', 'Video', ], - acceptableActorTypes: new Set(['Application', 'Group', 'Organization', 'Person', 'Service']), + acceptableActorTypes: new Set(['Application', 'Organization', 'Person', 'Service']), + acceptableGroupTypes: new Set(['Group']), requiredActorProps: ['inbox', 'outbox'], acceptedProtocols: ['https', ...(process.env.CI === 'true' ? ['http'] : [])], acceptable: { diff --git a/src/activitypub/mocks.js b/src/activitypub/mocks.js index 7726768016..1108937eee 100644 --- a/src/activitypub/mocks.js +++ b/src/activitypub/mocks.js @@ -216,7 +216,7 @@ Mocks.profile = async (actors) => { uploadedpicture: undefined, 'cover:url': !image || typeof image === 'string' ? image : image.url, 'cover:position': '50% 50%', - aboutme: summary, + aboutme: posts.sanitize(summary), followerCount, followingCount, @@ -233,6 +233,73 @@ Mocks.profile = async (actors) => { return profiles; }; +Mocks.category = async (actors) => { + const categories = await Promise.all(actors.map(async (actor) => { + if (!actor) { + return null; + } + + const cid = actor.id; + let hostname; + let { + url, preferredUsername, /* icon, */ image, + name, summary, followers, inbox, endpoints, tag, + } = actor; + preferredUsername = slugify(preferredUsername || name); + // const { followers: followerCount, following: followingCount } = await activitypub.actors.getLocalFollowCounts(uid); + + try { + ({ hostname } = new URL(actor.id)); + } catch (e) { + return null; + } + + // No support for category avatars yet ;( + // let picture; + // if (icon) { + // picture = typeof icon === 'string' ? icon : icon.url; + // } + const iconBackgrounds = await user.getIconBackgrounds(); + let bgColor = Array.prototype.reduce.call(preferredUsername, (cur, next) => cur + next.charCodeAt(), 0); + bgColor = iconBackgrounds[bgColor % iconBackgrounds.length]; + + // Replace emoji in summary + if (tag && Array.isArray(tag)) { + tag + .filter(tag => tag.type === 'Emoji' && + isEmojiShortcode.test(tag.name) && + tag.icon && tag.icon.mediaType && tag.icon.mediaType.startsWith('image/')) + .forEach((tag) => { + summary = summary.replace(new RegExp(tag.name, 'g'), ``); + }); + } + + const payload = { + cid, + name, + handle: preferredUsername, + slug: `${preferredUsername}@${hostname}`, + description: summary, + descriptionParsed: posts.sanitize(summary), + icon: 'fa-comments', + color: '#fff', + bgColor, + backgroundImage: !image || typeof image === 'string' ? image : image.url, + // followerCount, + // followingCount, + + url, + inbox, + sharedInbox: endpoints ? endpoints.sharedInbox : null, + followersUrl: followers, + }; + + return payload; + })); + + return categories; +}; + Mocks.post = async (objects) => { let single = false; if (!Array.isArray(objects)) { diff --git a/src/categories/data.js b/src/categories/data.js index 9ad2783203..9039672775 100644 --- a/src/categories/data.js +++ b/src/categories/data.js @@ -36,8 +36,8 @@ module.exports = function (Categories) { return []; } - cids = cids.map(cid => parseInt(cid, 10)); - const keys = cids.map(cid => `category:${cid}`); + cids = cids.map(cid => (utils.isNumber(cid) ? parseInt(cid, 10) : cid)); + const keys = cids.map(cid => (utils.isNumber(cid) ? `category:${cid}` : `categoryRemote:${cid}`)); const categories = await db.getObjects(keys, fields); // Handle cid -1 diff --git a/src/categories/index.js b/src/categories/index.js index cde2f02d6a..1a254f9b15 100644 --- a/src/categories/index.js +++ b/src/categories/index.js @@ -10,6 +10,7 @@ const plugins = require('../plugins'); const privileges = require('../privileges'); const cache = require('../cache'); const meta = require('../meta'); +const utils = require('../utils'); const Categories = module.exports; @@ -26,9 +27,14 @@ require('./search')(Categories); Categories.icons = require('./icon'); Categories.exists = async function (cids) { - return await db.exists( - Array.isArray(cids) ? cids.map(cid => `category:${cid}`) : `category:${cids}` - ); + let keys; + if (Array.isArray(cids)) { + keys = cids.map(cid => (utils.isNumber(cid) ? `category:${cid}` : `categoryRemote:${cid}`)); + } else { + keys = utils.isNumber(cids) ? `category:${cids}` : `categoryRemote:${cids}`; + } + + return await db.exists(keys); }; Categories.existsByHandle = async function (handle) { diff --git a/src/categories/topics.js b/src/categories/topics.js index 64e9046614..cd13deaf4b 100644 --- a/src/categories/topics.js +++ b/src/categories/topics.js @@ -31,6 +31,7 @@ module.exports = function (Categories) { Categories.getPinnedTids({ ...data, start: 0, stop: -1 }), Categories.buildTopicsSortedSet(data), ]); + console.log(set); const totalPinnedCount = pinnedTids.length; const pinnedTidsOnPage = pinnedTids.slice(data.start, data.stop !== -1 ? data.stop + 1 : undefined); diff --git a/src/controllers/category.js b/src/controllers/category.js index 27b5c749f9..a822022392 100644 --- a/src/controllers/category.js +++ b/src/controllers/category.js @@ -26,14 +26,22 @@ const validSorts = [ ]; categoryController.get = async function (req, res, next) { - const cid = req.params.category_id; + let cid = req.params.category_id; if (cid === '-1') { return helpers.redirect(res, `${res.locals.isAPI ? '/api' : ''}/world?${qs.stringify(req.query)}`); } + if (!utils.isNumber(cid)) { + const assertion = await activitypub.actors.assertGroup([cid]); + cid = await db.getObjectField('handle:cid', cid); + if (!assertion || !cid) { + return next(); + } + } + let currentPage = parseInt(req.query.page, 10) || 1; let topicIndex = utils.isNumber(req.params.topic_index) ? parseInt(req.params.topic_index, 10) - 1 : 0; - if ((req.params.topic_index && !utils.isNumber(req.params.topic_index)) || !utils.isNumber(cid)) { + if ((req.params.topic_index && !utils.isNumber(req.params.topic_index))) { return next(); } @@ -58,7 +66,7 @@ categoryController.get = async function (req, res, next) { return helpers.notAllowed(req, res); } - if (!res.locals.isAPI && !req.params.slug && (categoryFields.slug && categoryFields.slug !== `${cid}/`)) { + if (utils.isNumber(cid) && !res.locals.isAPI && !req.params.slug && (categoryFields.slug && categoryFields.slug !== `${cid}/`)) { return helpers.redirect(res, `/category/${categoryFields.slug}?${qs.stringify(req.query)}`, true); }