From b6fbfcc814a9fcb14ec524156460180bf2c0a002 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Tue, 3 Sep 2013 23:18:25 -0400 Subject: [PATCH 1/3] interim commit for new feed refactor --- src/feed.js | 118 +++++++++++++++++++++++----------------------------- 1 file changed, 52 insertions(+), 66 deletions(-) diff --git a/src/feed.js b/src/feed.js index 7127e30256..ee95309d71 100644 --- a/src/feed.js +++ b/src/feed.js @@ -4,106 +4,92 @@ posts = require('./posts.js'), topics = require('./topics.js'), fs = require('fs'), - rss = require('node-rss'), + rss = require('rss'), winston = require('winston'), path = require('path'); - function saveFeed(location, feed) { + Feed.defaults = { + ttl: 60, + basePath: path.join(__dirname, '../', 'feeds'), + baseUrl: nconf.get('url') + 'feeds' + }; + + Feed.saveFeed = function(location, feed) { var savePath = path.join(__dirname, '../', location); - fs.writeFile(savePath, rss.getFeedXML(feed), function (err) { + fs.writeFile(savePath, feed.xml(), function (err) { if(err) { winston.err(err); } }); } - function createFeed(title, description, feed_url, xml_url, author, urn) { - return rss.createNewFeed( - title, - feed_url, - description, - author, - xml_url, - { - 'urn' : urn - } - ); - } - - Feed.updateTopic = function(tid, cid) { - winston.info('[RSS] Updating RSS feeds for topic ' + tid); - var cache_time_in_seconds = 60; + if (process.env.NODE_ENV === 'development') winston.info('[rss] Updating RSS feeds for topic ' + tid); topics.getTopicWithPosts(tid, 0, 0, -1, function(err, topicData) { - if (err) winston.error('Problem saving topic RSS feed', err.stack); + if (err) return winston.error('Problem saving topic RSS feed', err.stack); - var location = '/topic/' + topicData.slug, - xml_url = '/topic/' + tid + '.rss'; - - var post = topicData.main_posts[0]; - var urn = 'urn:' + cid + ':' + tid; - - var feed = createFeed(topicData.topic_name, '', location, xml_url, post.username, urn); - var title; - - var topic_posts = topicData.main_posts.concat(topicData.posts); + var feed = new rss({ + title: topicData.topic_name, + description: topicData.main_posts[0].content, + feed_url: Feed.defaults.baseUrl + '/topics/' + tid + '.rss', + site_url: nconf.get('url') + 'topic/' + topicData.slug, + image_url: topicData.main_posts[0].picture, + author: topicData.main_posts[0].username, + pubDate: new Date(parseInt(topicData.main_posts[0].timestamp, 10)).toUTCString(), + ttl: Feed.defaults.ttl + }), + topic_posts = topicData.main_posts.concat(topicData.posts), + title, postData; for (var i = 0, ii = topic_posts.length; i < ii; i++) { - urn = 'urn:' + cid + ':' + tid + ':' + topic_posts[i].pid; title = 'Reply to ' + topicData.topic_name + ' on ' + (new Date(parseInt(topic_posts[i].timestamp, 10)).toUTCString()); + postData = topic_posts[i]; - feed.addNewItem( - title, - location, - topic_posts[i].timestamp, - topic_posts[i].content, - { - 'urn' : urn, - 'username' : topic_posts[i].username - } - ); + feed.item({ + title: title, + description: postData.content, + url: nconf.get('url') + 'topic/' + topicData.slug + '#' + postData.pid, + author: postData.username, + date: new Date(parseInt(postData.edited === 0 ? postData.timestamp : postData.edited, 10)).toUTCString() + }); } - saveFeed('feeds/topics/' + tid + '.rss', feed); + Feed.saveFeed('feeds/topics/' + tid + '.rss', feed); }); }; Feed.updateCategory = function(cid) { + if (process.env.NODE_ENV === 'development') winston.info('[rss] Updating RSS feeds for category ' + cid); categories.getCategoryById(cid, 0, function(err, categoryData) { - if (err) { - winston.error('Could not update RSS feed for category ' + cid, err.stack); - return; - } + if (err) return winston.error('Could not update RSS feed for category ' + cid, err.stack); - var location = '/category/' + categoryData.category_id + '/' + categoryData.category_name, - xml_url = '/category' + cid + '.rss'; - - var urn = 'urn:' + cid; - var feed = createFeed(categoryData.category_name, '', location, xml_url, 'NodeBB', urn); // not exactly sure if author for a category should be site_title? - - var title; - var topics = categoryData.topics; + var feed = new rss({ + title: categoryData.category_name, + description: categoryData.category_description, + feed_url: Feed.defaults.baseUrl + '/categories/' + cid + '.rss', + site_url: nconf.get('url') + 'category/' + categoryData.category_id, + pubDate: new Date(parseInt(categoryData.topics[0].lastposttime, 10)).toUTCString(), + ttl: Feed.defaults.ttl + }), + topics = categoryData.topics, + title, topicData; for (var i = 0, ii = topics.length; i < ii; i++) { - urn = 'urn:' + cid + ':' + topics[i].tid; title = topics[i].title + '. Posted on ' + (new Date(parseInt(topics[i].timestamp, 10)).toUTCString()); + topicData = topics[i]; - feed.addNewItem( - title, - location, - topics[i].timestamp, - topics[i].teaser_text, - { - 'urn' : urn, - 'username' : topics[i].username - } - ); + feed.item({ + title: title, + url: nconf.get('url') + 'topic/' + topicData.slug, + author: topicData.username, + date: new Date(parseInt(topicData.lastposttime, 10)).toUTCString() + }); } - saveFeed('feeds/categories/' + cid + '.rss', feed); + Feed.saveFeed('feeds/categories/' + cid + '.rss', feed); }); }; From 164977972e5aef6a22a8addc0e6b88057c0e2c81 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Tue, 3 Sep 2013 23:42:26 -0400 Subject: [PATCH 2/3] totally derped commit --- src/feed.js | 6 ++++-- src/posts.js | 2 +- src/webserver.js | 29 ++++++++++++++++++++--------- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/feed.js b/src/feed.js index ee95309d71..8799db8b85 100644 --- a/src/feed.js +++ b/src/feed.js @@ -24,7 +24,7 @@ }); } - Feed.updateTopic = function(tid, cid) { + Feed.updateTopic = function(tid, callback) { if (process.env.NODE_ENV === 'development') winston.info('[rss] Updating RSS feeds for topic ' + tid); topics.getTopicWithPosts(tid, 0, 0, -1, function(err, topicData) { @@ -57,11 +57,12 @@ } Feed.saveFeed('feeds/topics/' + tid + '.rss', feed); + if (callback) callback(); }); }; - Feed.updateCategory = function(cid) { + Feed.updateCategory = function(cid, callback) { if (process.env.NODE_ENV === 'development') winston.info('[rss] Updating RSS feeds for category ' + cid); categories.getCategoryById(cid, 0, function(err, categoryData) { if (err) return winston.error('Could not update RSS feed for category ' + cid, err.stack); @@ -90,6 +91,7 @@ } Feed.saveFeed('feeds/categories/' + cid + '.rss', feed); + if (callback) callback(); }); }; diff --git a/src/posts.js b/src/posts.js index 037e356df0..97f8d55bdf 100644 --- a/src/posts.js +++ b/src/posts.js @@ -314,7 +314,7 @@ var RDB = require('./redis.js'), topics.getTopicField(tid, 'cid', function(err, cid) { RDB.handle(err); - feed.updateTopic(tid, cid); + feed.updateTopic(tid); RDB.zadd('categories:recent_posts:cid:' + cid, timestamp, pid); RDB.zadd('categories:' + cid + ':tid', timestamp, tid); diff --git a/src/webserver.js b/src/webserver.js index 0abdcb36a3..6c7b9193e9 100644 --- a/src/webserver.js +++ b/src/webserver.js @@ -21,7 +21,8 @@ var express = require('express'), installRoute = require('./routes/install.js'), testBed = require('./routes/testbed.js'), auth = require('./routes/authentication.js'), - meta = require('./meta.js'); + meta = require('./meta.js'), + feed = require('./feed'); (function(app) { var templates = null; @@ -212,15 +213,25 @@ var express = require('express'), var tid = req.params.topic_id; if (tid.match(/^\d+\.rss$/)) { - fs.readFile(path.join(__dirname, '../', 'feeds/topics', tid), function (err, data) { - if (err) { - res.type('text').send(404, "Unable to locate an rss feed at this location."); - return; - } + tid = tid.slice(0, -4); + var rssPath = path.join(__dirname, '../', 'feeds/topics', tid + '.rss'), + loadFeed = function() { + fs.readFile(rssPath, function (err, data) { + if (err) { + res.type('text').send(404, "Unable to locate an rss feed at this location."); + return; + } - res.type('xml').set('Content-Length', data.length).send(data); - }); - return; + res.type('xml').set('Content-Length', data.length).send(data); + }); + return; + }; + + if (!fs.existsSync(rssPath)) { + feed.updateTopic(tid, function() { + loadFeed(); + }); + } else loadFeed(); } async.waterfall([ From 661fdfb43e1f9c2545db350b45018bccbe065a6b Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Wed, 4 Sep 2013 13:52:28 -0400 Subject: [PATCH 3/3] bugfixing and allowing feeds to be generated on request (as opposed to just updated on posting --- package.json | 3 ++- src/feed.js | 32 ++++++++++++++++++-------------- src/webserver.js | 36 +++++++++++++++++++++--------------- 3 files changed, 41 insertions(+), 30 deletions(-) diff --git a/package.json b/package.json index 17223e898b..8af51ccbc4 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,8 @@ "reds": "~0.2.4", "winston": "~0.7.2", "nodebb-plugin-mentions": "~0.1.0", - "nodebb-plugin-markdown": "~0.1.0" + "nodebb-plugin-markdown": "~0.1.0", + "rss": "~0.2.0" }, "bugs": { "url": "https://github.com/designcreateplay/NodeBB/issues" diff --git a/src/feed.js b/src/feed.js index 8799db8b85..2a3b1db083 100644 --- a/src/feed.js +++ b/src/feed.js @@ -14,13 +14,13 @@ baseUrl: nconf.get('url') + 'feeds' }; - Feed.saveFeed = function(location, feed) { + Feed.saveFeed = function(location, feed, callback) { var savePath = path.join(__dirname, '../', location); fs.writeFile(savePath, feed.xml(), function (err) { - if(err) { - winston.err(err); - } + if(err) return winston.err(err); + + if (callback) callback(err); }); } @@ -41,23 +41,25 @@ ttl: Feed.defaults.ttl }), topic_posts = topicData.main_posts.concat(topicData.posts), - title, postData; + title, postData, dateStamp; for (var i = 0, ii = topic_posts.length; i < ii; i++) { - title = 'Reply to ' + topicData.topic_name + ' on ' + (new Date(parseInt(topic_posts[i].timestamp, 10)).toUTCString()); postData = topic_posts[i]; + dateStamp = new Date(parseInt(postData.edited === 0 ? postData.timestamp : postData.edited, 10)).toUTCString(); + title = 'Reply to ' + topicData.topic_name + ' on ' + dateStamp; feed.item({ title: title, description: postData.content, url: nconf.get('url') + 'topic/' + topicData.slug + '#' + postData.pid, author: postData.username, - date: new Date(parseInt(postData.edited === 0 ? postData.timestamp : postData.edited, 10)).toUTCString() + date: dateStamp }); } - Feed.saveFeed('feeds/topics/' + tid + '.rss', feed); - if (callback) callback(); + Feed.saveFeed('feeds/topics/' + tid + '.rss', feed, function(err) { + if (callback) callback(); + }); }); }; @@ -76,22 +78,24 @@ ttl: Feed.defaults.ttl }), topics = categoryData.topics, - title, topicData; + title, topicData, dateStamp; for (var i = 0, ii = topics.length; i < ii; i++) { - title = topics[i].title + '. Posted on ' + (new Date(parseInt(topics[i].timestamp, 10)).toUTCString()); topicData = topics[i]; + dateStamp = new Date(parseInt(topicData.lastposttime, 10)).toUTCString(); + title = topics[i].title; feed.item({ title: title, url: nconf.get('url') + 'topic/' + topicData.slug, author: topicData.username, - date: new Date(parseInt(topicData.lastposttime, 10)).toUTCString() + date: dateStamp }); } - Feed.saveFeed('feeds/categories/' + cid + '.rss', feed); - if (callback) callback(); + Feed.saveFeed('feeds/categories/' + cid + '.rss', feed, function(err) { + if (callback) callback(); + }); }); }; diff --git a/src/webserver.js b/src/webserver.js index 6c7b9193e9..5c23978493 100644 --- a/src/webserver.js +++ b/src/webserver.js @@ -210,21 +210,17 @@ var express = require('express'), app.get('/topic/:topic_id/:slug?', function(req, res) { - var tid = req.params.topic_id; + if (tid.match(/^\d+\.rss$/)) { tid = tid.slice(0, -4); var rssPath = path.join(__dirname, '../', 'feeds/topics', tid + '.rss'), loadFeed = function() { fs.readFile(rssPath, function (err, data) { - if (err) { - res.type('text').send(404, "Unable to locate an rss feed at this location."); - return; - } - - res.type('xml').set('Content-Length', data.length).send(data); + if (err) res.type('text').send(404, "Unable to locate an rss feed at this location."); + else res.type('xml').set('Content-Length', data.length).send(data); }); - return; + }; if (!fs.existsSync(rssPath)) { @@ -232,6 +228,8 @@ var express = require('express'), loadFeed(); }); } else loadFeed(); + + return; } async.waterfall([ @@ -291,14 +289,22 @@ var express = require('express'), var cid = req.params.category_id; if (cid.match(/^\d+\.rss$/)) { - fs.readFile(path.join(__dirname, '../', 'feeds/categories', cid), function (err, data) { - if (err) { - res.type('text').send(404, "Unable to locate an rss feed at this location."); - return; - } + cid = cid.slice(0, -4); + var rssPath = path.join(__dirname, '../', 'feeds/categories', cid + '.rss'), + loadFeed = function() { + fs.readFile(rssPath, function (err, data) { + if (err) res.type('text').send(404, "Unable to locate an rss feed at this location."); + else res.type('xml').set('Content-Length', data.length).send(data); + }); + + }; + + if (!fs.existsSync(rssPath)) { + feed.updateCategory(cid, function() { + loadFeed(); + }); + } else loadFeed(); - res.type('xml').set('Content-Length', data.length).send(data); - }); return; }