From b2462e1b4c6eb3ed8fd3fd2ce97a83f70dc7cce5 Mon Sep 17 00:00:00 2001 From: Berkeley Martinez Date: Thu, 7 Apr 2016 13:00:51 -0700 Subject: [PATCH] Fix sitemap Add 24 hour cache Add jobs to sitemap Possible warning about to many eventemitter listeners --- server/boot/randomAPIs.js | 136 ++-------------------------- server/boot/sitemap.js | 64 +++++++++++++ server/utils/rx.js | 11 ++- server/views/resources/sitemap.jade | 21 ++--- 4 files changed, 89 insertions(+), 143 deletions(-) create mode 100644 server/boot/sitemap.js diff --git a/server/boot/randomAPIs.js b/server/boot/randomAPIs.js index 2995082fb2..3fd3f953bd 100644 --- a/server/boot/randomAPIs.js +++ b/server/boot/randomAPIs.js @@ -1,24 +1,15 @@ -var Rx = require('rx'), - async = require('async'), - moment = require('moment'), - request = require('request'), - debug = require('debug')('fcc:cntr:resources'), - constantStrings = require('../utils/constantStrings.json'), - labs = require('../resources/labs.json'), - testimonials = require('../resources/testimonials.json'), - secrets = require('../../config/secrets'); +import request from 'request'; +import constantStrings from '../utils/constantStrings.json'; +import labs from '../resources/labs.json'; +import testimonials from '../resources/testimonials.json'; +import secrets from '../../config/secrets'; module.exports = function(app) { - var router = app.loopback.Router(); - var User = app.models.User; - var Challenge = app.models.Challenge; - var Story = app.models.Story; - var Nonprofit = app.models.Nonprofit; - + const router = app.loopback.Router(); + const User = app.models.User; router.get('/api/github', githubCalls); router.get('/api/blogger', bloggerCalls); router.get('/api/trello', trelloCalls); - router.get('/sitemap.xml', sitemap); router.get('/chat', chat); router.get('/coding-bootcamp-cost-calculator', bootcampCalculator); router.get('/twitch', twitch); @@ -43,7 +34,6 @@ module.exports = function(app) { router.get('/how-nonprofit-projects-work', howNonprofitProjectsWork); router.get('/code-of-conduct', codeOfConduct); router.get('/academic-honesty', academicHonesty); - router.get( '/the-fastest-web-page-on-the-internet', theFastestWebPageOnTheInternet @@ -51,118 +41,6 @@ module.exports = function(app) { app.use(router); - function sitemap(req, res, next) { - var appUrl = 'http://www.freecodecamp.com'; - var now = moment(new Date()).format('YYYY-MM-DD'); - - // TODO(berks): refactor async to rx - async.parallel({ - users: function(callback) { - User.find( - { - where: { username: { nlike: '' } }, - fields: { username: true } - }, - function(err, users) { - if (err) { - debug('User err: ', err); - callback(err); - } else { - Rx.Observable.from(users, null, null, Rx.Scheduler.default) - .map(function(user) { - return user.username; - }) - .toArray() - .subscribe( - function(usernames) { - callback(null, usernames); - }, - callback - ); - } - }); - }, - - challenges: function(callback) { - Challenge.find( - { fields: { name: true } }, - function(err, challenges) { - if (err) { - debug('Challenge err: ', err); - callback(err); - } else { - Rx.Observable.from(challenges, null, null, Rx.Scheduler.default) - .map(function(challenge) { - return challenge.name; - }) - .toArray() - .subscribe( - callback.bind(callback, null), - callback - ); - } - }); - }, - stories: function(callback) { - Story.find( - { field: { link: true } }, - function(err, stories) { - if (err) { - debug('Story err: ', err); - callback(err); - } else { - Rx.Observable.from(stories, null, null, Rx.Scheduler.default) - .map(function(story) { - return story.link; - }) - .toArray() - .subscribe( - callback.bind(callback, null), - callback - ); - } - } - ); - }, - nonprofits: function(callback) { - Nonprofit.find( - { field: { name: true } }, - function(err, nonprofits) { - if (err) { - debug('User err: ', err); - callback(err); - } else { - Rx.Observable.from(nonprofits, null, null, Rx.Scheduler.default) - .map(function(nonprofit) { - return nonprofit.name; - }) - .toArray() - .subscribe( - callback.bind(callback, null), - callback - ); - } - }); - } - }, function(err, results) { - if (err) { - return next(err); - } - return process.nextTick(function() { - res.header('Content-Type', 'application/xml'); - res.render('resources/sitemap', { - appUrl: appUrl, - now: now, - users: results.users, - challenges: results.challenges, - stories: results.stories, - nonprofits: results.nonprofits - }); - }); - } - ); - } - function chat(req, res) { res.redirect('https://gitter.im/FreeCodeCamp/FreeCodeCamp'); } diff --git a/server/boot/sitemap.js b/server/boot/sitemap.js new file mode 100644 index 0000000000..fc001a8b2d --- /dev/null +++ b/server/boot/sitemap.js @@ -0,0 +1,64 @@ +import moment from 'moment'; +import { Scheduler, Observable } from 'rx'; +import { timeCache, observeQuery } from '../utils/rx'; +import { dasherize } from '../utils'; + +const cacheTimeout = [ 24, 'hours' ]; +const appUrl = 'https://www.freecodecamp.com'; + +// getCachedObservable( +// app: ExpressApp, +// modelName: String, +// nameProp: String, +// map: (nameProp: String) => String +// ) => Observable[models] +function getCachedObservable(app, modelName, nameProp, map) { + return observeQuery( + app.models[modelName], + 'find', + { fields: { [nameProp]: true } } + ) + .flatMap(models => { + return Observable.from(models, null, null, Scheduler.default); + }) + .filter(model => !!model[nameProp]) + .map(model => model[nameProp]) + .map(map ? map : (x) => x) + .toArray() + ::timeCache(cacheTimeout[0], cacheTimeout[1]); +} + +export default function sitemapRouter(app) { + const router = app.loopback.Router(); + const challenges$ = getCachedObservable(app, 'Challenge', 'dashedName'); + const stories$ = getCachedObservable(app, 'Story', 'storyLink', dasherize); + const jobs$ = getCachedObservable(app, 'Job', 'id'); + function sitemap(req, res, next) { + const now = moment(new Date()).format('YYYY-MM-DD'); + return Observable.combineLatest( + challenges$, + stories$, + jobs$, + ( + challenges, + stories, + jobs + ) => ({ challenges, stories, jobs }) + ) + .subscribe( + ({ challenges, stories, jobs }) => { + res.header('Content-Type', 'application/xml'); + res.render('resources/sitemap', { + appUrl, + now, + challenges, + stories, + jobs + }); + }, + next + ); + } + router.get('/sitemap.xml', sitemap); + app.use(router); +} diff --git a/server/utils/rx.js b/server/utils/rx.js index 20976dfeb8..36bd6c7ae1 100644 --- a/server/utils/rx.js +++ b/server/utils/rx.js @@ -25,15 +25,20 @@ export function saveInstance(instance) { // alias saveInstance export const saveUser = saveInstance; -export function observeQuery(Model, method, query) { - return Rx.Observable.fromNodeCallback(Model[method], Model)(query); +// observeQuery(Model: Object, methodName: String, query: Any) => Observable +export function observeQuery(Model, methodName, query) { + return Rx.Observable.fromNodeCallback(Model[methodName], Model)(query); } +// observeMethod( +// context: Object, methodName: String +// ) => (query: Any) => Observable export function observeMethod(context, methodName) { return Rx.Observable.fromNodeCallback(context[methodName], context); } -// timeChache(amount: Number, unit: String) => Observable +// must be bound to an observable instance +// timeCache(amount: Number, unit: String) => Observable export function timeCache(time, unit) { const source = this; let cache; diff --git a/server/views/resources/sitemap.jade b/server/views/resources/sitemap.jade index 7ce295adb1..0f58d13062 100644 --- a/server/views/resources/sitemap.jade +++ b/server/views/resources/sitemap.jade @@ -49,24 +49,23 @@ urlset(xmlns="http://www.sitemaps.org/schemas/sitemap/0.9") lastmod= now priority= 0.9 - //- User - each user in users - url - loc #{appUrl}/#{user} - lastmod= now - changefreq weekly - priority= 0.5 - each challenge in challenges url - loc #{appUrl}/challenges/#{challenge.replace(/\s/g, '-')} + loc #{appUrl}/challenges/#{challenge} lastmod= now changefreq weekly priority= 0.9 each story in stories url - loc #{appUrl}/news/#{story.replace(/\s/g, '-')} + loc #{appUrl}/news/#{story} lastmod= now - changefreq weekly + changefreq monthly + priority= 0.5 + + each job in jobs + url + loc #{appUrl}/jobs/#{job} + lastmod= now + changefreq monthly priority= 0.5