Refactor sitemap function to use async for for easier reasoning. Factor out the user fetch as this was causing a huge blocking operation and eating a massive amount of memory.

controllers/resources.js

@@ -21,6 +21,12 @@ var async = require('async'),
   request = require('request'),
   R = require('ramda');
 
+/**
+ * Cached values
+ */
+var allBonfireIds, allBonfireNames, allCoursewareIds, allCoursewareNames,
+  allFieldGuideIds, allFieldGuideNames, allNonprofitNames;
+
 /**
  * GET /
  * Resources.
@@ -48,53 +54,77 @@ module.exports = {
     var appUrl = 'http://www.freecodecamp.com';
     var now = moment(new Date()).format('YYYY-MM-DD');
 
-    User.find({'profile.username': {'$ne': '' }}, function(err, users) {
+
-      if (err) {
+    async.parallel({
-        debug('User err: ', err);
+        challenges: function (callback) {
-        return next(err);
-      }
           Courseware.find({}, function (err, challenges) {
             if (err) {
-          debug('User err: ', err);
+              debug('Courseware err: ', err);
-          return next(err);
+              callback(err);
+            } else {
+              callback(null, challenges);
             }
+          });
+        },
+        bonfires: function (callback) {
           Bonfire.find({}, function (err, bonfires) {
             if (err) {
-            debug('User err: ', err);
+              debug('Bonfire err: ', err);
-            return next(err);
+              callback(err);
+            } else {
+              callback(null, bonfires);
             }
+          });
+        },
+        stories: function (callback) {
           Story.find({}, function (err, stories) {
             if (err) {
-              debug('User err: ', err);
+              debug('Story err: ', err);
-              return next(err);
+              callback(err);
+            } else {
+              callback(null, stories);
             }
+          });
+        },
+        nonprofits: function (callback) {
           Nonprofit.find({}, function (err, nonprofits) {
             if (err) {
               debug('User err: ', err);
-                return next(err);
+              callback(err);
+            } else {
+              callback(null, nonprofits);
             }
+          });
+        },
+        fieldGuides: function (callback) {
           FieldGuide.find({}, function (err, fieldGuides) {
             if (err) {
               debug('User err: ', err);
-                  return next(err);
+              callback(err);
+            } else {
+              callback(null, fieldGuides);
             }
+          });
+        }
+      }, function (err, results) {
+        if (err) {
+          return next(err);
+        } else {
+          setTimeout(function() {
             res.header('Content-Type', 'application/xml');
             res.render('resources/sitemap', {
               appUrl: appUrl,
               now: now,
-                  users: users,
+              challenges: results.challenges,
-                  challenges: challenges,
+              bonfires: results.bonfires,
-                  bonfires: bonfires,
+              stories: results.stories,
-                  stories: stories,
+              nonprofits: results.nonprofits,
-                  nonprofits: nonprofits,
+              fieldGuides: results.fieldGuides
-                  fieldGuides: fieldGuides
-                });
-              });
-            });
-          });
-        });
-      });
             });
+          }, 0);
+        }
+      }
+    );
   },
 
   chat: function chat(req, res) {
@@ -161,6 +191,8 @@ module.exports = {
         debug('User err: ', err);
         return next(err);
       }
+
+      // todo is this necessary anymore?
       User.count({'points': {'$gt': 53}}, function (err, all) {
         if (err) {
           debug('User err: ', err);
@@ -178,115 +210,154 @@ module.exports = {
   },
 
   randomPhrase: function() {
-    var phrases = resources.phrases;
+    return resources.phrases[Math.floor(
-    return phrases[Math.floor(Math.random() * phrases.length)];
+      Math.random() * resources.phrases.length)];
   },
 
   randomVerb: function() {
-    var verbs = resources.verbs;
+    return resources.verbs[Math.floor(
-    return verbs[Math.floor(Math.random() * verbs.length)];
+      Math.random() * resources.verbs.length)];
   },
 
   randomCompliment: function() {
-    var compliments = resources.compliments;
+    return resources.compliments[Math.floor(
-    return compliments[Math.floor(Math.random() * compliments.length)];
+      Math.random() * resources.compliments.length)];
   },
 
   allBonfireIds: function() {
-    return bonfires.map(function(elem) {
+    if (allBonfireIds) {
+      return allBonfireIds;
+    } else {
+      allBonfireIds = bonfires.
+        map(function (elem) {
           return {
             _id: elem._id,
             difficulty: elem.difficulty
-      }
+          };
-    })
+        }).
-      .sort(function(a, b) {
+        sort(function (a, b) {
           return a.difficulty - b.difficulty;
-      })
+        }).
-      .map(function(elem) {
+        map(function (elem) {
           return elem._id;
         });
+      return allBonfireIds;
+    }
   },
 
   allFieldGuideIds: function() {
-    return fieldGuides.map(function(elem) {
+    if (allFieldGuideIds) {
+      return allFieldGuideIds;
+    } else {
+      allFieldGuideIds = fieldGuides.
+        map(function (elem) {
           return {
-        _id: elem._id,
+            _id: elem._id
-      }
+          };
-    })
-    .map(function(elem) {
-      return elem._id;
         });
+      return allFieldGuideIds;
+    }
   },
 
   allBonfireNames: function() {
-    return bonfires.map(function(elem) {
+    if (allBonfireNames) {
+      return allBonfireNames;
+    } else {
+      allBonfireNames = bonfires.
+        map(function (elem) {
           return {
             name: elem.name,
             difficulty: elem.difficulty,
             _id: elem._id
-      }
+          };
-    })
+        }).
-      .sort(function(a, b) {
+        sort(function (a, b) {
           return a.difficulty - b.difficulty;
-      })
+        }).
-      .map (function(elem) {
+        map(function (elem) {
           return {
             name: elem.name,
             _id: elem._id
-      }
+          };
         });
+      return allBonfireNames;
+    }
   },
 
   allFieldGuideNames: function() {
-    return fieldGuides.map(function(elem) {
+    if (allFieldGuideNames) {
+      return allFieldGuideNames;
+    } else {
+      allFieldGuideNames = fieldGuides.
+        map(function (elem) {
           return {
             name: elem.name
+          };
+        });
+      return allFieldGuideNames;
     }
-    })
   },
 
   allNonprofitNames: function() {
-    return nonprofits.map(function(elem) {
+    if (allNonprofitNames) {
+      return allNonprofitNames;
+    } else {
+      allNonprofitNames = nonprofits.
+        map(function (elem) {
           return {
             name: elem.name
+          };
+        });
+      return allNonprofitNames;
     }
-    })
   },
 
   allCoursewareIds: function() {
-    return coursewares.map(function(elem) {
+    if (allCoursewareIds) {
+      return allCoursewareIds;
+    } else {
+      allCoursewareIds = coursewares.
+        map(function (elem) {
           return {
             _id: elem._id,
             difficulty: elem.difficulty
           };
-    })
+        }).
-      .sort(function(a, b) {
+        sort(function (a, b) {
           return a.difficulty - b.difficulty;
-      })
+        }).
-      .map(function(elem) {
+        map(function (elem) {
           return elem._id;
         });
+      return allCoursewareIds;
+    }
   },
 
   allCoursewareNames: function() {
-    return coursewares.map(function(elem) {
+    if (allCoursewareNames) {
+      return allCoursewareNames;
+    } else {
+      allCoursewareNames = coursewares.
+        map(function (elem) {
           return {
             name: elem.name,
             difficulty: elem.difficulty,
             challengeType: elem.challengeType,
             _id: elem._id
           };
-    })
+        }).
-      .sort(function(a, b) {
+        sort(function (a, b) {
           return a.difficulty - b.difficulty;
-      })
+        }).
-      .map (function(elem) {
+        map(function (elem) {
         return {
           name: elem.name,
           challengeType: elem.challengeType,
           _id: elem._id
         };
       });
+      return allCoursewareNames;
+    }
   },
 
   whichEnvironment: function() {
@@ -302,8 +373,9 @@ module.exports = {
           var metaDescription = $("meta[name='description']");
           var metaImage =  $("meta[property='og:image']");
           var urlImage = metaImage.attr('content') ? metaImage.attr('content') : '';
+          var metaTitle = $('title');
           var description = metaDescription.attr('content') ? metaDescription.attr('content') : '';
-          result.title = $('title').text().length < 141 ? $('title').text() : $('title').text().slice(0, 137) + " ...";
+          result.title = metaTitle.text().length < 141 ? metaTitle.text() : metaTitle.text().slice(0, 137) + " ...";
           result.image = urlImage;
           result.description = description;
           callback(null, result);
@@ -314,6 +386,7 @@ module.exports = {
     })();
   },
 
+  // todo analyze this function in detail
   updateUserStoryPictures: function(userId, picture, username, cb) {
 
     var counter = 0,
@@ -360,7 +433,9 @@ module.exports = {
         });
       }, foundStories);
       async.parallel(tasks, function(err) {
-        if (err) { return cb(err); }
+        if (err) {
+          return cb(err);
+        }
         cb();
       });
     }

views/resources/sitemap.jade

@@ -32,14 +32,6 @@ urlset(xmlns="http://www.sitemaps.org/schemas/sitemap/0.9")
     changefreq daily
     priority= 0.8
 
-  //- Users
-  each user in users
-    url
-      loc #{appUrl}/#{user.profile.username}
-      lastmod= now
-      changefreq daily
-      priority= 0.9
-
   //- Products
   each bonfire in bonfires
     url