Merge pull request #6854 from FreeCodeCamp/feature/reduce-query-load

Reduce query load/ add completed challenge validation

client/commonFramework/bindings.js

@@ -81,9 +81,15 @@ window.common = (function(global) {
             data = {
               id: common.challengeId,
               name: common.challengeName,
-              challengeType: common.challengeType
+              challengeType: +common.challengeType
             };
-            $.post('/completed-challenge/', data)
+            $.ajax({
+              url: '/completed-challenge/',
+              type: 'POST',
+              data: JSON.stringify(data),
+              contentType: 'application/json',
+              dataType: 'json'
+            })
               .success(function(res) {
                 if (!res) {
                   return;
@@ -92,7 +98,7 @@ window.common = (function(global) {
                   common.challengeId;
               })
               .fail(function() {
-                window.location.href = '/challenges';
+                window.location.replace(window.location.href);
               });
 
             break;
@@ -106,7 +112,13 @@ window.common = (function(global) {
               githubLink
             };
 
-            $.post('/completed-zipline-or-basejump/', data)
+            $.ajax({
+              url: '/completed-zipline-or-basejump/',
+              type: 'POST',
+              data: JSON.stringify(data),
+              contentType: 'application/json',
+              dataType: 'json'
+            })
               .success(function() {
                 window.location.href = '/challenges/next-challenge?id=' +
                   common.challengeId;

client/commonFramework/show-completion.js

@@ -57,21 +57,31 @@ window.common = (function(global) {
         `;
         console.error(err);
       }
-      const data = {
+      const data = JSON.stringify({
         id: common.challengeId,
         name: common.challengeName,
         completedWith: didCompleteWith,
-        challengeType: common.challengeType,
+        challengeType: +common.challengeType,
         solution,
         timezone
-      };
-
-      $.post('/completed-challenge/', data, function(res) {
-        if (res) {
-          window.location =
-            '/challenges/next-challenge?id=' + common.challengeId;
-        }
       });
+
+      $.ajax({
+        url: '/completed-challenge/',
+        type: 'POST',
+        data,
+        contentType: 'application/json',
+        dataType: 'json'
+      })
+        .success(function(res) {
+          if (res) {
+            window.location =
+              '/challenges/next-challenge?id=' + common.challengeId;
+          }
+        })
+        .fail(function() {
+          window.location.replace(window.location.href);
+        });
     });
   };
 

client/commonFramework/step-challenge.js

@@ -149,38 +149,45 @@ window.common = (function({ $, common = { init: [] }}) {
     e.preventDefault();
 
     $('#submit-challenge')
-    .attr('disabled', 'true')
+      .attr('disabled', 'true')
-    .removeClass('btn-primary')
+      .removeClass('btn-primary')
-    .addClass('btn-warning disabled');
+      .addClass('btn-warning disabled');
 
     var $checkmarkContainer = $('#checkmark-container');
     $checkmarkContainer.css({ height: $checkmarkContainer.innerHeight() });
 
     $('#challenge-checkmark')
-    .addClass('zoomOutUp')
+      .addClass('zoomOutUp')
-    .delay(1000)
+      .delay(1000)
-    .queue(function(next) {
+      .queue(function(next) {
-      $(this).replaceWith(
+        $(this).replaceWith(
-        '<div id="challenge-spinner" ' +
+          '<div id="challenge-spinner" ' +
-          'class="animated zoomInUp inner-circles-loader">' +
+            'class="animated zoomInUp inner-circles-loader">' +
-          'submitting...</div>'
+            'submitting...</div>'
-      );
+        );
-      next();
+        next();
-    });
+      });
 
-    $.post(
+    $.ajax({
-      '/completed-challenge/', {
+      url: '/completed-challenge/',
+      type: 'POST',
+      data: JSON.stringify({
         id: common.challengeId,
         name: common.challengeName,
-        challengeType: common.challengeType
+        challengeType: +common.challengeType
-      },
+      }),
-      function(res) {
+      contentType: 'application/json',
+      dataType: 'json'
+    })
+      .success(function(res) {
         if (res) {
           window.location =
             '/challenges/next-challenge?id=' + common.challengeId;
         }
-      }
+      })
-    );
+      .fail(function() {
+        window.location.replace(window.location.href);
+      });
   }
 
   common.init.push(function($) {

common/models/user.js

@@ -52,6 +52,12 @@ module.exports = function(User) {
   User.validatesUniquenessOf('username');
   User.settings.emailVerificationRequired = false;
 
+  User.on('dataSourceAttached', () => {
+    User.findOne$ = Observable.fromNodeCallback(User.findOne, User);
+    User.update$ = Observable.fromNodeCallback(User.updateAll, User);
+    User.count$ = Observable.fromNodeCallback(User.count, User);
+  });
+
   User.observe('before save', function({ instance: user }, next) {
     if (user) {
       user.username = user.username.trim().toLowerCase();
@@ -416,4 +422,23 @@ module.exports = function(User) {
       }
     }
   );
+
+  // user.updateTo$(updateData: Object) => Observable[Number]
+  User.prototype.update$ = function update$(updateData) {
+    const id = this.getId();
+    const updateOptions = { allowExtendedOperators: true };
+    if (
+        !updateData ||
+        typeof updateData !== 'object' ||
+        !Object.keys(updateData).length
+    ) {
+      return Observable.throw(new Error(
+        dedent`
+          updateData must be an object with at least one key,
+          but got ${updateData} with ${Object.keys(updateData).length}
+        `.split('\n').join(' ')
+      ));
+    }
+    return this.constructor.update$({ id }, updateData, updateOptions);
+  };
 };

common/models/user.json

@@ -148,6 +148,16 @@
       "default": false,
       "description": "Campers is full stack certified"
     },
+    "isChallengeMapMigrated": {
+      "type": "boolean",
+      "default": false,
+      "description": "Migrate completedChallenges array to challenge map"
+    },
+    "challengeMap": {
+      "type": "object",
+      "default": {},
+      "description": "A map by id of all the user completed challenges"
+    },
     "completedChallenges": {
       "type": [
         {

seed/challenges/01-front-end-development-certification/basic-javascript.json

@@ -1462,7 +1462,7 @@
       ]
     },
     {
-      "id": "bg9997c9c79feddfaeb9bdef",
+      "id": "56bbb991ad1ed5201cd392ca",
       "title": "Access Array Data with Indexes",
       "description": [
         "We can access the data inside arrays using <code>indexes</code>.",
@@ -1581,7 +1581,7 @@
       "challengeType": 1
     },
     {
-      "id": "bg9995c9c69feddfaeb9bdef",
+      "id": "56bbb991ad1ed5201cd392cb",
       "title": "Manipulate Arrays With push()",
       "description": [
         "An easy way to append data to the end of an array is via the <code>push()</code> function.",
@@ -1621,7 +1621,7 @@
       ]
     },
     {
-      "id": "bg9994c9c69feddfaeb9bdef",
+      "id": "56bbb991ad1ed5201cd392cc",
       "title": "Manipulate Arrays With pop()",
       "description": [
         "Another way to change the data in an array is with the <code>.pop()</code> function.",
@@ -1666,7 +1666,7 @@
       ]
     },
     {
-      "id": "bg9996c9c69feddfaeb9bdef",
+      "id": "56bbb991ad1ed5201cd392cd",
       "title": "Manipulate Arrays With shift()",
       "description": [
         "<code>pop()</code> always removes the last element of an array. What if you want to remove the first?",
@@ -1708,7 +1708,7 @@
       ]
     },
     {
-      "id": "bg9997c9c69feddfaeb9bdef",
+      "id": "56bbb991ad1ed5201cd392ce",
       "title": "Manipulate Arrays With unshift()",
       "description": [
         "Not only can you <code>shift</code> elements off of the beginning of an array, you can also <code>unshift</code> elements to the beginning of an array i.e. add elements in front of the array.",
@@ -1805,7 +1805,7 @@
       "challengeType": 1
     },
     {
-      "id": "bg9997c9c89feddfaeb9bdef",
+      "id": "56bbb991ad1ed5201cd392cf",
       "title": "Write Reusable JavaScript with Functions",
       "description": [
         "In JavaScript, we can divide up our code into reusable parts called <dfn>functions</dfn>.",
@@ -3314,7 +3314,7 @@
       "challengeType": 1
     },
     {
-      "id": "bg9998c9c99feddfaeb9bdef",
+      "id": "56bbb991ad1ed5201cd392d0",
       "title": "Build JavaScript Objects",
       "description": [
         "You may have heard the term <code>object</code> before.",
@@ -3500,7 +3500,7 @@
       "challengeType": 1
     },
     {
-      "id": "bg9999c9c99feddfaeb9bdef",
+      "id": "56bbb991ad1ed5201cd392d1",
       "title": "Updating Object Properties",
       "description": [
         "After you've created a JavaScript object, you can update its properties at any time just like you would update any other variable. You can use either dot or bracket notation to update.",
@@ -3565,7 +3565,7 @@
       ]
     },
     {
-      "id": "bg9999c9c99feedfaeb9bdef",
+      "id": "56bbb991ad1ed5201cd392d2",
       "title": "Add New Properties to a JavaScript Object",
       "description": [
         "You can add new properties to existing JavaScript objects the same way you would modify them.",
@@ -3621,7 +3621,7 @@
       ]
     },
     {
-      "id": "bg9999c9c99fdddfaeb9bdef",
+      "id": "56bbb991ad1ed5201cd392d3",
       "title": "Delete Properties from a JavaScript Object",
       "description": [
         "We can also delete properties from objects like this:",

seed/challenges/01-front-end-development-certification/json-apis-and-ajax.json

@@ -5,7 +5,7 @@
   "helpRoom": "Help",
   "challenges": [
     {
-      "id": "bb000000000000000000001",
+      "id": "56bbb991ad1ed5201cd392d4",
       "title": "Trigger Click Events with jQuery",
       "description": [
         "In this section, we'll learn how to get data from APIs. APIs - or Application Programming Interfaces - are tools that computers use to communicate with one another.",
@@ -69,7 +69,7 @@
       ]
     },
     {
-      "id": "bc000000000000000000001",
+      "id": "56bbb991ad1ed5201cd392d5",
       "title": "Change Text with Click Events",
       "description": [
         "When our click event happens, we can use Ajax to update an HTML element.",
@@ -128,7 +128,7 @@
       ]
     },
     {
-      "id": "bb000000000000000000002",
+      "id": "56bbb991ad1ed5201cd392d6",
       "title": "Get JSON with the jQuery getJSON Method",
       "description": [
         "You can also request data from an external source. This is where APIs come into play.",
@@ -209,7 +209,7 @@
       ]
     },
     {
-      "id": "bb000000000000000000003",
+      "id": "56bbb991ad1ed5201cd392d7",
       "title": "Convert JSON Data to HTML",
       "description": [
         "Now that we're getting data from a JSON API, let's display it in our HTML.",
@@ -283,7 +283,7 @@
       ]
     },
     {
-      "id": "bb000000000000000000004",
+      "id": "56bbb991ad1ed5201cd392d8",
       "title": "Render Images from Data Sources",
       "description": [
         "We've seen from the last two lessons that each object in our JSON array contains an <code>imageLink</code> key with a value that is the url of a cat's image.",
@@ -361,7 +361,7 @@
       ]
     },
     {
-      "id": "bb000000000000000000005",
+      "id": "56bbb991ad1ed5201cd392d9",
       "title": "Prefilter JSON",
       "description": [
         "If we don't want to render every cat photo we get from our Free Code Camp's Cat Photo JSON API, we can pre-filter the json before we loop through it.",
@@ -440,7 +440,7 @@
       ]
     },
     {
-      "id": "bb000000000000000000006",
+      "id": "56bbb991ad1ed5201cd392da",
       "title": "Get Geo-location Data",
       "description": [
         "Another cool thing we can do is access our user's current location. Every browser has a built in navigator that can give us this information.",

server/boot/certificate.js

@@ -1,17 +1,14 @@
 import _ from 'lodash';
 import dedent from 'dedent';
 import { Observable } from 'rx';
-import debugFactory from 'debug';
+import debug from 'debug';
 
 import {
   ifNoUser401,
   ifNoUserSend
 } from '../utils/middleware';
 
-import {
+import { observeQuery } from '../utils/rx';
-  saveUser,
-  observeQuery
-} from '../utils/rx';
 
 import {
   frontEndChallengeId,
@@ -25,17 +22,13 @@ import {
 
 import certTypes from '../utils/certTypes.json';
 
-const debug = debugFactory('freecc:certification');
+const log = debug('freecc:certification');
 const sendMessageToNonUser = ifNoUserSend(
   'must be logged in to complete.'
 );
 
-function isCertified(ids, { completedChallenges }) {
+function isCertified(ids, challengeMap = {}) {
-  return _.every(ids, ({ id }) => {
+  return _.every(ids, ({ id }) => challengeMap[id]);
-    return _.some(completedChallenges, (challenge) => {
-      return challenge.id === id || challenge._id === id;
-    });
-  });
 }
 
 function getIdsForCert$(id, Challenge) {
@@ -90,12 +83,9 @@ export default function certificate(app) {
   app.use(router);
 
   function verifyCert(certType, req, res, next) {
-    Observable.just({})
+    const { user } = req;
-      .flatMap(() => {
+    return certTypeIds[certType]
-        return certTypeIds[certType];
-      })
       .flatMap(challenge => {
-        const { user } = req;
         const {
           id,
           tests,
@@ -104,38 +94,39 @@ export default function certificate(app) {
         } = challenge;
         if (
           !user[certType] &&
-          isCertified(tests, user)
+          isCertified(tests, user.challengeMap)
         ) {
-          user[certType] = true;
+          const updateData = {
-          user.completedChallenges.push({
+            $set: {
-            id,
+              [`challengeMap.${id}`]: {
-            name,
+                id,
-            completedDate: new Date(),
+                name,
-            challengeType
+                completedDate: new Date(),
-          });
+                challengeType
+              },
+              [certType]: true
+            }
+          };
 
-          return saveUser(user)
+          return req.user.update$(updateData)
             // If user has commited to nonprofit,
             // this will complete his pledge
             .flatMap(
-              user => completeCommitment$(user),
+              () => completeCommitment$(user),
-              (user, pledgeOrMessage) => {
+              ({ count }, pledgeOrMessage) => {
                 if (typeof pledgeOrMessage === 'string') {
-                  debug(pledgeOrMessage);
+                  log(pledgeOrMessage);
                 }
-                // we are only interested in the user object
+                log(`${count} documents updated`);
-                // so we ignore return from completeCommitment$
+                return true;
-                return user;
               }
             );
         }
-        return Observable.just(user);
+        return Observable.just(false);
       })
       .subscribe(
-        user => {
+        (didCertify) => {
-          if (
+          if (didCertify) {
-            user[certType]
-          ) {
             return res.status(200).send(true);
           }
           return res.status(200).send(
@@ -150,14 +141,9 @@ export default function certificate(app) {
   }
 
   function postHonest(req, res, next) {
-    const { user } = req;
+    return req.user.update$({ $set: { isHonest: true } }).subscribe(
-    user.isHonest = true;
+      () => res.status(200).send(true),
-    saveUser(user)
+      next
-      .subscribe(
+    );
-        (user) => {
-          res.status(200).send(!!user.isHonest);
-        },
-        next
-      );
   }
 }

server/boot/challenge.js

@@ -2,7 +2,7 @@ import _ from 'lodash';
 import dedent from 'dedent';
 import moment from 'moment';
 import { Observable, Scheduler } from 'rx';
-import debugFactory from 'debug';
+import debug from 'debug';
 import accepts from 'accepts';
 
 import {
@@ -14,7 +14,7 @@ import {
   randomCompliment
 } from '../utils';
 
-import { saveUser, observeMethod } from '../utils/rx';
+import { observeMethod } from '../utils/rx';
 
 import {
   ifNoUserSend
@@ -24,7 +24,7 @@ import getFromDisk$ from '../utils/getFromDisk$';
 
 const isDev = process.env.NODE_ENV !== 'production';
 const isBeta = !!process.env.BETA;
-const debug = debugFactory('freecc:challenges');
+const log = debug('freecc:challenges');
 const challengesRegex = /^(bonfire|waypoint|zipline|basejump|checkpoint)/i;
 const challengeView = {
   0: 'challenges/showHTML',
@@ -40,8 +40,7 @@ function isChallengeCompleted(user, challengeId) {
   if (!user) {
     return false;
   }
-  return user.completedChallenges.some(challenge =>
+  return !!user.challengeMap[challengeId];
-    challenge.id === challengeId );
 }
 
 /*
@@ -50,36 +49,55 @@ function numberWithCommas(x) {
 }
 */
 
-function updateUserProgress(user, challengeId, completedChallenge) {
+function buildUserUpdate(
-  let { completedChallenges } = user;
+  user,
+  challengeId,
+  completedChallenge,
+  timezone
+) {
+  const updateData = { $set: {} };
+  let finalChallenge;
+  const { timezone: userTimezone, challengeMap = {} } = user;
 
-  const indexOfChallenge = _.findIndex(completedChallenges, {
+  const oldChallenge = challengeMap[challengeId];
-    id: challengeId
+  const alreadyCompleted = !!oldChallenge;
-  });
 
-  const alreadyCompleted = indexOfChallenge !== -1;
 
-  if (!alreadyCompleted) {
+  if (alreadyCompleted) {
-    user.progressTimestamps.push({
+    // add data from old challenge
-      timestamp: Date.now(),
+    finalChallenge = {
-      completedChallenge: challengeId
+      ...completedChallenge,
-    });
+      completedDate: oldChallenge.completedDate,
-    user.completedChallenges.push(completedChallenge);
+      lastUpdated: completedChallenge.completedDate
-    return user;
+    };
+  } else {
+    updateData.$push = {
+      progressTimestamps: {
+        timestamp: Date.now(),
+        completedChallenge: challengeId
+      }
+    };
+    finalChallenge = completedChallenge;
   }
 
-  const oldCompletedChallenge = completedChallenges[indexOfChallenge];
+  updateData.$set = {
-  user.completedChallenges[indexOfChallenge] =
+    [`challengeMap.${challengeId}`]: finalChallenge
-    Object.assign(
+  };
-      {},
-      completedChallenge,
-      {
-        completedDate: oldCompletedChallenge.completedDate,
-        lastUpdated: completedChallenge.completedDate
-      }
-    );
 
-  return { user, alreadyCompleted };
+  if (
+    timezone &&
+    timezone !== 'UTC' &&
+    (!userTimezone || userTimezone === 'UTC')
+  ) {
+    updateData.$set = {
+      ...updateData.$set,
+      timezone: userTimezone
+    };
+  }
+
+  log('user update data', updateData);
+
+  return { alreadyCompleted, updateData };
 }
 
 
@@ -117,7 +135,7 @@ function getRenderData$(user, challenge$, origChallengeName, solution) {
     .replace(challengesRegex, '');
 
   const testChallengeName = new RegExp(challengeName, 'i');
-  debug('looking for %s', testChallengeName);
+  log('looking for %s', testChallengeName);
 
   return challenge$
     .map(challenge => challenge.toJSON())
@@ -136,7 +154,7 @@ function getRenderData$(user, challenge$, origChallengeName, solution) {
 
       // Handle not found
       if (!challenge) {
-        debug('did not find challenge for ' + origChallengeName);
+        log('did not find challenge for ' + origChallengeName);
         return Observable.just({
           type: 'redirect',
           redirectUrl: '/map',
@@ -187,25 +205,21 @@ function getRenderData$(user, challenge$, origChallengeName, solution) {
     });
 }
 
-function getCompletedChallengeIds(user = {}) {
-  // if user
-  // get the id's of all the users completed challenges
-  return !user.completedChallenges ?
-    [] :
-    _.uniq(user.completedChallenges)
-      .map(({ id, _id }) => id || _id);
-}
-
 // create a stream of an array of all the challenge blocks
-function getSuperBlocks$(challenge$, completedChallenges) {
+function getSuperBlocks$(challenge$, challengeMap) {
   return challenge$
     // mark challenge completed
     .map(challengeModel => {
       const challenge = challengeModel.toJSON();
-      if (completedChallenges.indexOf(challenge.id) !== -1) {
+      challenge.completed = !!challengeMap[challenge.id];
-        challenge.completed = true;
-      }
       challenge.markNew = shouldShowNew(challenge);
+
+      if (challenge.type === 'hike') {
+        challenge.url = '/videos/' + challenge.dashedName;
+      } else {
+        challenge.url = '/challenges/' + challenge.dashedName;
+      }
+
       return challenge;
     })
     // group challenges by block | returns a stream of observables
@@ -223,15 +237,6 @@ function getSuperBlocks$(challenge$, completedChallenges) {
       const isComingSoon = _.every(blockArray, 'isComingSoon');
       const isRequired = _.every(blockArray, 'isRequired');
 
-      blockArray = blockArray.map(challenge => {
-        if (challenge.challengeType == 6 && challenge.type === 'hike') {
-          challenge.url = '/videos/' + challenge.dashedName;
-        } else {
-          challenge.url = '/challenges/' + challenge.dashedName;
-        }
-        return challenge;
-      });
-
       return {
         isBeta,
         isComingSoon,
@@ -428,7 +433,7 @@ module.exports = function(app) {
     getChallengeById$(challenge$, challengeId)
       .doOnNext(({ dashedName })=> {
         if (!dashedName) {
-          debug('no challenge found for %s', challengeId);
+          log('no challenge found for %s', challengeId);
           req.flash('info', {
             msg: `We coudn't find a challenge with the id ${challengeId}`
           });
@@ -473,7 +478,7 @@ module.exports = function(app) {
         return getNextChallenge$(challenge$, blocks$, challengeId)
           .doOnNext(({ dashedName } = {}) => {
             if (!dashedName) {
-              debug('no challenge found for %s', challengeId);
+              log('no challenge found for %s', challengeId);
               res.redirect('/map');
             }
             res.redirect('/challenges/' + dashedName);
@@ -495,7 +500,7 @@ module.exports = function(app) {
             });
           }
           if (type === 'redirect') {
-            debug('redirecting to %s', redirectUrl);
+            log('redirecting to %s', redirectUrl);
             return res.redirect(redirectUrl);
           }
           var view = challengeView[data.challengeType];
@@ -510,8 +515,28 @@ module.exports = function(app) {
   }
 
   function completedChallenge(req, res, next) {
+    req.checkBody('id', 'id must be a ObjectId').isMongoId();
+
+    req.checkBody('name', 'name must be at least 3 characters')
+      .isString()
+      .isLength({ min: 3 });
+
+    req.checkBody('challengeType', 'challengeType must be an integer')
+      .isNumber()
+      .isInt();
     const type = accepts(req).type('html', 'json', 'text');
 
+    const errors = req.validationErrors(true);
+
+    if (errors) {
+      if (type === 'json') {
+        return res.status(403).send({ errors });
+      }
+
+      log('errors', errors);
+      return res.sendStatus(403);
+    }
+
     const completedDate = Date.now();
     const {
       id,
@@ -521,7 +546,7 @@ module.exports = function(app) {
       timezone
     } = req.body;
 
-    const { alreadyCompleted } = updateUserProgress(
+    const { alreadyCompleted, updateData } = buildUserUpdate(
       req.user,
       id,
       {
@@ -529,26 +554,25 @@ module.exports = function(app) {
         challengeType,
         solution,
         name,
-        completedDate,
+        completedDate
-        verified: true
+      },
-      }
+      timezone
     );
 
-    if (timezone && (!req.user.timezone || req.user.timezone !== timezone)) {
+    const user = req.user;
-      req.user.timezone = timezone;
+    const points = alreadyCompleted ?
-    }
+      user.progressTimestamps.length :
+      user.progressTimestamps.length + 1;
 
-    let user = req.user;
+    return user.update$(updateData)
-    saveUser(req.user)
+      .doOnNext(({ count }) => log('%s documents updated', count))
       .subscribe(
-        function(user) {
+        () => {},
-          user = user;
-        },
         next,
         function() {
           if (type === 'json') {
             return res.json({
-              points: user.progressTimestamps.length,
+              points,
               alreadyCompleted
             });
           }
@@ -558,38 +582,35 @@ module.exports = function(app) {
   }
 
   function completedZiplineOrBasejump(req, res, next) {
-    const { body = {} } = req;
+    const type = accepts(req).type('html', 'json', 'text');
+    req.checkBody('id', 'id must be an ObjectId').isMongoId();
+    req.checkBody('name', 'Name must be at least 3 characters')
+      .isString()
+      .isLength({ min: 3 });
+    req.checkBody('challengeType', 'must be a number')
+      .isNumber()
+      .isInt();
+    req.checkBody('solution', 'solution must be a url').isURL();
 
-    let completedChallenge;
+    const errors = req.validationErrors(true);
-    // backwards compatibility
-    // please remove once in production
-    // to allow users to transition to new client code
-    if (body.challengeInfo) {
 
-      if (!body.challengeInfo.challengeId) {
+    if (errors) {
-        req.flash('error', { msg: 'No id returned during save' });
+      if (type === 'json') {
-        return res.sendStatus(403);
+        return res.status(403).send({ errors });
       }
-
+      log('errors', errors);
-      completedChallenge = {
+      return res.sendStatus(403);
-        id: body.challengeInfo.challengeId,
-        name: body.challengeInfo.challengeName || '',
-        completedDate: Date.now(),
-
-        challengeType: +body.challengeInfo.challengeType === 4 ? 4 : 3,
-
-        solution: body.challengeInfo.publicURL,
-        githubLink: body.challengeInfo.githubURL
-      };
-    } else {
-      completedChallenge = _.pick(
-        body,
-        [ 'id', 'name', 'solution', 'githubLink', 'challengeType' ]
-      );
-      completedChallenge.challengeType = +completedChallenge.challengeType;
-      completedChallenge.completedDate = Date.now();
     }
 
+    const { user, body = {} } = req;
+
+    const completedChallenge = _.pick(
+      body,
+      [ 'id', 'name', 'solution', 'githubLink', 'challengeType' ]
+    );
+    completedChallenge.challengeType = +completedChallenge.challengeType;
+    completedChallenge.completedDate = Date.now();
+
     if (
       !completedChallenge.solution ||
       // only basejumps require github links
@@ -600,22 +621,37 @@ module.exports = function(app) {
     ) {
       req.flash('errors', {
         msg: 'You haven\'t supplied the necessary URLs for us to inspect ' +
-        'your work.'
+          'your work.'
       });
       return res.sendStatus(403);
     }
 
 
-    updateUserProgress(req.user, completedChallenge.id, completedChallenge);
+    const {
+      alreadyCompleted,
+      updateData
+    } = buildUserUpdate(req.user, completedChallenge.id, completedChallenge);
 
-    return saveUser(req.user)
+    return user.update$(updateData)
-      .doOnNext(() => res.status(200).send(true))
+      .doOnNext(({ count }) => log('%s documents updated', count))
+      .doOnNext(() => {
+        if (type === 'json') {
+          return res.send({
+            alreadyCompleted,
+            points: alreadyCompleted ?
+              user.progressTimestamps.length :
+              user.progressTimestamps.length + 1
+          });
+        }
+        res.status(200).send(true);
+      })
       .subscribe(() => {}, next);
   }
 
-  function showMap(showAside, { user }, res, next) {
+  function showMap(showAside, { user = {} }, res, next) {
+    const { challengeMap = {} } = user;
 
-    getSuperBlocks$(challenge$, getCompletedChallengeIds(user))
+    return getSuperBlocks$(challenge$, challengeMap)
       .subscribe(
         superBlocks => {
           res.render('map/show', {

server/boot/user.js

@@ -1,4 +1,3 @@
-import _ from 'lodash';
 import dedent from 'dedent';
 import moment from 'moment-timezone';
 import { Observable } from 'rx';
@@ -60,6 +59,66 @@ function encodeFcc(value = '') {
   return replaceScriptTags(replaceFormAction(value));
 }
 
+function isAlgorithm(challenge) {
+  // test if name starts with hike/waypoint/basejump/zipline
+  // fix for bug that saved different challenges with incorrect
+  // challenge types
+  return !(/^(waypoint|hike|zipline|basejump)/i).test(challenge.name) &&
+    +challenge.challengeType === 5;
+}
+
+function isProject(challenge) {
+  return +challenge.challengeType === 3 ||
+    +challenge.challengeType === 4;
+}
+
+function getChallengeGroup(challenge) {
+  if (isProject(challenge)) {
+    return 'projects';
+  } else if (isAlgorithm(challenge)) {
+    return 'algorithms';
+  }
+  return 'challenges';
+}
+
+// buildDisplayChallenges(challengeMap: Object, tz: String) => Observable[{
+//   algorithms: Array,
+//   projects: Array,
+//   challenges: Array
+// }]
+function buildDisplayChallenges(challengeMap = {}, timezone) {
+  return Observable.from(Object.keys(challengeMap))
+    .map(challengeId => challengeMap[challengeId])
+    .map(challenge => {
+      let finalChallenge = { ...challenge };
+      if (challenge.completedDate) {
+        finalChallenge.completedDate = moment
+          .tz(challenge.completedDate, timezone)
+          .format(dateFormat);
+      }
+
+      if (challenge.lastUpdated) {
+        finalChallenge.lastUpdated = moment
+          .tz(challenge.lastUpdated, timezone)
+          .format(dateFormat);
+      }
+
+      return finalChallenge;
+    })
+    .groupBy(getChallengeGroup)
+    .flatMap(group$ => {
+      return group$.toArray().map(challenges => ({
+        [getChallengeGroup(challenges[0])]: challenges
+      }));
+    })
+    .reduce((output, group) => ({ ...output, ...group}), {})
+    .map(groups => ({
+      algorithms: groups.algorithms || [],
+      projects: groups.projects || [],
+      challenges: groups.challenges || []
+    }));
+}
+
 module.exports = function(app) {
   var router = app.loopback.Router();
   var User = app.models.User;
@@ -170,33 +229,35 @@ module.exports = function(app) {
 
   function returnUser(req, res, next) {
     const username = req.params.username.toLowerCase();
-    const { path } = req;
+    const { user, path } = req;
-    User.findOne(
+
-      {
+    // timezone of signed-in account
-        where: { username },
+    // to show all date related components
-        include: 'pledge'
+    // using signed-in account's timezone
-      },
+    // not of the profile she is viewing
-      function(err, profileUser) {
+    const timezone = user && user.timezone ?
-        if (err) {
+      user.timezone :
-          return next(err);
+      'UTC';
-        }
+
-        if (!profileUser) {
+    const query = {
+      where: { username },
+      include: 'pledge'
+    };
+
+    return User.findOne$(query)
+      .filter(userPortfolio => {
+        if (!userPortfolio) {
           req.flash('errors', {
-            msg: `404: We couldn't find path ${ path }`
+            msg: `We couldn't find a page for ${ path }`
           });
-          console.log('404');
+          res.redirect('/');
-          return res.redirect('/');
         }
-        profileUser = profileUser.toJSON();
+        return !!userPortfolio;
+      })
+      .flatMap(userPortfolio => {
+        userPortfolio = userPortfolio.toJSON();
 
-        // timezone of signed-in account
+        const timestamps = userPortfolio
-        // to show all date related components
-        // using signed-in account's timezone
-        // not of the profile she is viewing
-        const timezone = req.user &&
-          req.user.timezone ? req.user.timezone : 'UTC';
-
-        const timestamps = profileUser
           .progressTimestamps
           .map(objOrNum => {
             return typeof objOrNum === 'number' ?
@@ -206,10 +267,10 @@ module.exports = function(app) {
 
         const uniqueDays = prepUniqueDays(timestamps, timezone);
 
-        profileUser.currentStreak = calcCurrentStreak(uniqueDays, timezone);
+        userPortfolio.currentStreak = calcCurrentStreak(uniqueDays, timezone);
-        profileUser.longestStreak = calcLongestStreak(uniqueDays, timezone);
+        userPortfolio.longestStreak = calcLongestStreak(uniqueDays, timezone);
 
-        const data = profileUser
+        const calender = userPortfolio
           .progressTimestamps
           .map((objOrNum) => {
             return typeof objOrNum === 'number' ?
@@ -224,89 +285,30 @@ module.exports = function(app) {
             return data;
           }, {});
 
-        function filterAlgos(challenge) {
+        return buildDisplayChallenges(userPortfolio.challengeMap, timezone)
-          // test if name starts with hike/waypoint/basejump/zipline
+          .map(displayChallenges => ({
-          // fix for bug that saved different challenges with incorrect
+            ...userPortfolio,
-          // challenge types
+            ...displayChallenges,
-          return !(/^(waypoint|hike|zipline|basejump)/i).test(challenge.name) &&
+            title: 'Camper ' + userPortfolio.username + '\'s Code Portfolio',
-            +challenge.challengeType === 5;
+            calender,
-        }
+            github: userPortfolio.githubURL,
-
+            moment,
-        function filterProjects(challenge) {
+            encodeFcc
-          return +challenge.challengeType === 3 ||
+          }));
-            +challenge.challengeType === 4;
+      })
-        }
+      .doOnNext(data => {
-
+        return res.render('account/show', data);
-        const completedChallenges = profileUser.completedChallenges
+      })
-          .filter(({ name }) => typeof name === 'string')
+      .subscribe(
-          .map(challenge => {
+        () => {},
-              challenge = { ...challenge };
+        next
-              if (challenge.completedDate) {
+      );
-                challenge.completedDate =
-                  moment.tz(challenge.completedDate, timezone)
-                    .format(dateFormat);
-              }
-              if (challenge.lastUpdated) {
-                challenge.lastUpdated =
-                  moment.tz(challenge.lastUpdated, timezone).format(dateFormat);
-              }
-              return challenge;
-          });
-
-        const projects = completedChallenges.filter(filterProjects);
-
-        const algos = completedChallenges.filter(filterAlgos);
-
-        const challenges = completedChallenges
-          .filter(challenge => !filterAlgos(challenge))
-          .filter(challenge => !filterProjects(challenge));
-
-        res.render('account/show', {
-          title: 'Camper ' + profileUser.username + '\'s Code Portfolio',
-          username: profileUser.username,
-          name: profileUser.name,
-
-          isMigrationGrandfathered: profileUser.isMigrationGrandfathered,
-          isGithubCool: profileUser.isGithubCool,
-          isLocked: !!profileUser.isLocked,
-
-          pledge: profileUser.pledge,
-
-          isFrontEndCert: profileUser.isFrontEndCert,
-          isDataVisCert: profileUser.isDataVisCert,
-          isBackEndCert: profileUser.isBackEndCert,
-          isFullStackCert: profileUser.isFullStackCert,
-          isHonest: profileUser.isHonest,
-
-          location: profileUser.location,
-          calender: data,
-
-          github: profileUser.githubURL,
-          linkedin: profileUser.linkedin,
-          google: profileUser.google,
-          facebook: profileUser.facebook,
-          twitter: profileUser.twitter,
-          picture: profileUser.picture,
-
-          progressTimestamps: profileUser.progressTimestamps,
-
-          projects,
-          algos,
-          challenges,
-          moment,
-
-          longestStreak: profileUser.longestStreak,
-          currentStreak: profileUser.currentStreak,
-
-          encodeFcc
-        });
-      }
-    );
   }
 
   function showCert(certType, req, res, next) {
     const username = req.params.username.toLowerCase();
     const { user } = req;
+    const certId = certIds[certType];
     Observable.just(user)
       .flatMap(user => {
         if (user && user.username === username) {
@@ -321,9 +323,9 @@ module.exports = function(app) {
           isBackEndCert: true,
           isFullStackCert: true,
           isHonest: true,
-          completedChallenges: true,
           username: true,
-          name: true
+          name: true,
+          [ `challengesMap.${certId}` ]: true
         });
       })
       .subscribe(
@@ -376,15 +378,8 @@ module.exports = function(app) {
 
           if (user[certType]) {
 
-            // find challenge in user profile
+            const { completedDate = new Date() } =
-            // if not found supply empty object
+              user.challengeMap[certId] || {};
-            // if found grab date
-            // if no date use todays date
-            var { completedDate = new Date() } =
-              _.find(
-                user.completedChallenges,
-                { id: certIds[certType] }
-            ) || {};
 
             return res.render(
               certViews[certType],
@@ -511,29 +506,6 @@ module.exports = function(app) {
     });
   }
 
-  /*
-  function updateUserStoryPictures(userId, picture, username, cb) {
-    Story.find({ 'author.userId': userId }, function(err, stories) {
-      if (err) { return cb(err); }
-
-      const tasks = [];
-      stories.forEach(function(story) {
-        story.author.picture = picture;
-        story.author.username = username;
-        tasks.push(function(cb) {
-          story.save(cb);
-        });
-      });
-      async.parallel(tasks, function(err) {
-        if (err) {
-          return cb(err);
-        }
-        cb();
-      });
-    });
-  }
-  */
-
   function vote1(req, res, next) {
     if (req.user) {
       req.user.tshirtVote = 1;

server/middleware.json

@@ -47,7 +47,8 @@
     "./middlewares/express-rx": {},
     "./middlewares/jade-helpers": {},
     "./middlewares/global-locals": {},
-    "./middlewares/revision-helpers": {}
+    "./middlewares/revision-helpers": {},
+    "./middlewares/migrate-completed-challenges": {}
   },
   "routes": {
   },

server/middlewares/migrate-completed-challenges.js

@@ -0,0 +1,119 @@
+import { Observable, Scheduler } from 'rx';
+import { ObjectID } from 'mongodb';
+import debug from 'debug';
+
+import idMap from '../utils/bad-id-map';
+
+const log = debug('freecc:migrate');
+const challengeTypes = {
+  html: 0,
+  js: 1,
+  video: 2,
+  zipline: 3,
+  basejump: 4,
+  bonfire: 5,
+  hikes: 6,
+  step: 7,
+  waypoint: 0
+};
+
+const challengeTypeReg = /^(waypoint|hike|zipline|basejump)/i;
+const challengeTypeRegWithColon =
+  /^(bonfire|checkpoint|waypoint|hike|zipline|basejump):\s+/i;
+
+function updateName(challenge) {
+  if (
+    challenge.name &&
+    challenge.challengeType === 5 &&
+    challengeTypeReg.test(challenge.name)
+  ) {
+
+    challenge.name.replace(challengeTypeReg, match => {
+      // find the correct type
+      const type = challengeTypes[''.toLowerCase.call(match)];
+      // if type found, replace current type
+      //
+      if (type) {
+        challenge.challengeType = type;
+      }
+
+      return match;
+    });
+
+  }
+
+  if (challenge.name) {
+    challenge.oldName = challenge.name;
+    challenge.name = challenge.name.replace(challengeTypeRegWithColon, '');
+  }
+  return challenge;
+}
+
+function updateId(challenge) {
+  if (idMap.hasOwnProperty(challenge.id)) {
+    challenge.id = idMap[challenge.id];
+  }
+
+  return challenge;
+}
+
+// buildChallengeMap(
+//  userId: String,
+//  completedChallenges: Object[],
+//  User: User
+// ) => Observable
+function buildChallengeMap(userId, completedChallenges = [], User) {
+  return Observable.from(
+    completedChallenges,
+    null,
+    null,
+    Scheduler.default
+  )
+    .map(challenge => {
+      return challenge && typeof challenge.toJSON === 'function' ?
+        challenge.toJSON() :
+        challenge;
+    })
+    .map(updateId)
+    .filter(({ id, _id }) => ObjectID.isValid(id || _id))
+    .map(updateName)
+    .reduce((challengeMap, challenge) => {
+      const id = challenge.id || challenge._id;
+
+      challengeMap[id] = challenge;
+      return challengeMap;
+    }, {})
+    .flatMap(challengeMap => {
+      const updateData = {
+        '$set': {
+          challengeMap,
+          isChallengeMapMigrated: true
+        }
+      };
+      return Observable.fromNodeCallback(User.updateAll, User)(
+        { id: userId },
+        updateData,
+        { allowExtendedOperators: true }
+      );
+    });
+}
+
+export default function migrateCompletedChallenges() {
+  return ({ user, app }, res, next) => {
+    const User = app.models.User;
+    if (!user || user.isChallengeMapMigrated) {
+      return next();
+    }
+    return buildChallengeMap(
+      user.id.toString(),
+      user.completedChallenges,
+      User
+    )
+      .subscribe(
+        count => log('documents update', count),
+        // errors go here
+        next,
+        next
+      );
+  };
+}

server/middlewares/validator.js

@@ -1,9 +1,17 @@
 import validator from 'express-validator';
 
-export default validator.bind(validator, {
+export default function() {
-  customValidators: {
+  return validator({
-    matchRegex: function matchRegex(param, regex) {
+    customValidators: {
-      return regex.test(param);
+      matchRegex(param, regex) {
+        return regex.test(param);
+      },
+      isString(value) {
+        return typeof value === 'string';
+      },
+      isNumber(value) {
+        return typeof value === 'number';
+      }
     }
-  }
+  });
-});
+}

server/utils/bad-id-map.js

@@ -0,0 +1,19 @@
+export default {
+  bg9997c9c79feddfaeb9bdef: '56bbb991ad1ed5201cd392ca',
+  bg9995c9c69feddfaeb9bdef: '56bbb991ad1ed5201cd392cb',
+  bg9994c9c69feddfaeb9bdef: '56bbb991ad1ed5201cd392cc',
+  bg9996c9c69feddfaeb9bdef: '56bbb991ad1ed5201cd392cd',
+  bg9997c9c69feddfaeb9bdef: '56bbb991ad1ed5201cd392ce',
+  bg9997c9c89feddfaeb9bdef: '56bbb991ad1ed5201cd392cf',
+  bg9998c9c99feddfaeb9bdef: '56bbb991ad1ed5201cd392d0',
+  bg9999c9c99feddfaeb9bdef: '56bbb991ad1ed5201cd392d1',
+  bg9999c9c99feedfaeb9bdef: '56bbb991ad1ed5201cd392d2',
+  bg9999c9c99fdddfaeb9bdef: '56bbb991ad1ed5201cd392d3',
+  bb000000000000000000001: '56bbb991ad1ed5201cd392d4',
+  bc000000000000000000001: '56bbb991ad1ed5201cd392d5',
+  bb000000000000000000002: '56bbb991ad1ed5201cd392d6',
+  bb000000000000000000003: '56bbb991ad1ed5201cd392d7',
+  bb000000000000000000004: '56bbb991ad1ed5201cd392d8',
+  bb000000000000000000005: '56bbb991ad1ed5201cd392d9',
+  bb000000000000000000006: '56bbb991ad1ed5201cd392da'
+};

server/views/account/show.jade

@@ -140,7 +140,7 @@ block content
                                       a(href=challenge.solution, target='_blank') View my project
                                   td.col-xs-12.visible-xs
                                       a(href=challenge.solution, target='_blank')= removeOldTerms(challenge.name)
-              if (algos.length > 0)
+              if (algorithms.length > 0)
                   .col-sm-12
                       table.table.table-striped
                           thead
@@ -149,7 +149,7 @@ block content
                                   th.col-xs-2.hidden-xs Completed
                                   th.col-xs-2.hidden-xs Last Updated
                                   th.col-xs-2.hidden-xs Solution
-                          for challenge in algos
+                          for challenge in algorithms
                               tr
                                   td.col-xs-5.hidden-xs= removeOldTerms(challenge.name)
                                   td.col-xs-2.hidden-xs= challenge.completedDate ? challenge.completedDate : 'Not Available'