Data creation script

get-challenge-completion.js

@@ -1,34 +1,45 @@
 /* eslint-disable no-process-exit */
+
+/**
+ * Data creation script for academic data pushes. Run this script with
+ * node seed/get-challenge-completion.js >> ~/output.json
+ * For large data sets supply the flag --max_old_space_size=2000000 to node.
+ * Run from the root FCC directory. Beware, this will generate a very large file
+ * if you have a large user collection such as the production mongo db.
+*/
+
 require('dotenv').load();
 var secrets = require('../config/secrets'),
     mongodb = require('mongodb'),
-    MongoClient = mongodb.MongoClient,
+    MongoClient = mongodb.MongoClient;
-    _ = require('lodash');
 
 MongoClient.connect(secrets.db, function(err, database) {
   if (err) {
     throw err;
   }
-
+  var stream = database.collection('user')
-  database.collection('user').aggregate([
+        .find({'completedChallenges': { $ne: null }},
-    {$match: { 'completedChallenges': { $exists: true } } },
+              {'completedChallenges': true})
-    {$match: { 'completedChallenges': { $ne: '' } } },
+        .stream();
-    {$match: { 'completedChallenges': { $ne: null } } },
+  console.log('[');
-    {$group: { '_id': 1, 'completedChallenges': {$addToSet: '$completedChallenges' } } }
+  stream.on('data', function(results) {
-  ], function(err, results) {
+    if (!results.completedChallenges) {
-    if (err) { throw err; }
+      // dud
-    var testout = results.map(function(camper) {
+    } else {
-      return _.flatten(camper.completedChallenges.map(function(challenges) {
+      var testout = [];
-        return challenges.map(function(challenge) {
+      results.completedChallenges.forEach(function(challenge) {
-          return { 
+        testout.push({
           name: challenge.name,
           completedDate: challenge.completedDate,
           solution: challenge.solution
-          };
         });
-      }), true); 
       });
-    console.log(JSON.stringify(testout));
+      if (testout.length) {
+          console.log(JSON.stringify(testout) + ',');
+      }
+    }
+  }).on('end', function() {
+    console.log(']');
     process.exit(0);
   });
 });