From dfb85e4ba7582fdaa314e5438093c1f548988b3e Mon Sep 17 00:00:00 2001 From: Berkeley Martinez Date: Fri, 14 Aug 2015 17:08:01 -0700 Subject: [PATCH 1/5] fix prevent crash from user migrated profiles --- common/models/User-Identity.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/models/User-Identity.js b/common/models/User-Identity.js index 7aa88b295e..3b1bcb8750 100644 --- a/common/models/User-Identity.js +++ b/common/models/User-Identity.js @@ -158,14 +158,14 @@ export default function(UserIdent) { userChanged = true; } - if (!(/github/).test(provider)) { + if (!(/github/).test(provider) && profile) { debug('setting social', provider, (/github/g).test(provider)); debug('profile username', profile.username); user[provider] = profile.username; } // if user signed in with github refresh their info - if (/github/.test(provider)) { + if (/github/.test(provider) && profile && profile._json) { debug("user isn't github cool or username from github is different"); setProfileFromGithub(user, profile, profile._json); userChanged = true; From 59609b655c8c39fae8ed1ce077ccf5c5c9ae45e4 Mon Sep 17 00:00:00 2001 From: Berkeley Martinez Date: Fri, 14 Aug 2015 17:31:32 -0700 Subject: [PATCH 2/5] migration improvements --- seed/loopbackMigration.js | 53 ++++++++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/seed/loopbackMigration.js b/seed/loopbackMigration.js index e300bd217b..ab86785b34 100644 --- a/seed/loopbackMigration.js +++ b/seed/loopbackMigration.js @@ -6,6 +6,7 @@ var Rx = require('rx'), mongodb = require('mongodb'), secrets = require('../config/secrets'); +const batchSize = 100; var MongoClient = mongodb.MongoClient; Rx.config.longStackSupport = true; @@ -27,10 +28,12 @@ function debug() { function createConnection(URI) { return Rx.Observable.create(function(observer) { + debug('connecting to db'); MongoClient.connect(URI, function(err, database) { if (err) { return observer.onError(err); } + debug('db connected'); observer.onNext(database); observer.onCompleted(); }); @@ -62,6 +65,23 @@ function createQuery(db, collection, options, batchSize) { }); } +function getUserCount(db) { + return Rx.Observable.create(function(observer) { + var cursor = db.collection('users').count(function(err, count) { + if (err) { + return observer.onError(err); + } + observer.onNext(count); + observer.onCompleted(); + + return Rx.Disposable.create(function() { + debug('closing user count'); + cursor.close(); + }); + }); + }); +} + function insertMany(db, collection, users, options) { return Rx.Observable.create(function(observer) { db.collection(collection).insertMany(users, options, function(err) { @@ -76,12 +96,18 @@ function insertMany(db, collection, users, options) { var count = 0; // will supply our db object -var dbObservable = createConnection(secrets.db).shareReplay(); +var dbObservable = createConnection(secrets.db).replay(); + +var totalUser = dbObservable + .flatMap(function(db) { + return getUserCount(db); + }) + .shareReplay(); var users = dbObservable .flatMap(function(db) { // returns user document, n users per loop where n is the batchsize. - return createQuery(db, 'users', {}); + return createQuery(db, 'users', {}, batchSize); }) .map(function(user) { // flatten user @@ -105,7 +131,7 @@ var users = dbObservable // batch them into arrays of twenty documents var userSavesCount = users - .bufferWithCount(20) + .bufferWithCount(batchSize) // get bd object ready for insert .withLatestFrom(dbObservable, function(users, db) { return { @@ -117,6 +143,13 @@ var userSavesCount = users // bulk insert into new collection for loopback return insertMany(dats.db, 'user', dats.users, { w: 1 }); }) + .flatMap(function() { + return totalUser; + }) + .doOnNext(function(totalUsers) { + count = count + batchSize; + debug('user progress %s', count / totalUsers * 100); + }) // count how many times insert completes .count(); @@ -137,7 +170,7 @@ var userIdentityCount = users return Rx.Observable.from(ids); }) - .bufferWithCount(20) + .bufferWithCount(batchSize) .withLatestFrom(dbObservable, function(identities, db) { return { identities: identities, @@ -153,9 +186,9 @@ var userIdentityCount = users var storyCount = dbObservable .flatMap(function(db) { - return createQuery(db, 'stories', {}); + return createQuery(db, 'stories', {}, batchSize); }) - .bufferWithCount(20) + .bufferWithCount(batchSize) .withLatestFrom(dbObservable, function(stories, db) { return { stories: stories, @@ -173,9 +206,9 @@ Rx.Observable.combineLatest( storyCount, function(userIdentCount, userCount, storyCount) { return { - userIdentCount: userIdentCount * 20, - userCount: userCount * 20, - storyCount: storyCount * 20 + userIdentCount: userIdentCount * batchSize, + userCount: userCount * batchSize, + storyCount: storyCount * batchSize }; }) .subscribe( @@ -191,3 +224,5 @@ Rx.Observable.combineLatest( process.exit(0); } ); + +dbObservable.connect(); From 42303dc6aa431cec5b828ecddf83feb8f189a3a9 Mon Sep 17 00:00:00 2001 From: Berkeley Martinez Date: Fri, 14 Aug 2015 18:05:04 -0700 Subject: [PATCH 3/5] fix progressBar not showing reduce function needs initial seed closes #1753 --- server/boot/challenge.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/boot/challenge.js b/server/boot/challenge.js index 02d9ff92c5..3c0999741f 100644 --- a/server/boot/challenge.js +++ b/server/boot/challenge.js @@ -533,7 +533,7 @@ module.exports = function(app) { return sum + 1; } return sum; - }); + }, 0); return { name: blockArray[0].block, From bcdda9c62a15c4774e4bfa7777939cd703a19f4f Mon Sep 17 00:00:00 2001 From: Berkeley Martinez Date: Fri, 14 Aug 2015 18:25:34 -0700 Subject: [PATCH 4/5] fix unexpected syntax error on map --- server/views/challengeMap/show.jade | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/views/challengeMap/show.jade b/server/views/challengeMap/show.jade index 22927bdedb..b8be825254 100644 --- a/server/views/challengeMap/show.jade +++ b/server/views/challengeMap/show.jade @@ -1,7 +1,5 @@ extends ../layout block content - script. - var challenges = !{JSON.stringify(challenges)}; .bg-danger.default-border-radius p      a(href='https://github.com/FreeCodeCamp/freecodecamp/wiki/beta' target='_blank') You're using our experimental beta site. None of your progress here will be saved. Please click this to learn more. From f710fa1af60159b1c1bc9db8c398a1638ee8e233 Mon Sep 17 00:00:00 2001 From: Berkeley Martinez Date: Fri, 14 Aug 2015 18:48:24 -0700 Subject: [PATCH 5/5] remove pm2 memory fanciness --- pm2Start.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pm2Start.js b/pm2Start.js index 0884965273..2f27cfbd8e 100644 --- a/pm2Start.js +++ b/pm2Start.js @@ -5,8 +5,7 @@ pm2.connect(function() { script: 'server/production-start.js', 'exec_mode': 'cluster', instances: process.env.INSTANCES || 1, - 'max_memory_restart': - (process.env.MAX_MEMORY / process.env.INSTANCES || 1) || '300M', + 'max_memory_restart': process.env.MAX_MEMORY || '300M', 'NODE_ENV': 'production' }, function() { pm2.disconnect();