fix: re-revert the API decoupling (#41263)

* fix(api): decouple api from curriculum

This reverts commit 8f0e441644 and
introduces the implementations from #40703.

* fix(gitpod): add curriculum build to GitPod

This reverts commit 706d70f58d and
introduces implementations from #41234.

* docs: update DevOps manual for api change (#41259)

Co-authored-by: Oliver Eyton-Williams <ojeytonwilliams@gmail.com>

.eslintignore

@@ -2,6 +2,6 @@
 client/.cache/**
 client/static/**
 client/public/**
-api-server/public/**
+api-server/src/public/**
 api-server/lib/**
 tools/contributor/**

.github/workflows/cypress.yml

@@ -51,6 +51,7 @@ jobs:
         run: |
           npm ci
           npm run ensure-env
+          npm run build:curriculum
 
       - name: Seed Database
         run: npm run seed
@@ -60,9 +61,9 @@ jobs:
         with:
           record: ${{ env.CYPRESS_RECORD_KEY != 0 }}
           build: npm run build
-          # this should mirror the production build, but for now we're just using the dev
+          # this should mirror the production build, but for now we're using gatsby
-          # server and gatsby serve instead (the npm script serve:client needs updating!)
+          # serve instead (the npm script serve:client needs updating!)
-          start: npm start
+          start: npm run start-ci
           wait-on: http://localhost:8000
           # the site builds in about 8 minutes, so there is currently 12 minutes of time
           # left for testing.

.github/workflows/node.js-tests.yml

@@ -39,6 +39,7 @@ jobs:
         run: |
           npm ci
           npm run ensure-env
+          npm run build:curriculum
           npm run lint
 
   test:
@@ -79,6 +80,7 @@ jobs:
         run: |
           npm ci
           npm run ensure-env
+          npm run build:curriculum
 
       - name: Run Tests
         run: npm test

.gitignore

@@ -159,6 +159,7 @@ config/env.json
 config/client/sass-compile.json
 config/client/frame-runner.json
 config/client/test-evaluator.json
+config/curriculum.json
 
 ### Additional Files ###
 *.csv

.gitpod.yml

@@ -40,6 +40,7 @@ tasks:
       mongo --eval "db.fsyncLock(); db.fsyncUnlock()"
     command: >
       npm run ensure-env &&
+      npm run build:curriculum &&
       gp await-port 27017 &&
       npm run develop:server
 

api-server/package-lock.json

@@ -4922,10 +4922,9 @@
       "dev": true
     },
     "@types/passport": {
-      "version": "1.0.0",
+      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/@types/passport/-/passport-1.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/@types/passport/-/passport-1.0.5.tgz",
-      "integrity": "sha512-R2FXqM+AgsMIym0PuKj08Ybx+GR6d2rU3b1/8OcHolJ+4ga2pRPX105wboV6hq1AJvMo2frQzYKdqXS5+4cyMw==",
+      "integrity": "sha512-wNL4kT/5rnZgyGkqX7V2qH/R/te+bklv+nXcvHzyX99vNggx9DGN+F8CEOW3P/gRi7Cjm991uidRgTHsYkSuMg==",
-      "dev": true,
       "requires": {
         "@types/express": "*"
       }
@@ -4936,6 +4935,11 @@
       "integrity": "sha512-UEyp8LwZ4Dg30kVU2Q3amHHyTn1jEdhCIE59ANed76GaT1Vp76DD3ZWSAxgCrw6wJ0TqeoBpqmfUHiUDPs//HQ==",
       "dev": true
     },
+    "@types/qs": {
+      "version": "6.9.5",
+      "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.5.tgz",
+      "integrity": "sha512-/JHkVHtx/REVG0VVToGRGH2+23hsYLHdyG+GrvoUGlGAd0ErauXDyvHtRI/7H7mzLm+tBCKA7pfcpkQ1lf58iQ=="
+    },
     "@types/range-parser": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.2.tgz",
@@ -6458,7 +6462,6 @@
       "version": "7.0.3",
       "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz",
       "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==",
-      "dev": true,
       "requires": {
         "cross-spawn": "^7.0.1"
       }
@@ -6908,6 +6911,11 @@
         "is-symbol": "^1.0.2"
       }
     },
+    "es6-promise": {
+      "version": "4.2.8",
+      "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz",
+      "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w=="
+    },
     "es6-promisify": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz",
@@ -15176,7 +15184,6 @@
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/passport-mock-strategy/-/passport-mock-strategy-2.0.0.tgz",
       "integrity": "sha512-9YUT0sja/7n+HfQ+Jwx4XETERRh1uciRjpHhEZMcYS1FBnMrfrSlKVS42bMU06ewSFiPhXztazAE6XwiZdZQ/g==",
-      "dev": true,
       "requires": {
         "@types/express": "^4.16.1",
         "@types/passport": "^1.0.0",
@@ -15185,21 +15192,25 @@
       },
       "dependencies": {
         "@types/express": {
-          "version": "4.17.0",
+          "version": "4.17.11",
-          "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.0.tgz",
+          "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.11.tgz",
-          "integrity": "sha512-CjaMu57cjgjuZbh9DpkloeGxV45CnMGlVd+XpG7Gm9QgVrd7KFq+X4HY0vM+2v0bczS48Wg7bvnMY5TN+Xmcfw==",
+          "integrity": "sha512-no+R6rW60JEc59977wIxreQVsIEOAYwgCqldrA/vkpCnbD7MqTefO97lmoBe4WE0F156bC4uLSP1XHDOySnChg==",
-          "dev": true,
           "requires": {
             "@types/body-parser": "*",
-            "@types/express-serve-static-core": "*",
+            "@types/express-serve-static-core": "^4.17.18",
+            "@types/qs": "*",
             "@types/serve-static": "*"
           }
         },
-        "es6-promise": {
+        "@types/express-serve-static-core": {
-          "version": "4.2.8",
+          "version": "4.17.18",
-          "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz",
+          "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.18.tgz",
-          "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==",
+          "integrity": "sha512-m4JTwx5RUBNZvky/JJ8swEJPKFd8si08pPF2PfizYjGZOKr/svUWPcoUmLow6MmPzhasphB7gSTINY67xn3JNA==",
-          "dev": true
+          "requires": {
+            "@types/node": "*",
+            "@types/qs": "*",
+            "@types/range-parser": "*"
+          }
         }
       }
     },

api-server/package.json

@@ -4,11 +4,10 @@
   "version": "0.0.1",
   "repository": "freecodecamp/freecodecamp",
   "scripts": {
-    "develop": "cross-env DEBUG=fcc* node development-start.js",
+    "develop": "cross-env DEBUG=fcc* node src/development-start.js",
-    "babel-dev-server": "babel-node --inspect=0.0.0.0 ./server/index.js",
+    "babel-dev-server": "babel-node --inspect=0.0.0.0 ./src/server/index.js",
-    "build": "babel server --out-dir lib --ignore 'node_modules /**/*','/**/*.test.js' --copy-files",
+    "build": "babel src --out-dir lib --ignore 'node_modules /**/*','/**/*.test.js' --copy-files",
-    "ensure-env": "node ../tools/scripts/build/ensure-env.js",
+    "start": "cross-env DEBUG=fcc* node lib/production-start.js",
-    "start": "cross-env DEBUG=fcc* node production-start.js",
     "test": "jest"
   },
   "license": "BSD-3-Clause",
@@ -23,6 +22,7 @@
     "connect-mongo": "^3.2.0",
     "cookie-parser": "^1.4.5",
     "cors": "^2.8.5",
+    "cross-env": "^7.0.3",
     "csurf": "^1.11.0",
     "d3": "~3.5.17",
     "date-fns": "^1.30.1",
@@ -51,6 +51,7 @@
     "passport-auth0": "^1.4.0",
     "passport-local": "^1.0.0",
     "query-string": "^6.14.0",
+    "passport-mock-strategy": "^2.0.0",
     "rx": "^4.1.0",
     "stripe": "^6.36.0",
     "uuid": "^3.4.0",
@@ -69,11 +70,9 @@
     "babel-jest": "^24.9.0",
     "babel-plugin-transform-function-bind": "^6.22.0",
     "babel-plugin-transform-imports": "^1.5.1",
-    "cross-env": "^7.0.3",
     "jest": "^26.6.3",
     "loopback-component-explorer": "^6.4.0",
     "nodemon": "^2.0.7",
-    "passport-mock-strategy": "^2.0.0",
     "smee-client": "^1.2.2"
   },
   "resolutions": {

api-server/server/utils/get-curriculum.js

@@ -1,30 +0,0 @@
-import { flatten } from 'lodash';
-
-import { getChallengesForLang } from '../../../curriculum/getChallenges';
-
-// TODO: this caching is handy if we want to field requests that need to 'query'
-// the curriculum, but if we force the client to handle
-// redirectToCurrentChallenge and, instead, only report the current challenge
-// id via the user object, then we should *not* store this so it can be garbage
-// collected.
-
-let curriculum;
-export async function getCurriculum() {
-  // NOTE: this is always 'english' because we are only interested in the slugs
-  // and those should not change between the languages.
-  curriculum = curriculum ? curriculum : getChallengesForLang('english');
-  return curriculum;
-}
-
-export async function getChallenges() {
-  return getCurriculum().then(curriculum => {
-    return Object.keys(curriculum)
-      .map(key => curriculum[key].blocks)
-      .reduce((challengeArray, superBlock) => {
-        const challengesForBlock = Object.keys(superBlock).map(
-          key => superBlock[key].challenges
-        );
-        return [...challengeArray, ...flatten(challengesForBlock)];
-      }, []);
-  });
-}

api-server/src/common/models/user.js

@@ -15,7 +15,7 @@ import _ from 'lodash';
 import generate from 'nanoid/generate';
 import badwordFilter from 'bad-words';
 
-import { apiLocation } from '../../../config/env';
+import { apiLocation } from '../../../../config/env';
 
 import {
   fixCompletedChallengeItem,

api-server/src/development-start.js

@@ -1,5 +1,5 @@
 const path = require('path');
-require('dotenv').config({ path: path.resolve(__dirname, '../.env') });
+require('dotenv').config({ path: path.resolve(__dirname, '../../.env') });
 
 const nodemon = require('nodemon');
 const SmeeClient = require('smee-client');

api-server/src/production-start.js

@@ -1,5 +1,4 @@
 // this ensures node understands the future
-require('@babel/register');
 const _ = require('lodash');
 const createDebugger = require('debug');
 

api-server/src/server/boot/authentication.js

@@ -4,7 +4,7 @@ import { check } from 'express-validator';
 import { isEmail } from 'validator';
 import jwt from 'jsonwebtoken';
 
-import { jwtSecret } from '../../../config/secrets';
+import { jwtSecret } from '../../../../config/secrets';
 
 import {
   createPassportCallbackAuthenticator,

api-server/src/server/boot/certificate.js

@@ -20,7 +20,7 @@ import {
   certTypeIdMap,
   certIds,
   oldDataVizId
-} from '../../../config/certification-settings';
+} from '../../../../config/certification-settings';
 
 const {
   legacyFrontEndChallengeId,
@@ -42,12 +42,11 @@ const {
 
 const log = debug('fcc:certification');
 
-export default function bootCertificate(app, done) {
+export default function bootCertificate(app) {
   const api = app.loopback.Router();
   // TODO: rather than getting all the challenges, then grabbing the certs,
   // consider just getting the certs.
-  getChallenges().then(allChallenges => {
+  const certTypeIds = createCertTypeIds(getChallenges());
-    const certTypeIds = createCertTypeIds(allChallenges);
   const showCert = createShowCert(app);
   const verifyCert = createVerifyCert(certTypeIds, app);
 
@@ -55,8 +54,6 @@ export default function bootCertificate(app, done) {
   api.get('/certificate/showCert/:username/:cert', showCert);
 
   app.use(api);
-    done();
-  });
 }
 
 export function getFallbackFrontEndDate(completedChallenges, completedDate) {

api-server/src/server/boot/challenge.js

@@ -28,7 +28,7 @@ export default async function bootChallenge(app, done) {
   const api = app.loopback.Router();
   const router = app.loopback.Router();
   const challengeUrlResolver = await createChallengeUrlResolver(
-    await getChallenges()
+    getChallenges()
   );
   const redirectToCurrentChallenge = createRedirectToCurrentChallenge(
     challengeUrlResolver,

api-server/src/server/boot/donate.js

@@ -12,8 +12,8 @@ import {
   durationKeysConfig,
   donationOneTimeConfig,
   donationSubscriptionConfig
-} from '../../../config/donation-settings';
+} from '../../../../config/donation-settings';
-import keys from '../../../config/secrets';
+import keys from '../../../../config/secrets';
 
 const log = debug('fcc:boot:donate');
 

api-server/src/server/boot/randomAPIs.js

@@ -1,6 +1,6 @@
 import request from 'request';
 
-import { gitHubUserAgent } from '../../../config/misc';
+import { gitHubUserAgent } from '../../../../config/misc';
 import { getRedirectParams } from '../utils/redirection';
 
 const githubClient = process.env.GITHUB_ID;

api-server/src/server/boot/settings.js

@@ -4,7 +4,7 @@ import { check } from 'express-validator';
 import { ifNoUser401, createValidatorErrorHandler } from '../utils/middleware';
 import { themes } from '../../common/utils/themes.js';
 import { alertTypes } from '../../common/utils/flash.js';
-import { isValidUsername } from '../../../utils/validate';
+import { isValidUsername } from '../../../../utils/validate';
 
 const log = debug('fcc:boot:settings');
 

api-server/src/server/component-passport.js

@@ -9,13 +9,13 @@ import dedent from 'dedent';
 import { getUserById } from './utils/user-stats';
 import passportProviders from './passport-providers';
 import { setAccessTokenToResponse } from './utils/getSetAccessToken';
-import { jwtSecret } from '../../config/secrets';
 import {
   getReturnTo,
   getRedirectBase,
   getRedirectParams,
   isRootPath
 } from './utils/redirection';
+import { jwtSecret } from '../../../config/secrets';
 
 const passportOptions = {
   emailOptional: true,

api-server/src/server/datasources.production.js

@@ -1,4 +1,4 @@
-var secrets = require('../../config/secrets');
+var secrets = require('../../../config/secrets');
 
 module.exports = {
   db: {

api-server/src/server/index.js

@@ -1,5 +1,5 @@
 const path = require('path');
-require('dotenv').config({ path: path.resolve(__dirname, '../../.env') });
+require('dotenv').config({ path: path.resolve(__dirname, '../../../.env') });
 
 const _ = require('lodash');
 const Rx = require('rx');
@@ -9,7 +9,7 @@ const createDebugger = require('debug');
 const morgan = require('morgan');
 const Sentry = require('@sentry/node');
 
-const { sentry } = require('../../config/secrets');
+const { sentry } = require('../../../config/secrets');
 const { setupPassport } = require('./component-passport');
 
 const log = createDebugger('fcc:server');

api-server/src/server/middlewares/constant-headers.js

@@ -1,5 +1,5 @@
-import { homeLocation } from '../../../config/env';
+import { homeLocation } from '../../../../config/env';
-import { allowedOrigins } from '../../../config/cors-settings';
+import { allowedOrigins } from '../../../../config/cors-settings';
 
 export default function constantHeaders() {
   return function(req, res, next) {

api-server/src/server/middlewares/csp.js

@@ -1,6 +1,6 @@
 import helmet from 'helmet';
 
-import { homeLocation } from '../../../config/env';
+import { homeLocation } from '../../../../config/env';
 
 let trusted = [
   "'self'",