fix: add more tests for curriclum testing (#38464)

Co-Authored-By: Oliver Eyton-Williams <ojeytonwilliams@gmail.com>
Co-authored-by: Oliver Eyton-Williams <ojeytonwilliams@gmail.com>

curriculum/challenges/_meta/applied-accessibility/meta.json

@@ -86,7 +86,7 @@
     ],
     [
       "587d7790367417b2b2512aaf",
-      "Make Links Navigatable with HTML Access Keys"
+      "Make Links Navigable with HTML Access Keys"
     ],
     [
       "587d7790367417b2b2512ab0",

curriculum/getChallenges.js

@@ -2,6 +2,7 @@ const path = require('path');
 const { findIndex } = require('lodash');
 const readDirP = require('readdirp-walk');
 const { parseMarkdown } = require('@freecodecamp/challenge-md-parser');
+const fs = require('fs');
 
 const { dasherize } = require('../utils/slugs');
 
@@ -14,7 +15,14 @@ function getChallengesDirForLang(lang) {
   return path.resolve(challengesDir, `./${lang}`);
 }
 
+function getMetaForBlock(block) {
+  return JSON.parse(
+    fs.readFileSync(path.resolve(metaDir, `./${block}/meta.json`), 'utf8')
+  );
+}
+
 exports.getChallengesDirForLang = getChallengesDirForLang;
+exports.getMetaForBlock = getMetaForBlock;
 
 exports.getChallengesForLang = function getChallengesForLang(lang) {
   let curriculum = {};

curriculum/package-lock.json

@@ -5575,8 +5575,7 @@
         "ansi-regex": {
           "version": "2.1.1",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "aproba": {
           "version": "1.2.0",
@@ -5597,14 +5596,12 @@
         "balanced-match": {
           "version": "1.0.0",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "brace-expansion": {
           "version": "1.1.11",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "balanced-match": "^1.0.0",
             "concat-map": "0.0.1"
@@ -5619,20 +5616,17 @@
         "code-point-at": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "concat-map": {
           "version": "0.0.1",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "console-control-strings": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "core-util-is": {
           "version": "1.0.2",
@@ -5749,8 +5743,7 @@
         "inherits": {
           "version": "2.0.4",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "ini": {
           "version": "1.3.5",
@@ -5762,7 +5755,6 @@
           "version": "1.0.0",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "number-is-nan": "^1.0.0"
           }
@@ -5777,7 +5769,6 @@
           "version": "3.0.4",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "brace-expansion": "^1.1.7"
           }
@@ -5785,14 +5776,12 @@
         "minimist": {
           "version": "0.0.8",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "minipass": {
           "version": "2.9.0",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "safe-buffer": "^5.1.2",
             "yallist": "^3.0.0"
@@ -5811,7 +5800,6 @@
           "version": "0.5.1",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "minimist": "0.0.8"
           }
@@ -5901,8 +5889,7 @@
         "number-is-nan": {
           "version": "1.0.1",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "object-assign": {
           "version": "4.1.1",
@@ -5914,7 +5901,6 @@
           "version": "1.4.0",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "wrappy": "1"
           }
@@ -6000,8 +5986,7 @@
         "safe-buffer": {
           "version": "5.1.2",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "safer-buffer": {
           "version": "2.1.2",
@@ -6037,7 +6022,6 @@
           "version": "1.0.2",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "code-point-at": "^1.0.0",
             "is-fullwidth-code-point": "^1.0.0",
@@ -6057,7 +6041,6 @@
           "version": "3.0.1",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "ansi-regex": "^2.0.0"
           }
@@ -6101,14 +6084,12 @@
         "wrappy": {
           "version": "1.0.2",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "yallist": {
           "version": "3.1.1",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         }
       }
     },
@@ -14674,6 +14655,12 @@
       "integrity": "sha1-2sMECGkMIfPDYwo/86BYd73L1zY=",
       "dev": true
     },
+    "string-similarity": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/string-similarity/-/string-similarity-4.0.1.tgz",
+      "integrity": "sha512-v36MJzloekKVvKAsYi6O/qpn2mIuvwEFIT9Gx3yg4spkNjXYsk7yxc37g4ZTyMVIBvt/9PZGxnqEtme8XHK+Mw==",
+      "dev": true
+    },
     "string-width": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",

curriculum/package.json

@@ -60,6 +60,7 @@
     "readdirp-walk": "^1.7.0",
     "rx": "^4.1.0",
     "semantic-release": "^15.13.24",
+    "string-similarity": "^4.0.1",
     "validator": "^10.4.0"
   },
   "keywords": [

curriculum/test/test-challenges.js

@@ -1,5 +1,7 @@
+/* eslint-disable no-loop-func */
 const path = require('path');
 const liveServer = require('live-server');
+const stringSimilarity = require('string-similarity');
 
 const spinner = require('ora')();
 
@@ -32,7 +34,7 @@ const vm = require('vm');
 
 const puppeteer = require('puppeteer');
 
-const { getChallengesForLang } = require('../getChallenges');
+const { getChallengesForLang, getMetaForBlock } = require('../getChallenges');
 
 const MongoIds = require('./utils/mongoIds');
 const ChallengeTitles = require('./utils/challengeTitles');
@@ -86,59 +88,138 @@ spinner.text = 'Populate tests.';
 let browser;
 let page;
 
-runTests();
+setup()
-
+  .then(runTests)
-async function runTests() {
+  .catch(err => {
-  process.on('unhandledRejection', err => {
+    cleanup();
-    spinner.stop();
+    // setting the error code because node does not (yet) exit with a non-zero
-    throw new Error(`unhandledRejection: ${err.name}, ${err.message}`);
+    // code on unhandled exceptions.
+    process.exitCode = 1;
+    throw err;
   });
 
-  const testLangs = testedLangs();
+async function setup() {
+  if (process.env.npm_config_superblock && process.env.npm_config_block) {
+    throw new Error(`Please do not use both a block and superblock as input.`);
+  }
 
-  const challenges = await Promise.all(
+  // liveServer starts synchronously
+  liveServer.start({
+    host: '127.0.0.1',
+    port: '8080',
+    root: path.resolve(__dirname, 'stubs'),
+    mount: [['/js', path.join(clientPath, 'static/js')]],
+    open: false,
+    logLevel: 0
+  });
+  browser = await puppeteer.launch({
+    args: [
+      // Required for Docker version of Puppeteer
+      '--no-sandbox',
+      '--disable-setuid-sandbox',
+      // This will write shared memory files into /tmp instead of /dev/shm,
+      // because Docker’s default for /dev/shm is 64MB
+      '--disable-dev-shm-usage'
+      // dumpio: true
+    ]
+  });
+  global.Worker = createPseudoWorker(await newPageContext(browser));
+  page = await newPageContext(browser);
+  await page.setViewport({ width: 300, height: 150 });
+  const testLangs = testedLangs();
+  if (testLangs.length > 1)
+    throw Error(
+      `Testing more than one language at once is not currently supported
+please change the TEST_CHALLENGES_FOR_LANGS env variable to a single language`
+    );
+  const challengesForLang = await Promise.all(
     testLangs.map(lang => getChallenges(lang))
   );
 
-  describe('Check challenges', function() {
+  // the next few statements create a list of all blocks and superblocks
-    before(async function() {
+  // as they appear in the list of challenges
-      spinner.text = 'Testing';
+  const blocks = challengesForLang[0].challenges.map(({ block }) => block);
-      this.timeout(50000);
+  const superBlocks = challengesForLang[0].challenges.map(
-      liveServer.start({
+    ({ superBlock }) => superBlock
-        host: '127.0.0.1',
+  );
-        port: '8080',
+  const targetBlockStrings = [...new Set(blocks)];
-        root: path.resolve(__dirname, 'stubs'),
+  const targetSuperBlockStrings = [...new Set(superBlocks)];
-        mount: [['/js', path.join(clientPath, 'static/js')]],
-        open: false,
-        logLevel: 0
-      });
-      browser = await puppeteer.launch({
-        args: [
-          // Required for Docker version of Puppeteer
-          '--no-sandbox',
-          '--disable-setuid-sandbox',
-          // This will write shared memory files into /tmp instead of /dev/shm,
-          // because Docker’s default for /dev/shm is 64MB
-          '--disable-dev-shm-usage'
-          // dumpio: true
-        ]
-      });
-      global.Worker = createPseudoWorker(await newPageContext(browser));
-      page = await newPageContext(browser);
-      await page.setViewport({ width: 300, height: 150 });
-    });
-    after(async function() {
-      this.timeout(30000);
-      if (browser) {
-        await browser.close();
-      }
-      liveServer.shutdown();
-      spinner.stop();
-    });
 
-    challenges.forEach(populateTestsForLang);
+  // the next few statements will filter challenges based on command variables
+  if (process.env.npm_config_superblock) {
+    const filter = stringSimilarity.findBestMatch(
+      process.env.npm_config_superblock,
+      targetSuperBlockStrings
+    ).bestMatch.target;
+
+    console.log(`\nsuperBlock being tested: ${filter}`);
+    challengesForLang[0].challenges = challengesForLang[0].challenges.filter(
+      challenge => challenge.superBlock === filter
+    );
+
+    if (!challengesForLang[0].challenges.length) {
+      throw new Error(`No challenges found with superBlock "${filter}"`);
+    }
+  }
+
+  if (process.env.npm_config_block) {
+    const filter = stringSimilarity.findBestMatch(
+      process.env.npm_config_block,
+      targetBlockStrings
+    ).bestMatch.target;
+
+    console.log(`\nblock being tested: ${filter}`);
+    challengesForLang[0].challenges = challengesForLang[0].challenges.filter(
+      challenge => challenge.block === filter
+    );
+
+    if (!challengesForLang[0].challenges.length) {
+      throw new Error(`No challenges found with block "${filter}"`);
+    }
+  }
+
+  const meta = {};
+  for (const { lang, challenges } of challengesForLang) {
+    meta[lang] = {};
+    for (const challenge of challenges) {
+      const dashedBlockName = dasherize(challenge.block);
+      if (!meta[dashedBlockName]) {
+        meta[lang][dashedBlockName] = (await getMetaForBlock(
+          dashedBlockName
+        )).challengeOrder;
+      }
+    }
+  }
+  return {
+    meta,
+    challengesForLang
+  };
+}
+
+// cleanup calls some async functions, but it's the last thing that happens, so
+// no need to await anything.
+function cleanup() {
+  if (browser) {
+    browser.close();
+  }
+  liveServer.shutdown();
+  spinner.stop();
+}
+
+function runTests({ challengesForLang, meta }) {
+  process.on('unhandledRejection', err => {
+    throw new Error(`unhandledRejection: ${err.name}, ${err.message}`);
   });
 
+  describe('Check challenges', function() {
+    after(function() {
+      cleanup();
+    });
+    for (const challenge of challengesForLang) {
+      populateTestsForLang(challenge, meta);
+    }
+  });
+  spinner.text = 'Testing';
   run();
 }
 
@@ -165,130 +246,156 @@ function validateBlock(challenge) {
   }
 }
 
-function populateTestsForLang({ lang, challenges }) {
+function populateTestsForLang({ lang, challenges }, meta) {
   const mongoIds = new MongoIds();
   const challengeTitles = new ChallengeTitles();
   const validateChallenge = challengeSchemaValidator(lang);
 
   describe(`Check challenges (${lang})`, function() {
     this.timeout(5000);
-
     challenges.forEach(challenge => {
-      describe(challenge.title || 'No title', function() {
+      const dashedBlockName = dasherize(challenge.block);
-        it('Common checks', function() {
+      describe(challenge.block || 'No block', function() {
-          const result = validateChallenge(challenge);
+        describe(challenge.title || 'No title', function() {
-          const invalidBlock = validateBlock(challenge);
+          it('Matches a title in meta.json', function() {
+            const index = meta[lang][dashedBlockName].findIndex(
+              arr => arr[1] === challenge.title
+            );
 
-          if (result.error) {
+            if (index < 0) {
-            throw new AssertionError(result.error);
+              throw new AssertionError(
+                `Cannot find title "${challenge.title}" in meta.json file`
+              );
+            }
+          });
+
+          it('Matches an ID in meta.json', function() {
+            const index = meta[lang][dashedBlockName].findIndex(
+              arr => arr[0] === challenge.id
+            );
+
+            if (index < 0) {
+              throw new AssertionError(
+                `Cannot find ID "${challenge.id}" in meta.json file`
+              );
+            }
+          });
+
+          it('Common checks', function() {
+            const result = validateChallenge(challenge);
+            const invalidBlock = validateBlock(challenge);
+
+            if (result.error) {
+              throw new AssertionError(result.error);
+            }
+            if (challenge.challengeType !== 7 && invalidBlock) {
+              throw new Error(invalidBlock);
+            }
+            const { id, title } = challenge;
+            mongoIds.check(id, title);
+            challengeTitles.check(title);
+          });
+
+          const { challengeType } = challenge;
+          if (
+            challengeType !== challengeTypes.html &&
+            challengeType !== challengeTypes.js &&
+            challengeType !== challengeTypes.bonfire &&
+            challengeType !== challengeTypes.modern &&
+            challengeType !== challengeTypes.backend
+          ) {
+            return;
           }
-          if (challenge.challengeType !== 7 && invalidBlock) {
+
-            throw new Error(invalidBlock);
+          let { tests = [] } = challenge;
+          tests = tests.filter(test => !!test.testString);
+          if (tests.length === 0) {
+            it('Check tests. No tests.');
+            return;
           }
-          const { id, title } = challenge;
-          mongoIds.check(id, title);
-          challengeTitles.check(title);
-        });
 
-        const { challengeType } = challenge;
+          describe('Check tests syntax', function() {
-        if (
+            tests.forEach(test => {
-          challengeType !== challengeTypes.html &&
+              it(`Check for: ${test.text}`, function() {
-          challengeType !== challengeTypes.js &&
+                assert.doesNotThrow(() => new vm.Script(test.testString));
-          challengeType !== challengeTypes.bonfire &&
+              });
-          challengeType !== challengeTypes.modern &&
-          challengeType !== challengeTypes.backend
-        ) {
-          return;
-        }
-
-        let { tests = [] } = challenge;
-        tests = tests.filter(test => !!test.testString);
-        if (tests.length === 0) {
-          it('Check tests. No tests.');
-          return;
-        }
-
-        describe('Check tests syntax', function() {
-          tests.forEach(test => {
-            it(`Check for: ${test.text}`, function() {
-              assert.doesNotThrow(() => new vm.Script(test.testString));
             });
           });
-        });
 
-        let { files = [] } = challenge;
+          let { files = [] } = challenge;
-        let createTestRunner;
+          let createTestRunner;
-        if (challengeType === challengeTypes.backend) {
+          if (challengeType === challengeTypes.backend) {
-          it('Check tests is not implemented.');
+            it('Check tests is not implemented.');
-          return;
+            return;
-        } else if (
+          } else if (
-          challengeType === challengeTypes.js ||
+            challengeType === challengeTypes.js ||
-          challengeType === challengeTypes.bonfire
+            challengeType === challengeTypes.bonfire
-        ) {
+          ) {
-          createTestRunner = createTestRunnerForJSChallenge;
+            createTestRunner = createTestRunnerForJSChallenge;
-        } else if (files.length === 1) {
+          } else if (files.length === 1) {
-          createTestRunner = createTestRunnerForDOMChallenge;
+            createTestRunner = createTestRunnerForDOMChallenge;
-        } else {
+          } else {
-          it('Check tests.', () => {
+            it('Check tests.', () => {
-            throw new Error('Seed file should be only the one.');
+              throw new Error('Seed file should be only the one.');
-          });
+            });
-          return;
+            return;
-        }
-
-        files = files.map(createPoly);
-        it('Test suite must fail on the initial contents', async function() {
-          this.timeout(5000 * tests.length + 1000);
-          // suppress errors in the console.
-          const oldConsoleError = console.error;
-          console.error = () => {};
-          let fails = false;
-          let testRunner;
-          try {
-            testRunner = await createTestRunner(
-              { ...challenge, files },
-              '',
-              page
-            );
-          } catch {
-            fails = true;
           }
-          if (!fails) {
-            for (const test of tests) {
-              try {
-                await testRunner(test);
-              } catch (e) {
-                fails = true;
-                break;
-              }
-            }
-          }
-          console.error = oldConsoleError;
-          assert(fails, 'Test suit does not fail on the initial contents');
-        });
 
-        let { solutions = [] } = challenge;
+          files = files.map(createPoly);
-        const noSolution = new RegExp('// solution required');
+          it('Test suite must fail on the initial contents', async function() {
-        solutions = solutions.filter(
+            this.timeout(5000 * tests.length + 1000);
-          solution => !!solution && !noSolution.test(solution)
+            // suppress errors in the console.
-        );
+            const oldConsoleError = console.error;
-
+            console.error = () => {};
-        if (solutions.length === 0) {
+            let fails = false;
-          it('Check tests. No solutions');
+            let testRunner;
-          return;
+            try {
-        }
+              testRunner = await createTestRunner(
-
-        describe('Check tests against solutions', function() {
-          solutions.forEach((solution, index) => {
-            it(`Solution ${index + 1} must pass the tests`, async function() {
-              this.timeout(5000 * tests.length + 1000);
-              const testRunner = await createTestRunner(
                 { ...challenge, files },
-                solution,
+                '',
                 page
               );
+            } catch {
+              fails = true;
+            }
+            if (!fails) {
               for (const test of tests) {
-                await testRunner(test);
+                try {
+                  await testRunner(test);
+                } catch (e) {
+                  fails = true;
+                  break;
+                }
               }
+            }
+            console.error = oldConsoleError;
+            assert(fails, 'Test suit does not fail on the initial contents');
+          });
+
+          let { solutions = [] } = challenge;
+          const noSolution = new RegExp('// solution required');
+          solutions = solutions.filter(
+            solution => !!solution && !noSolution.test(solution)
+          );
+
+          if (solutions.length === 0) {
+            it('Check tests. No solutions');
+            return;
+          }
+
+          describe('Check tests against solutions', function() {
+            solutions.forEach((solution, index) => {
+              it(`Solution ${index + 1} must pass the tests`, async function() {
+                this.timeout(5000 * tests.length + 1000);
+                const testRunner = await createTestRunner(
+                  { ...challenge, files },
+                  solution,
+                  page
+                );
+                for (const test of tests) {
+                  await testRunner(test);
+                }
+              });
             });
           });
         });

docs/how-to-setup-freecodecamp-locally.md

@@ -307,16 +307,18 @@ A quick reference to the commands that you will need when working locally.
 
 **Local Build:**
 
-| command                   | description                                                                         |
+| command                                                        | description                                                                         |
-| ------------------------- | ----------------------------------------------------------------------------------- |
+| -------------------------------------------------------------- | ----------------------------------------------------------------------------------- |
-| `npm ci`                  | Installs / re-install all dependencies and bootstraps the different services.       |
+| `npm ci`                                                       | Installs / re-install all dependencies and bootstraps the different services.       |
-| `npm run seed`            | Parses all the challenge markdown files and inserts them into MongoDB.              |
+| `npm run seed`                                                 | Parses all the challenge markdown files and inserts them into MongoDB.              |
-| `npm run develop`         | Starts the freeCodeCamp API Server and Client Applications.                         |
+| `npm run develop`                                              | Starts the freeCodeCamp API Server and Client Applications.                         |
-| `npm test`                | Run all JS tests in the system, including client, server, lint and challenge tests. |
+| `npm test`                                                     | Run all JS tests in the system, including client, server, lint and challenge tests. |
-| `npm run test:client`     | Run the client test suite.                                                          |
+| `npm run test:client`                                          | Run the client test suite.                                                          |
-| `npm run test:curriculum` | Run the curriculum test suite.                                                      |
+| `npm run test:curriculum`                                      | Run the curriculum test suite.                                                      |
-| `npm run test:server`     | Run the server test suite.                                                          |
+| `npm run test:curriculum --block='Basic HTML and HTML5'`       | Test a specific Block.                                                              |
-| `npm run clean`           | Uninstalls all dependencies and cleans up caches.                                   |
+| `npm run test:curriculum --superblock='responsive-web-design'` | Test a specific SuperBlock.                                                         |
+| `npm run test:server`                                          | Run the server test suite.                                                          |
+| `npm run clean`                                                | Uninstalls all dependencies and cleans up caches.                                   |
 
 ## Making changes locally