fix: add more tests for curriclum testing (#38464)

Co-Authored-By: Oliver Eyton-Williams <ojeytonwilliams@gmail.com>
Co-authored-by: Oliver Eyton-Williams <ojeytonwilliams@gmail.com>

curriculum/challenges/_meta/applied-accessibility/meta.json

@@ -86,7 +86,7 @@
     ],
     [
       "587d7790367417b2b2512aaf",
-      "Make Links Navigatable with HTML Access Keys"
+      "Make Links Navigable with HTML Access Keys"
     ],
     [
       "587d7790367417b2b2512ab0",

curriculum/getChallenges.js

@@ -2,6 +2,7 @@ const path = require('path');
 const { findIndex } = require('lodash');
 const readDirP = require('readdirp-walk');
 const { parseMarkdown } = require('@freecodecamp/challenge-md-parser');
+const fs = require('fs');
 
 const { dasherize } = require('../utils/slugs');
 
@@ -14,7 +15,14 @@ function getChallengesDirForLang(lang) {
   return path.resolve(challengesDir, `./${lang}`);
 }
 
+function getMetaForBlock(block) {
+  return JSON.parse(
+    fs.readFileSync(path.resolve(metaDir, `./${block}/meta.json`), 'utf8')
+  );
+}
+
 exports.getChallengesDirForLang = getChallengesDirForLang;
+exports.getMetaForBlock = getMetaForBlock;
 
 exports.getChallengesForLang = function getChallengesForLang(lang) {
   let curriculum = {};

curriculum/package-lock.json

@@ -5575,8 +5575,7 @@
         "ansi-regex": {
           "version": "2.1.1",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "aproba": {
           "version": "1.2.0",
@@ -5597,14 +5596,12 @@
         "balanced-match": {
           "version": "1.0.0",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "brace-expansion": {
           "version": "1.1.11",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "balanced-match": "^1.0.0",
             "concat-map": "0.0.1"
@@ -5619,20 +5616,17 @@
         "code-point-at": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "concat-map": {
           "version": "0.0.1",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "console-control-strings": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "core-util-is": {
           "version": "1.0.2",
@@ -5749,8 +5743,7 @@
         "inherits": {
           "version": "2.0.4",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "ini": {
           "version": "1.3.5",
@@ -5762,7 +5755,6 @@
           "version": "1.0.0",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "number-is-nan": "^1.0.0"
           }
@@ -5777,7 +5769,6 @@
           "version": "3.0.4",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "brace-expansion": "^1.1.7"
           }
@@ -5785,14 +5776,12 @@
         "minimist": {
           "version": "0.0.8",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "minipass": {
           "version": "2.9.0",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "safe-buffer": "^5.1.2",
             "yallist": "^3.0.0"
@@ -5811,7 +5800,6 @@
           "version": "0.5.1",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "minimist": "0.0.8"
           }
@@ -5901,8 +5889,7 @@
         "number-is-nan": {
           "version": "1.0.1",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "object-assign": {
           "version": "4.1.1",
@@ -5914,7 +5901,6 @@
           "version": "1.4.0",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "wrappy": "1"
           }
@@ -6000,8 +5986,7 @@
         "safe-buffer": {
           "version": "5.1.2",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "safer-buffer": {
           "version": "2.1.2",
@@ -6037,7 +6022,6 @@
           "version": "1.0.2",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "code-point-at": "^1.0.0",
             "is-fullwidth-code-point": "^1.0.0",
@@ -6057,7 +6041,6 @@
           "version": "3.0.1",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "ansi-regex": "^2.0.0"
           }
@@ -6101,14 +6084,12 @@
         "wrappy": {
           "version": "1.0.2",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "yallist": {
           "version": "3.1.1",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         }
       }
     },
@@ -14674,6 +14655,12 @@
       "integrity": "sha1-2sMECGkMIfPDYwo/86BYd73L1zY=",
       "dev": true
     },
+    "string-similarity": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/string-similarity/-/string-similarity-4.0.1.tgz",
+      "integrity": "sha512-v36MJzloekKVvKAsYi6O/qpn2mIuvwEFIT9Gx3yg4spkNjXYsk7yxc37g4ZTyMVIBvt/9PZGxnqEtme8XHK+Mw==",
+      "dev": true
+    },
     "string-width": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",

curriculum/package.json

@@ -60,6 +60,7 @@
     "readdirp-walk": "^1.7.0",
     "rx": "^4.1.0",
     "semantic-release": "^15.13.24",
+    "string-similarity": "^4.0.1",
     "validator": "^10.4.0"
   },
   "keywords": [

curriculum/test/test-challenges.js

@@ -1,5 +1,7 @@
+/* eslint-disable no-loop-func */
 const path = require('path');
 const liveServer = require('live-server');
+const stringSimilarity = require('string-similarity');
 
 const spinner = require('ora')();
 
@@ -32,7 +34,7 @@ const vm = require('vm');
 
 const puppeteer = require('puppeteer');
 
-const { getChallengesForLang } = require('../getChallenges');
+const { getChallengesForLang, getMetaForBlock } = require('../getChallenges');
 
 const MongoIds = require('./utils/mongoIds');
 const ChallengeTitles = require('./utils/challengeTitles');
@@ -86,24 +88,22 @@ spinner.text = 'Populate tests.';
 let browser;
 let page;
 
-runTests();
+setup()
-
+  .then(runTests)
-async function runTests() {
+  .catch(err => {
-  process.on('unhandledRejection', err => {
+    cleanup();
-    spinner.stop();
+    // setting the error code because node does not (yet) exit with a non-zero
-    throw new Error(`unhandledRejection: ${err.name}, ${err.message}`);
+    // code on unhandled exceptions.
+    process.exitCode = 1;
+    throw err;
   });
 
-  const testLangs = testedLangs();
+async function setup() {
+  if (process.env.npm_config_superblock && process.env.npm_config_block) {
+    throw new Error(`Please do not use both a block and superblock as input.`);
+  }
 
-  const challenges = await Promise.all(
+  // liveServer starts synchronously
-    testLangs.map(lang => getChallenges(lang))
-  );
-
-  describe('Check challenges', function() {
-    before(async function() {
-      spinner.text = 'Testing';
-      this.timeout(50000);
   liveServer.start({
     host: '127.0.0.1',
     port: '8080',
@@ -126,19 +126,100 @@ async function runTests() {
   global.Worker = createPseudoWorker(await newPageContext(browser));
   page = await newPageContext(browser);
   await page.setViewport({ width: 300, height: 150 });
-    });
+  const testLangs = testedLangs();
-    after(async function() {
+  if (testLangs.length > 1)
-      this.timeout(30000);
+    throw Error(
+      `Testing more than one language at once is not currently supported
+please change the TEST_CHALLENGES_FOR_LANGS env variable to a single language`
+    );
+  const challengesForLang = await Promise.all(
+    testLangs.map(lang => getChallenges(lang))
+  );
+
+  // the next few statements create a list of all blocks and superblocks
+  // as they appear in the list of challenges
+  const blocks = challengesForLang[0].challenges.map(({ block }) => block);
+  const superBlocks = challengesForLang[0].challenges.map(
+    ({ superBlock }) => superBlock
+  );
+  const targetBlockStrings = [...new Set(blocks)];
+  const targetSuperBlockStrings = [...new Set(superBlocks)];
+
+  // the next few statements will filter challenges based on command variables
+  if (process.env.npm_config_superblock) {
+    const filter = stringSimilarity.findBestMatch(
+      process.env.npm_config_superblock,
+      targetSuperBlockStrings
+    ).bestMatch.target;
+
+    console.log(`\nsuperBlock being tested: ${filter}`);
+    challengesForLang[0].challenges = challengesForLang[0].challenges.filter(
+      challenge => challenge.superBlock === filter
+    );
+
+    if (!challengesForLang[0].challenges.length) {
+      throw new Error(`No challenges found with superBlock "${filter}"`);
+    }
+  }
+
+  if (process.env.npm_config_block) {
+    const filter = stringSimilarity.findBestMatch(
+      process.env.npm_config_block,
+      targetBlockStrings
+    ).bestMatch.target;
+
+    console.log(`\nblock being tested: ${filter}`);
+    challengesForLang[0].challenges = challengesForLang[0].challenges.filter(
+      challenge => challenge.block === filter
+    );
+
+    if (!challengesForLang[0].challenges.length) {
+      throw new Error(`No challenges found with block "${filter}"`);
+    }
+  }
+
+  const meta = {};
+  for (const { lang, challenges } of challengesForLang) {
+    meta[lang] = {};
+    for (const challenge of challenges) {
+      const dashedBlockName = dasherize(challenge.block);
+      if (!meta[dashedBlockName]) {
+        meta[lang][dashedBlockName] = (await getMetaForBlock(
+          dashedBlockName
+        )).challengeOrder;
+      }
+    }
+  }
+  return {
+    meta,
+    challengesForLang
+  };
+}
+
+// cleanup calls some async functions, but it's the last thing that happens, so
+// no need to await anything.
+function cleanup() {
   if (browser) {
-        await browser.close();
+    browser.close();
   }
   liveServer.shutdown();
   spinner.stop();
+}
+
+function runTests({ challengesForLang, meta }) {
+  process.on('unhandledRejection', err => {
+    throw new Error(`unhandledRejection: ${err.name}, ${err.message}`);
   });
 
-    challenges.forEach(populateTestsForLang);
+  describe('Check challenges', function() {
+    after(function() {
+      cleanup();
     });
-
+    for (const challenge of challengesForLang) {
+      populateTestsForLang(challenge, meta);
+    }
+  });
+  spinner.text = 'Testing';
   run();
 }
 
@@ -165,16 +246,41 @@ function validateBlock(challenge) {
   }
 }
 
-function populateTestsForLang({ lang, challenges }) {
+function populateTestsForLang({ lang, challenges }, meta) {
   const mongoIds = new MongoIds();
   const challengeTitles = new ChallengeTitles();
   const validateChallenge = challengeSchemaValidator(lang);
 
   describe(`Check challenges (${lang})`, function() {
     this.timeout(5000);
-
     challenges.forEach(challenge => {
+      const dashedBlockName = dasherize(challenge.block);
+      describe(challenge.block || 'No block', function() {
         describe(challenge.title || 'No title', function() {
+          it('Matches a title in meta.json', function() {
+            const index = meta[lang][dashedBlockName].findIndex(
+              arr => arr[1] === challenge.title
+            );
+
+            if (index < 0) {
+              throw new AssertionError(
+                `Cannot find title "${challenge.title}" in meta.json file`
+              );
+            }
+          });
+
+          it('Matches an ID in meta.json', function() {
+            const index = meta[lang][dashedBlockName].findIndex(
+              arr => arr[0] === challenge.id
+            );
+
+            if (index < 0) {
+              throw new AssertionError(
+                `Cannot find ID "${challenge.id}" in meta.json file`
+              );
+            }
+          });
+
           it('Common checks', function() {
             const result = validateChallenge(challenge);
             const invalidBlock = validateBlock(challenge);
@@ -295,6 +401,7 @@ function populateTestsForLang({ lang, challenges }) {
         });
       });
     });
+  });
 }
 
 async function createTestRunnerForDOMChallenge(

docs/how-to-setup-freecodecamp-locally.md

@@ -308,13 +308,15 @@ A quick reference to the commands that you will need when working locally.
 **Local Build:**
 
 | command                                                        | description                                                                         |
-| ------------------------- | ----------------------------------------------------------------------------------- |
+| -------------------------------------------------------------- | ----------------------------------------------------------------------------------- |
 | `npm ci`                                                       | Installs / re-install all dependencies and bootstraps the different services.       |
 | `npm run seed`                                                 | Parses all the challenge markdown files and inserts them into MongoDB.              |
 | `npm run develop`                                              | Starts the freeCodeCamp API Server and Client Applications.                         |
 | `npm test`                                                     | Run all JS tests in the system, including client, server, lint and challenge tests. |
 | `npm run test:client`                                          | Run the client test suite.                                                          |
 | `npm run test:curriculum`                                      | Run the curriculum test suite.                                                      |
+| `npm run test:curriculum --block='Basic HTML and HTML5'`       | Test a specific Block.                                                              |
+| `npm run test:curriculum --superblock='responsive-web-design'` | Test a specific SuperBlock.                                                         |
 | `npm run test:server`                                          | Run the server test suite.                                                          |
 | `npm run clean`                                                | Uninstalls all dependencies and cleans up caches.                                   |