fix: add more tests for curriclum testing (#38464)

Co-Authored-By: Oliver Eyton-Williams <ojeytonwilliams@gmail.com>
Co-authored-by: Oliver Eyton-Williams <ojeytonwilliams@gmail.com>
This commit is contained in:
Tom
2020-04-23 12:01:15 -05:00
committed by GitHub
parent 35898e27f8
commit 560aacd4eb
6 changed files with 294 additions and 189 deletions

View File

@ -86,7 +86,7 @@
], ],
[ [
"587d7790367417b2b2512aaf", "587d7790367417b2b2512aaf",
"Make Links Navigatable with HTML Access Keys" "Make Links Navigable with HTML Access Keys"
], ],
[ [
"587d7790367417b2b2512ab0", "587d7790367417b2b2512ab0",

View File

@ -2,6 +2,7 @@ const path = require('path');
const { findIndex } = require('lodash'); const { findIndex } = require('lodash');
const readDirP = require('readdirp-walk'); const readDirP = require('readdirp-walk');
const { parseMarkdown } = require('@freecodecamp/challenge-md-parser'); const { parseMarkdown } = require('@freecodecamp/challenge-md-parser');
const fs = require('fs');
const { dasherize } = require('../utils/slugs'); const { dasherize } = require('../utils/slugs');
@ -14,7 +15,14 @@ function getChallengesDirForLang(lang) {
return path.resolve(challengesDir, `./${lang}`); return path.resolve(challengesDir, `./${lang}`);
} }
function getMetaForBlock(block) {
return JSON.parse(
fs.readFileSync(path.resolve(metaDir, `./${block}/meta.json`), 'utf8')
);
}
exports.getChallengesDirForLang = getChallengesDirForLang; exports.getChallengesDirForLang = getChallengesDirForLang;
exports.getMetaForBlock = getMetaForBlock;
exports.getChallengesForLang = function getChallengesForLang(lang) { exports.getChallengesForLang = function getChallengesForLang(lang) {
let curriculum = {}; let curriculum = {};

View File

@ -5575,8 +5575,7 @@
"ansi-regex": { "ansi-regex": {
"version": "2.1.1", "version": "2.1.1",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"aproba": { "aproba": {
"version": "1.2.0", "version": "1.2.0",
@ -5597,14 +5596,12 @@
"balanced-match": { "balanced-match": {
"version": "1.0.0", "version": "1.0.0",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"brace-expansion": { "brace-expansion": {
"version": "1.1.11", "version": "1.1.11",
"bundled": true, "bundled": true,
"dev": true, "dev": true,
"optional": true,
"requires": { "requires": {
"balanced-match": "^1.0.0", "balanced-match": "^1.0.0",
"concat-map": "0.0.1" "concat-map": "0.0.1"
@ -5619,20 +5616,17 @@
"code-point-at": { "code-point-at": {
"version": "1.1.0", "version": "1.1.0",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"concat-map": { "concat-map": {
"version": "0.0.1", "version": "0.0.1",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"console-control-strings": { "console-control-strings": {
"version": "1.1.0", "version": "1.1.0",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"core-util-is": { "core-util-is": {
"version": "1.0.2", "version": "1.0.2",
@ -5749,8 +5743,7 @@
"inherits": { "inherits": {
"version": "2.0.4", "version": "2.0.4",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"ini": { "ini": {
"version": "1.3.5", "version": "1.3.5",
@ -5762,7 +5755,6 @@
"version": "1.0.0", "version": "1.0.0",
"bundled": true, "bundled": true,
"dev": true, "dev": true,
"optional": true,
"requires": { "requires": {
"number-is-nan": "^1.0.0" "number-is-nan": "^1.0.0"
} }
@ -5777,7 +5769,6 @@
"version": "3.0.4", "version": "3.0.4",
"bundled": true, "bundled": true,
"dev": true, "dev": true,
"optional": true,
"requires": { "requires": {
"brace-expansion": "^1.1.7" "brace-expansion": "^1.1.7"
} }
@ -5785,14 +5776,12 @@
"minimist": { "minimist": {
"version": "0.0.8", "version": "0.0.8",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"minipass": { "minipass": {
"version": "2.9.0", "version": "2.9.0",
"bundled": true, "bundled": true,
"dev": true, "dev": true,
"optional": true,
"requires": { "requires": {
"safe-buffer": "^5.1.2", "safe-buffer": "^5.1.2",
"yallist": "^3.0.0" "yallist": "^3.0.0"
@ -5811,7 +5800,6 @@
"version": "0.5.1", "version": "0.5.1",
"bundled": true, "bundled": true,
"dev": true, "dev": true,
"optional": true,
"requires": { "requires": {
"minimist": "0.0.8" "minimist": "0.0.8"
} }
@ -5901,8 +5889,7 @@
"number-is-nan": { "number-is-nan": {
"version": "1.0.1", "version": "1.0.1",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"object-assign": { "object-assign": {
"version": "4.1.1", "version": "4.1.1",
@ -5914,7 +5901,6 @@
"version": "1.4.0", "version": "1.4.0",
"bundled": true, "bundled": true,
"dev": true, "dev": true,
"optional": true,
"requires": { "requires": {
"wrappy": "1" "wrappy": "1"
} }
@ -6000,8 +5986,7 @@
"safe-buffer": { "safe-buffer": {
"version": "5.1.2", "version": "5.1.2",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"safer-buffer": { "safer-buffer": {
"version": "2.1.2", "version": "2.1.2",
@ -6037,7 +6022,6 @@
"version": "1.0.2", "version": "1.0.2",
"bundled": true, "bundled": true,
"dev": true, "dev": true,
"optional": true,
"requires": { "requires": {
"code-point-at": "^1.0.0", "code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0", "is-fullwidth-code-point": "^1.0.0",
@ -6057,7 +6041,6 @@
"version": "3.0.1", "version": "3.0.1",
"bundled": true, "bundled": true,
"dev": true, "dev": true,
"optional": true,
"requires": { "requires": {
"ansi-regex": "^2.0.0" "ansi-regex": "^2.0.0"
} }
@ -6101,14 +6084,12 @@
"wrappy": { "wrappy": {
"version": "1.0.2", "version": "1.0.2",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"yallist": { "yallist": {
"version": "3.1.1", "version": "3.1.1",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
} }
} }
}, },
@ -14674,6 +14655,12 @@
"integrity": "sha1-2sMECGkMIfPDYwo/86BYd73L1zY=", "integrity": "sha1-2sMECGkMIfPDYwo/86BYd73L1zY=",
"dev": true "dev": true
}, },
"string-similarity": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/string-similarity/-/string-similarity-4.0.1.tgz",
"integrity": "sha512-v36MJzloekKVvKAsYi6O/qpn2mIuvwEFIT9Gx3yg4spkNjXYsk7yxc37g4ZTyMVIBvt/9PZGxnqEtme8XHK+Mw==",
"dev": true
},
"string-width": { "string-width": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",

View File

@ -60,6 +60,7 @@
"readdirp-walk": "^1.7.0", "readdirp-walk": "^1.7.0",
"rx": "^4.1.0", "rx": "^4.1.0",
"semantic-release": "^15.13.24", "semantic-release": "^15.13.24",
"string-similarity": "^4.0.1",
"validator": "^10.4.0" "validator": "^10.4.0"
}, },
"keywords": [ "keywords": [

View File

@ -1,5 +1,7 @@
/* eslint-disable no-loop-func */
const path = require('path'); const path = require('path');
const liveServer = require('live-server'); const liveServer = require('live-server');
const stringSimilarity = require('string-similarity');
const spinner = require('ora')(); const spinner = require('ora')();
@ -32,7 +34,7 @@ const vm = require('vm');
const puppeteer = require('puppeteer'); const puppeteer = require('puppeteer');
const { getChallengesForLang } = require('../getChallenges'); const { getChallengesForLang, getMetaForBlock } = require('../getChallenges');
const MongoIds = require('./utils/mongoIds'); const MongoIds = require('./utils/mongoIds');
const ChallengeTitles = require('./utils/challengeTitles'); const ChallengeTitles = require('./utils/challengeTitles');
@ -86,24 +88,22 @@ spinner.text = 'Populate tests.';
let browser; let browser;
let page; let page;
runTests(); setup()
.then(runTests)
async function runTests() { .catch(err => {
process.on('unhandledRejection', err => { cleanup();
spinner.stop(); // setting the error code because node does not (yet) exit with a non-zero
throw new Error(`unhandledRejection: ${err.name}, ${err.message}`); // code on unhandled exceptions.
process.exitCode = 1;
throw err;
}); });
const testLangs = testedLangs(); async function setup() {
if (process.env.npm_config_superblock && process.env.npm_config_block) {
throw new Error(`Please do not use both a block and superblock as input.`);
}
const challenges = await Promise.all( // liveServer starts synchronously
testLangs.map(lang => getChallenges(lang))
);
describe('Check challenges', function() {
before(async function() {
spinner.text = 'Testing';
this.timeout(50000);
liveServer.start({ liveServer.start({
host: '127.0.0.1', host: '127.0.0.1',
port: '8080', port: '8080',
@ -126,19 +126,100 @@ async function runTests() {
global.Worker = createPseudoWorker(await newPageContext(browser)); global.Worker = createPseudoWorker(await newPageContext(browser));
page = await newPageContext(browser); page = await newPageContext(browser);
await page.setViewport({ width: 300, height: 150 }); await page.setViewport({ width: 300, height: 150 });
}); const testLangs = testedLangs();
after(async function() { if (testLangs.length > 1)
this.timeout(30000); throw Error(
`Testing more than one language at once is not currently supported
please change the TEST_CHALLENGES_FOR_LANGS env variable to a single language`
);
const challengesForLang = await Promise.all(
testLangs.map(lang => getChallenges(lang))
);
// the next few statements create a list of all blocks and superblocks
// as they appear in the list of challenges
const blocks = challengesForLang[0].challenges.map(({ block }) => block);
const superBlocks = challengesForLang[0].challenges.map(
({ superBlock }) => superBlock
);
const targetBlockStrings = [...new Set(blocks)];
const targetSuperBlockStrings = [...new Set(superBlocks)];
// the next few statements will filter challenges based on command variables
if (process.env.npm_config_superblock) {
const filter = stringSimilarity.findBestMatch(
process.env.npm_config_superblock,
targetSuperBlockStrings
).bestMatch.target;
console.log(`\nsuperBlock being tested: ${filter}`);
challengesForLang[0].challenges = challengesForLang[0].challenges.filter(
challenge => challenge.superBlock === filter
);
if (!challengesForLang[0].challenges.length) {
throw new Error(`No challenges found with superBlock "${filter}"`);
}
}
if (process.env.npm_config_block) {
const filter = stringSimilarity.findBestMatch(
process.env.npm_config_block,
targetBlockStrings
).bestMatch.target;
console.log(`\nblock being tested: ${filter}`);
challengesForLang[0].challenges = challengesForLang[0].challenges.filter(
challenge => challenge.block === filter
);
if (!challengesForLang[0].challenges.length) {
throw new Error(`No challenges found with block "${filter}"`);
}
}
const meta = {};
for (const { lang, challenges } of challengesForLang) {
meta[lang] = {};
for (const challenge of challenges) {
const dashedBlockName = dasherize(challenge.block);
if (!meta[dashedBlockName]) {
meta[lang][dashedBlockName] = (await getMetaForBlock(
dashedBlockName
)).challengeOrder;
}
}
}
return {
meta,
challengesForLang
};
}
// cleanup calls some async functions, but it's the last thing that happens, so
// no need to await anything.
function cleanup() {
if (browser) { if (browser) {
await browser.close(); browser.close();
} }
liveServer.shutdown(); liveServer.shutdown();
spinner.stop(); spinner.stop();
}
function runTests({ challengesForLang, meta }) {
process.on('unhandledRejection', err => {
throw new Error(`unhandledRejection: ${err.name}, ${err.message}`);
}); });
challenges.forEach(populateTestsForLang); describe('Check challenges', function() {
after(function() {
cleanup();
}); });
for (const challenge of challengesForLang) {
populateTestsForLang(challenge, meta);
}
});
spinner.text = 'Testing';
run(); run();
} }
@ -165,16 +246,41 @@ function validateBlock(challenge) {
} }
} }
function populateTestsForLang({ lang, challenges }) { function populateTestsForLang({ lang, challenges }, meta) {
const mongoIds = new MongoIds(); const mongoIds = new MongoIds();
const challengeTitles = new ChallengeTitles(); const challengeTitles = new ChallengeTitles();
const validateChallenge = challengeSchemaValidator(lang); const validateChallenge = challengeSchemaValidator(lang);
describe(`Check challenges (${lang})`, function() { describe(`Check challenges (${lang})`, function() {
this.timeout(5000); this.timeout(5000);
challenges.forEach(challenge => { challenges.forEach(challenge => {
const dashedBlockName = dasherize(challenge.block);
describe(challenge.block || 'No block', function() {
describe(challenge.title || 'No title', function() { describe(challenge.title || 'No title', function() {
it('Matches a title in meta.json', function() {
const index = meta[lang][dashedBlockName].findIndex(
arr => arr[1] === challenge.title
);
if (index < 0) {
throw new AssertionError(
`Cannot find title "${challenge.title}" in meta.json file`
);
}
});
it('Matches an ID in meta.json', function() {
const index = meta[lang][dashedBlockName].findIndex(
arr => arr[0] === challenge.id
);
if (index < 0) {
throw new AssertionError(
`Cannot find ID "${challenge.id}" in meta.json file`
);
}
});
it('Common checks', function() { it('Common checks', function() {
const result = validateChallenge(challenge); const result = validateChallenge(challenge);
const invalidBlock = validateBlock(challenge); const invalidBlock = validateBlock(challenge);
@ -295,6 +401,7 @@ function populateTestsForLang({ lang, challenges }) {
}); });
}); });
}); });
});
} }
async function createTestRunnerForDOMChallenge( async function createTestRunnerForDOMChallenge(

View File

@ -308,13 +308,15 @@ A quick reference to the commands that you will need when working locally.
**Local Build:** **Local Build:**
| command | description | | command | description |
| ------------------------- | ----------------------------------------------------------------------------------- | | -------------------------------------------------------------- | ----------------------------------------------------------------------------------- |
| `npm ci` | Installs / re-install all dependencies and bootstraps the different services. | | `npm ci` | Installs / re-install all dependencies and bootstraps the different services. |
| `npm run seed` | Parses all the challenge markdown files and inserts them into MongoDB. | | `npm run seed` | Parses all the challenge markdown files and inserts them into MongoDB. |
| `npm run develop` | Starts the freeCodeCamp API Server and Client Applications. | | `npm run develop` | Starts the freeCodeCamp API Server and Client Applications. |
| `npm test` | Run all JS tests in the system, including client, server, lint and challenge tests. | | `npm test` | Run all JS tests in the system, including client, server, lint and challenge tests. |
| `npm run test:client` | Run the client test suite. | | `npm run test:client` | Run the client test suite. |
| `npm run test:curriculum` | Run the curriculum test suite. | | `npm run test:curriculum` | Run the curriculum test suite. |
| `npm run test:curriculum --block='Basic HTML and HTML5'` | Test a specific Block. |
| `npm run test:curriculum --superblock='responsive-web-design'` | Test a specific SuperBlock. |
| `npm run test:server` | Run the server test suite. | | `npm run test:server` | Run the server test suite. |
| `npm run clean` | Uninstalls all dependencies and cleans up caches. | | `npm run clean` | Uninstalls all dependencies and cleans up caches. |