feat(tests): use client build pipeline and test runners for testing curriculum

This commit is contained in:
Valeriy S 2019-01-15 17:18:56 +03:00 committed by Stuart Taylor
parent 08cfd986c4
commit e063686fca
7 changed files with 3835 additions and 1554 deletions

View File

@ -1,4 +1,3 @@
import { throwers } from '../rechallenge/throwers';
import { transformers } from '../rechallenge/transformers'; import { transformers } from '../rechallenge/transformers';
import { cssToHtml, jsToHtml, concatHtml } from '../rechallenge/builders.js'; import { cssToHtml, jsToHtml, concatHtml } from '../rechallenge/builders.js';
@ -27,9 +26,8 @@ const applyFunction = fn =>
return newFile; return newFile;
} }
return file; return file;
} catch { } catch (error) {
// return { error }; return { ...file, error };
return file;
} }
}; };
@ -55,7 +53,7 @@ export function buildDOMChallenge(files, meta = {}) {
const { required = [], template = '' } = meta; const { required = [], template = '' } = meta;
const finalRequires = [...globalRequires, ...required, ...frameRunner]; const finalRequires = [...globalRequires, ...required, ...frameRunner];
const toHtml = [jsToHtml, cssToHtml]; const toHtml = [jsToHtml, cssToHtml];
const pipeLine = composeFunctions(...throwers, ...transformers, ...toHtml); const pipeLine = composeFunctions(...transformers, ...toHtml);
const finalFiles = Object.keys(files) const finalFiles = Object.keys(files)
.map(key => files[key]) .map(key => files[key])
.map(pipeLine); .map(pipeLine);
@ -68,7 +66,7 @@ export function buildDOMChallenge(files, meta = {}) {
} }
export function buildJSChallenge(files) { export function buildJSChallenge(files) {
const pipeLine = composeFunctions(...throwers, ...transformers); const pipeLine = composeFunctions(...transformers);
const finalFiles = Object.keys(files) const finalFiles = Object.keys(files)
.map(key => files[key]) .map(key => files[key])
.map(pipeLine); .map(pipeLine);

View File

@ -1,10 +1,9 @@
import { homeLocation } from '../../../../config/env.json';
class WorkerExecutor { class WorkerExecutor {
constructor(workerName) { constructor(workerName, location) {
this.workerName = workerName; this.workerName = workerName;
this.worker = null; this.worker = null;
this.observers = {}; this.observers = {};
this.location = location;
this.execute = this.execute.bind(this); this.execute = this.execute.bind(this);
this.killWorker = this.killWorker.bind(this); this.killWorker = this.killWorker.bind(this);
@ -13,7 +12,7 @@ class WorkerExecutor {
getWorker() { getWorker() {
if (this.worker === null) { if (this.worker === null) {
this.worker = new Worker(`${homeLocation}/js/${this.workerName}.js`); this.worker = new Worker(`${this.location}${this.workerName}.js`);
} }
return this.worker; return this.worker;
@ -72,6 +71,6 @@ class WorkerExecutor {
} }
} }
export default function createWorkerExecutor(workerName) { export default function createWorkerExecutor(workerName, location = '/js/') {
return new WorkerExecutor(workerName); return new WorkerExecutor(workerName, location);
} }

File diff suppressed because it is too large Load Diff

View File

@ -19,6 +19,7 @@
"lint": "eslint ./**/*.js --fix", "lint": "eslint ./**/*.js --fix",
"repack": "babel-node ./repack.js", "repack": "babel-node ./repack.js",
"semantic-release": "semantic-release", "semantic-release": "semantic-release",
"pretest": "cd ../client && npm run build:frame-runner",
"test": "mocha --delay --reporter progress --bail", "test": "mocha --delay --reporter progress --bail",
"unpack": "babel-node ./unpack.js" "unpack": "babel-node ./unpack.js"
}, },
@ -26,6 +27,10 @@
"invariant": "^2.2.4" "invariant": "^2.2.4"
}, },
"devDependencies": { "devDependencies": {
"@babel/core": "^7.2.2",
"@babel/register": "^7.0.0",
"@babel/polyfill": "^7.2.5",
"@babel/preset-env": "^7.2.3",
"@commitlint/cli": "^7.0.0", "@commitlint/cli": "^7.0.0",
"@commitlint/config-conventional": "^7.0.1", "@commitlint/config-conventional": "^7.0.1",
"@commitlint/travis-cli": "^7.0.0", "@commitlint/travis-cli": "^7.0.0",
@ -44,8 +49,6 @@
"babel-standalone": "^6.26.0", "babel-standalone": "^6.26.0",
"browserify": "^16.2.2", "browserify": "^16.2.2",
"chai": "4.2.0", "chai": "4.2.0",
"enzyme": "^3.7.0",
"enzyme-adapter-react-16": "^1.6.0",
"eslint": "^4.19.1", "eslint": "^4.19.1",
"eslint-config-freecodecamp": "^1.1.1", "eslint-config-freecodecamp": "^1.1.1",
"eslint-plugin-import": "^2.11.0", "eslint-plugin-import": "^2.11.0",
@ -59,18 +62,13 @@
"js-yaml": "^3.12.0", "js-yaml": "^3.12.0",
"jsdom": "^12.2.0", "jsdom": "^12.2.0",
"lint-staged": "^7.2.0", "lint-staged": "^7.2.0",
"live-server": "^1.2.1",
"lodash": "^4.17.10", "lodash": "^4.17.10",
"mocha": "5.2.0", "mocha": "5.2.0",
"node-sass": "4.9.4",
"prettier": "^1.13.5", "prettier": "^1.13.5",
"prettier-package-json": "^1.6.0", "prettier-package-json": "^1.6.0",
"puppeteer": "1.11.0", "puppeteer": "1.11.0",
"react": "^16.5.2",
"react-dom": "^16.5.2",
"react-redux": "^5.0.7",
"readdirp-walk": "^1.6.0", "readdirp-walk": "^1.6.0",
"redux": "^4.0.1",
"redux-thunk": "^2.3.0",
"rx": "^4.1.0", "rx": "^4.1.0",
"semantic-release": "^15.6.0", "semantic-release": "^15.6.0",
"validator": "^10.4.0" "validator": "^10.4.0"

View File

View File

@ -1,23 +1,38 @@
/* global browser, page */ const path = require('path');
const liveServer = require('live-server');
const spinner = require('ora')();
const clientPath = path.resolve(__dirname, '../../client');
require('@babel/polyfill');
require('@babel/register')({
root: clientPath,
babelrc: false,
presets: ['@babel/preset-env'],
ignore: [/node_modules/],
only: [clientPath]
});
const createPseudoWorker = require('./utils/pseudo-worker');
const {
default: createWorker
} = require('../../client/src/templates/Challenges/utils/worker-executor');
const { assert, AssertionError } = require('chai'); const { assert, AssertionError } = require('chai');
const Mocha = require('mocha'); const Mocha = require('mocha');
const { flatten } = require('lodash'); const { flatten } = require('lodash');
const path = require('path');
const fs = require('fs'); const jsdom = require('jsdom');
const dom = new jsdom.JSDOM('');
global.document = dom.window.document;
require('dotenv').config({ path: path.resolve(__dirname, '../../.env') }); require('dotenv').config({ path: path.resolve(__dirname, '../../.env') });
const vm = require('vm'); const vm = require('vm');
const puppeteer = require('puppeteer'); const puppeteer = require('puppeteer');
const jsdom = require('jsdom');
const jQuery = require('jquery');
const Sass = require('node-sass');
const Babel = require('babel-standalone');
const presetEnv = require('babel-preset-env');
const presetReact = require('babel-preset-react');
const { getChallengesForLang } = require('../getChallenges'); const { getChallengesForLang } = require('../getChallenges');
const MongoIds = require('./utils/mongoIds'); const MongoIds = require('./utils/mongoIds');
@ -27,6 +42,15 @@ const { challengeTypes } = require('../../client/utils/challengeTypes');
const { supportedLangs } = require('../utils'); const { supportedLangs } = require('../utils');
const {
buildDOMChallenge,
buildJSChallenge
} = require('../../client/src/templates/Challenges/utils/build');
const {
createPoly
} = require('../../client/src/templates/Challenges/utils/polyvinyl');
const oldRunnerFail = Mocha.Runner.prototype.fail; const oldRunnerFail = Mocha.Runner.prototype.fail;
Mocha.Runner.prototype.fail = function(test, err) { Mocha.Runner.prototype.fail = function(test, err) {
if (err instanceof AssertionError) { if (err instanceof AssertionError) {
@ -43,17 +67,18 @@ Mocha.Runner.prototype.fail = function(test, err) {
return oldRunnerFail.call(this, test, err); return oldRunnerFail.call(this, test, err);
}; };
const { JSDOM } = jsdom; async function newPageContext(browser) {
const page = await browser.newPage();
// it's needed for workers as context.
await page.goto('http://127.0.0.1:8080/index.html');
return page;
}
const babelOptions = { spinner.start();
plugins: ['transform-runtime'], spinner.text = 'Populate tests.';
presets: [presetEnv, presetReact]
};
const jQueryScript = fs.readFileSync( let browser;
path.resolve('./node_modules/jquery/dist/jquery.slim.min.js'), let page;
'utf8'
);
runTests(); runTests();
@ -66,13 +91,47 @@ async function runTests() {
testLangs = testLangs.filter(lang => filterLangs.includes(lang)); testLangs = testLangs.filter(lang => filterLangs.includes(lang));
} }
await Promise.all(testLangs.map(lang => populateTestsForLang(lang))); const challenges = await Promise.all(
testLangs.map(lang => getChallenges(lang))
);
describe('Check challenges', function() {
before(async function() {
spinner.text = 'Testing';
this.timeout(50000);
liveServer.start({
host: '127.0.0.1',
port: '8080',
root: path.resolve(__dirname, 'stubs'),
mount: [['/js', path.join(clientPath, 'static/js')]],
open: false,
logLevel: 0
});
browser = await puppeteer.launch({
args: ['--no-sandbox']
// dumpio: true
});
global.Worker = createPseudoWorker(await newPageContext(browser));
page = await newPageContext(browser);
await page.setViewport({ width: 300, height: 150 });
});
after(async function() {
this.timeout(30000);
if (browser) {
await browser.close();
}
liveServer.shutdown();
spinner.stop();
});
challenges.forEach(populateTestsForLang);
});
run(); run();
} }
async function populateTestsForLang(lang) { async function getChallenges(lang) {
const allChallenges = await getChallengesForLang(lang).then(curriculum => const challenges = await getChallengesForLang(lang).then(curriculum =>
Object.keys(curriculum) Object.keys(curriculum)
.map(key => curriculum[key].blocks) .map(key => curriculum[key].blocks)
.reduce((challengeArray, superBlock) => { .reduce((challengeArray, superBlock) => {
@ -82,28 +141,19 @@ async function populateTestsForLang(lang) {
return [...challengeArray, ...flatten(challengesForBlock)]; return [...challengeArray, ...flatten(challengesForBlock)];
}, []) }, [])
); );
return { lang, challenges };
}
function populateTestsForLang({ lang, challenges }) {
const mongoIds = new MongoIds(); const mongoIds = new MongoIds();
const challengeTitles = new ChallengeTitles(); const challengeTitles = new ChallengeTitles();
const validateChallenge = challengeSchemaValidator(lang); const validateChallenge = challengeSchemaValidator(lang);
describe(`Check challenges (${lang})`, async function() { describe(`Check challenges (${lang})`, function() {
before(async function() {
this.timeout(30000);
global.browser = await puppeteer.launch({ args: ['--no-sandbox'] });
global.page = await browser.newPage();
await page.setViewport({ width: 300, height: 150 });
});
after(async function() {
if (global.browser) {
await browser.close();
}
});
this.timeout(5000); this.timeout(5000);
allChallenges.forEach(challenge => { challenges.forEach(challenge => {
describe(challenge.title || 'No title', async function() { describe(challenge.title || 'No title', function() {
it('Common checks', function() { it('Common checks', function() {
const result = validateChallenge(challenge); const result = validateChallenge(challenge);
if (result.error) { if (result.error) {
@ -140,38 +190,26 @@ async function populateTestsForLang(lang) {
}); });
}); });
const { files = [], required = [] } = challenge; let { files = [] } = challenge;
const exts = Array.from(new Set(files.map(({ ext }) => ext)));
const groupedFiles = exts.reduce((result, ext) => {
const file = files.filter(file => file.ext === ext).reduce(
(result, file) => ({
head: result.head + '\n' + file.head,
contents: result.contents + '\n' + file.contents,
tail: result.tail + '\n' + file.tail
}),
{ head: '', contents: '', tail: '' }
);
return {
...result,
[ext]: file
};
}, {});
let evaluateTest; let evaluateTest;
if ( if (challengeType === challengeTypes.backend) {
challengeType === challengeTypes.modern && it('Check tests is not implemented.');
(groupedFiles.js || groupedFiles.jsx) return;
} else if (
challengeType === challengeTypes.js ||
challengeType === challengeTypes.bonfire
) { ) {
evaluateTest = evaluateReactReduxTest;
} else if (groupedFiles.html) {
evaluateTest = evaluateHtmlTest;
} else if (groupedFiles.js) {
evaluateTest = evaluateJsTest; evaluateTest = evaluateJsTest;
} else if (files.length === 1) {
evaluateTest = evaluateHtmlTest;
} else { } else {
it('Check tests. Unknown file type.'); it('Check tests.', () => {
throw new Error('Seed file should be only the one.');
});
return; return;
} }
files = files.map(createPoly);
it('Test suite must fail on the initial contents', async function() { it('Test suite must fail on the initial contents', async function() {
this.timeout(20000); this.timeout(20000);
// suppress errors in the console. // suppress errors in the console.
@ -180,12 +218,14 @@ async function populateTestsForLang(lang) {
let fails = (await Promise.all( let fails = (await Promise.all(
tests.map(async function(test) { tests.map(async function(test) {
try { try {
await evaluateTest({ await evaluateTest(
challengeType, {
required, ...challenge,
files: groupedFiles, files,
test test
}); },
page
);
return false; return false;
} catch (e) { } catch (e) {
return true; return true;
@ -207,18 +247,20 @@ async function populateTestsForLang(lang) {
return; return;
} }
describe('Check tests against solutions', async function() { describe('Check tests against solutions', function() {
solutions.forEach((solution, index) => { solutions.forEach((solution, index) => {
describe(`Solution ${index + 1}`, async function() { describe(`Solution ${index + 1}`, function() {
tests.forEach(test => { tests.forEach(test => {
it(test.text, async function() { it(test.text, async function() {
await evaluateTest({ await evaluateTest(
challengeType, {
solution, ...challenge,
required, files,
files: groupedFiles, solution,
test test
}); },
page
);
}); });
}); });
}); });
@ -229,314 +271,68 @@ async function populateTestsForLang(lang) {
}); });
} }
// Fake Deep Equal dependency async function evaluateHtmlTest(
const DeepEqual = (a, b) => JSON.stringify(a) === JSON.stringify(b); { solution, required = [], template, files, test },
context
// Hardcode Deep Freeze dependency ) {
const DeepFreeze = o => { if (solution) {
Object.freeze(o); files[0].contents = solution;
Object.getOwnPropertyNames(o).forEach(function(prop) {
if (
o.hasOwnProperty(prop) &&
o[prop] !== null &&
(typeof o[prop] === 'object' || typeof o[prop] === 'function') &&
!Object.isFrozen(o[prop])
) {
DeepFreeze(o[prop]);
}
});
return o;
};
function transformSass(solution) {
const fragment = JSDOM.fragment(`<div>${solution}</div>`);
const styleTags = fragment.querySelectorAll('style[type="text/sass"]');
if (styleTags.length > 0) {
styleTags.forEach(styleTag => {
styleTag.innerHTML = Sass.renderSync({ data: styleTag.innerHTML }).css;
styleTag.type = 'text/css';
});
return fragment.children[0].innerHTML;
} }
return solution;
}
async function evaluateHtmlTest({ solution, required, files, test }) { const loadEnzyme = files[0].ext === 'jsx';
const { head = '', contents = '', tail = '' } = files.html;
if (!solution) {
solution = contents;
}
const code = solution;
const links = required const { build, sources } = await buildDOMChallenge(files, {
.map(({ link, src }) => { required,
if (link && src) { template
throw new Error(`
A required file can not have both a src and a link: src = ${src}, link = ${link}
`);
}
if (src) {
return `<script src='${src}' type='text/javascript'></script>`;
}
if (link) {
return `<link href='${link}' rel='stylesheet' />`;
}
return '';
})
.reduce((head, required) => head.concat(required), '');
const scripts = `
<head>
${links}
</head>
`;
const sandbox = { solution, transformSass };
const context = vm.createContext(sandbox);
vm.runInContext('solution = transformSass(solution);', context, {
timeout: 2000
}); });
solution = sandbox.solution;
await preparePageToTest(); await context.reload();
await context.setContent(build);
await page.setContent( const result = await context.evaluate(
` async(sources, testString, loadEnzyme) => {
<!doctype html> document.__source = sources && 'index' in sources ? sources['index'] : '';
<html> document.__getUserInput = fileName => sources[fileName];
${scripts} document.__frameReady = () => {};
${head} document.__loadEnzyme = loadEnzyme;
${solution} await document.__initTestFrame();
${tail}
</html>
`
);
await runTestInBrowser(code, test.testString); const { pass, err } = await document.__runTest(testString);
} if (pass) {
return true;
async function preparePageToTest() { } else {
await page.reload(); return { ...err };
await page.addScriptTag({
url: 'https://cdnjs.cloudflare.com/ajax/libs/chai/4.2.0/chai.min.js'
});
await page.evaluate(() => {
window.assert = window.chai.assert;
});
await page.evaluate(jQueryScript);
}
async function runTestInBrowser(code, testString) {
const result = await page.evaluate(
async function(code, testString) {
/* eslint-disable no-unused-vars */
// Fake Deep Equal dependency
const DeepEqual = (a, b) => JSON.stringify(a) === JSON.stringify(b);
// Hardcode Deep Freeze dependency
const DeepFreeze = o => {
Object.freeze(o);
Object.getOwnPropertyNames(o).forEach(function(prop) {
if (
o.hasOwnProperty(prop) &&
o[prop] !== null &&
(typeof o[prop] === 'object' || typeof o[prop] === 'function') &&
!Object.isFrozen(o[prop])
) {
DeepFreeze(o[prop]);
}
});
return o;
};
const editor = {
getValue() {
return code;
}
};
/* eslint-enable no-unused-vars */
try {
// eslint-disable-next-line no-eval
const test = eval(testString);
if (typeof test === 'function') {
await test(() => code);
}
} catch (e) {
return {
message: e.message,
isAssertion: e instanceof window.chai.AssertionError
};
} }
return true;
}, },
code, sources,
testString test.testString,
loadEnzyme
); );
if (result !== true) { if (result !== true) {
throw result.isAssertion throw AssertionError(result.message);
? new AssertionError(result.message)
: new Error(result.message);
} }
} }
async function evaluateJsTest({ solution, files, test }) { async function evaluateJsTest({ solution, files, test }) {
const virtualConsole = new jsdom.VirtualConsole(); if (solution) {
const dom = new JSDOM('', { runScripts: 'dangerously', virtualConsole }); files[0].contents = solution;
const { head = '', contents = '', tail = '' } = files.js;
let scriptString = '';
if (!solution) {
solution = contents;
try {
scriptString =
Babel.transform(head, babelOptions).code +
'\n' +
Babel.transform(contents, babelOptions).code +
'\n' +
Babel.transform(tail, babelOptions).code +
'\n';
} catch (e) {
scriptString = '';
}
} else {
scriptString =
Babel.transform(head, babelOptions).code +
'\n' +
Babel.transform(solution, babelOptions).code +
'\n' +
Babel.transform(tail, babelOptions).code +
'\n';
} }
dom.window.require = require; const { build, sources } = await buildJSChallenge(files);
dom.window.code = solution; const code = sources && 'index' in sources ? sources['index'] : '';
await runTestInJsdom(dom, test.testString, scriptString); const script = build + '\n' + test.testString;
}
async function evaluateReactReduxTest({ solution, files, test }) { const testWorker = createWorker('test-evaluator');
let head = '',
tail = '';
if (files.js) {
const { head: headJs = '', tail: tailJs = '' } = files.js;
head += headJs + '\n';
tail += tailJs + '\n';
}
if (files.jsx) {
const { head: headJsx = '', tail: tailJsx = '' } = files.jsx;
head += headJsx + '\n';
tail += tailJsx + '\n';
}
/* Transpile ALL the code try {
* (we may use JSX in head or tail or tests, too): */ const { pass, err } = await testWorker.execute(
{ script, code, sources },
let scriptString = ''; 5000
if (!solution) { );
const contents = if (!pass) {
(files.js ? files.js.contents || '' : '') + throw new AssertionError(err.message);
(files.jsx ? files.jsx.contents || '' : '');
solution = contents;
try {
scriptString =
Babel.transform(head, babelOptions).code +
'\n' +
Babel.transform(contents, babelOptions).code +
'\n' +
Babel.transform(tail, babelOptions).code +
'\n';
} catch (e) {
scriptString = '';
} }
} else { } finally {
scriptString = testWorker.killWorker();
Babel.transform(head, babelOptions).code +
'\n' +
Babel.transform(solution, babelOptions).code +
'\n' +
Babel.transform(tail, babelOptions).code +
'\n';
}
const code = solution;
const virtualConsole = new jsdom.VirtualConsole();
// Mock DOM document for ReactDOM.render method
const dom = new JSDOM(
`
<!doctype html>
<html>
<body>
<div id="root"><div id="challenge-node"></div>
</body>
</html>
`,
{
runScripts: 'dangerously',
virtualConsole,
url: 'http://localhost'
}
);
const { window } = dom;
const document = window.document;
global.window = window;
global.document = document;
global.navigator = {
userAgent: 'node.js'
};
global.requestAnimationFrame = callback => setTimeout(callback, 0);
global.cancelAnimationFrame = id => clearTimeout(id);
// Provide dependencies, just provide all of them
dom.window.React = require('react');
dom.window.ReactDOM = require('react-dom');
dom.window.PropTypes = require('prop-types');
dom.window.Redux = require('redux');
dom.window.ReduxThunk = require('redux-thunk');
dom.window.ReactRedux = require('react-redux');
dom.window.Enzyme = require('enzyme');
const Adapter16 = require('enzyme-adapter-react-16');
dom.window.Enzyme.configure({ adapter: new Adapter16() });
dom.window.require = require;
dom.window.code = code;
dom.window.editor = {
getValue() {
return code;
}
};
await runTestInJsdom(dom, test.testString, scriptString);
}
async function runTestInJsdom(dom, testString, scriptString = '') {
// jQuery used by tests
jQuery(dom.window);
dom.window.assert = assert;
dom.window.DeepEqual = DeepEqual;
dom.window.DeepFreeze = DeepFreeze;
dom.window.__test = testString;
scriptString += `;
window.__result =
(async () => {
try {
const testResult = eval(__test);
if (typeof testResult === 'function') {
await testResult(() => code);
}
}catch (e) {
window.__error = e;
}
})();`;
const script = new vm.Script(scriptString);
dom.runVMScript(script, { timeout: 5000 });
await dom.window.__result;
if (dom.window.__error) {
throw dom.window.__error;
} }
} }

View File

@ -0,0 +1,85 @@
function createPseudoWorker(context) {
class PseudoWorker {
constructor(path) {
this.terminated = false;
this.worker = context.evaluateHandle(path => new Worker(path), path);
this.listenToWorker('onmessage');
this.listenToWorker('onerror');
}
terminate() {
this.terminated = true;
this.worker.then(worker =>
context.evaluate(worker => worker.terminate(), worker)
);
}
listenToWorker(eventName) {
this.worker.then(async worker => {
const producer = await context.evaluateHandle(
(worker, eventName) => {
let callback;
const queue = [];
function send(event) {
if (!queue.length && callback) {
callback();
}
queue.push(event);
}
worker[eventName] = e => send(e);
const resolver = resolve => (callback = resolve);
async function* produce() {
while (true) {
while (queue.length) {
yield queue.shift();
}
await new Promise(resolver);
}
}
return produce();
},
worker,
eventName
);
while (!this.terminated) {
try {
const data = await context.evaluate(
producer =>
producer
.next()
.then(({ value: { data, message } }) => ({ data, message })),
producer
);
if (this[eventName]) {
this[eventName](data);
}
} catch (err) {
break;
}
}
});
}
async postMessage(msg) {
if (this.terminated) {
throw new Error('Worker is terminated.');
}
try {
await this.worker.then(worker =>
worker
.executionContext()
.evaluate((worker, msg) => worker.postMessage(msg), worker, msg)
);
} catch (e) {
if (this.onerror) {
this.onerror({ message: e.message });
}
}
}
}
return PseudoWorker;
}
module.exports = createPseudoWorker;