feat(tests): use client build pipeline and test runners for testing curriculum

This commit is contained in:
Valeriy S 2019-01-15 17:18:56 +03:00 committed by Stuart Taylor
parent 08cfd986c4
commit e063686fca
7 changed files with 3835 additions and 1554 deletions

View File

@ -1,4 +1,3 @@
import { throwers } from '../rechallenge/throwers';
import { transformers } from '../rechallenge/transformers';
import { cssToHtml, jsToHtml, concatHtml } from '../rechallenge/builders.js';
@ -27,9 +26,8 @@ const applyFunction = fn =>
return newFile;
}
return file;
} catch {
// return { error };
return file;
} catch (error) {
return { ...file, error };
}
};
@ -55,7 +53,7 @@ export function buildDOMChallenge(files, meta = {}) {
const { required = [], template = '' } = meta;
const finalRequires = [...globalRequires, ...required, ...frameRunner];
const toHtml = [jsToHtml, cssToHtml];
const pipeLine = composeFunctions(...throwers, ...transformers, ...toHtml);
const pipeLine = composeFunctions(...transformers, ...toHtml);
const finalFiles = Object.keys(files)
.map(key => files[key])
.map(pipeLine);
@ -68,7 +66,7 @@ export function buildDOMChallenge(files, meta = {}) {
}
export function buildJSChallenge(files) {
const pipeLine = composeFunctions(...throwers, ...transformers);
const pipeLine = composeFunctions(...transformers);
const finalFiles = Object.keys(files)
.map(key => files[key])
.map(pipeLine);

View File

@ -1,10 +1,9 @@
import { homeLocation } from '../../../../config/env.json';
class WorkerExecutor {
constructor(workerName) {
constructor(workerName, location) {
this.workerName = workerName;
this.worker = null;
this.observers = {};
this.location = location;
this.execute = this.execute.bind(this);
this.killWorker = this.killWorker.bind(this);
@ -13,7 +12,7 @@ class WorkerExecutor {
getWorker() {
if (this.worker === null) {
this.worker = new Worker(`${homeLocation}/js/${this.workerName}.js`);
this.worker = new Worker(`${this.location}${this.workerName}.js`);
}
return this.worker;
@ -72,6 +71,6 @@ class WorkerExecutor {
}
}
export default function createWorkerExecutor(workerName) {
return new WorkerExecutor(workerName);
export default function createWorkerExecutor(workerName, location = '/js/') {
return new WorkerExecutor(workerName, location);
}

File diff suppressed because it is too large Load Diff

View File

@ -19,6 +19,7 @@
"lint": "eslint ./**/*.js --fix",
"repack": "babel-node ./repack.js",
"semantic-release": "semantic-release",
"pretest": "cd ../client && npm run build:frame-runner",
"test": "mocha --delay --reporter progress --bail",
"unpack": "babel-node ./unpack.js"
},
@ -26,6 +27,10 @@
"invariant": "^2.2.4"
},
"devDependencies": {
"@babel/core": "^7.2.2",
"@babel/register": "^7.0.0",
"@babel/polyfill": "^7.2.5",
"@babel/preset-env": "^7.2.3",
"@commitlint/cli": "^7.0.0",
"@commitlint/config-conventional": "^7.0.1",
"@commitlint/travis-cli": "^7.0.0",
@ -44,8 +49,6 @@
"babel-standalone": "^6.26.0",
"browserify": "^16.2.2",
"chai": "4.2.0",
"enzyme": "^3.7.0",
"enzyme-adapter-react-16": "^1.6.0",
"eslint": "^4.19.1",
"eslint-config-freecodecamp": "^1.1.1",
"eslint-plugin-import": "^2.11.0",
@ -59,18 +62,13 @@
"js-yaml": "^3.12.0",
"jsdom": "^12.2.0",
"lint-staged": "^7.2.0",
"live-server": "^1.2.1",
"lodash": "^4.17.10",
"mocha": "5.2.0",
"node-sass": "4.9.4",
"prettier": "^1.13.5",
"prettier-package-json": "^1.6.0",
"puppeteer": "1.11.0",
"react": "^16.5.2",
"react-dom": "^16.5.2",
"react-redux": "^5.0.7",
"readdirp-walk": "^1.6.0",
"redux": "^4.0.1",
"redux-thunk": "^2.3.0",
"rx": "^4.1.0",
"semantic-release": "^15.6.0",
"validator": "^10.4.0"

View File

View File

@ -1,23 +1,38 @@
/* global browser, page */
const path = require('path');
const liveServer = require('live-server');
const spinner = require('ora')();
const clientPath = path.resolve(__dirname, '../../client');
require('@babel/polyfill');
require('@babel/register')({
root: clientPath,
babelrc: false,
presets: ['@babel/preset-env'],
ignore: [/node_modules/],
only: [clientPath]
});
const createPseudoWorker = require('./utils/pseudo-worker');
const {
default: createWorker
} = require('../../client/src/templates/Challenges/utils/worker-executor');
const { assert, AssertionError } = require('chai');
const Mocha = require('mocha');
const { flatten } = require('lodash');
const path = require('path');
const fs = require('fs');
const jsdom = require('jsdom');
const dom = new jsdom.JSDOM('');
global.document = dom.window.document;
require('dotenv').config({ path: path.resolve(__dirname, '../../.env') });
const vm = require('vm');
const puppeteer = require('puppeteer');
const jsdom = require('jsdom');
const jQuery = require('jquery');
const Sass = require('node-sass');
const Babel = require('babel-standalone');
const presetEnv = require('babel-preset-env');
const presetReact = require('babel-preset-react');
const { getChallengesForLang } = require('../getChallenges');
const MongoIds = require('./utils/mongoIds');
@ -27,6 +42,15 @@ const { challengeTypes } = require('../../client/utils/challengeTypes');
const { supportedLangs } = require('../utils');
const {
buildDOMChallenge,
buildJSChallenge
} = require('../../client/src/templates/Challenges/utils/build');
const {
createPoly
} = require('../../client/src/templates/Challenges/utils/polyvinyl');
const oldRunnerFail = Mocha.Runner.prototype.fail;
Mocha.Runner.prototype.fail = function(test, err) {
if (err instanceof AssertionError) {
@ -43,17 +67,18 @@ Mocha.Runner.prototype.fail = function(test, err) {
return oldRunnerFail.call(this, test, err);
};
const { JSDOM } = jsdom;
async function newPageContext(browser) {
const page = await browser.newPage();
// it's needed for workers as context.
await page.goto('http://127.0.0.1:8080/index.html');
return page;
}
const babelOptions = {
plugins: ['transform-runtime'],
presets: [presetEnv, presetReact]
};
spinner.start();
spinner.text = 'Populate tests.';
const jQueryScript = fs.readFileSync(
path.resolve('./node_modules/jquery/dist/jquery.slim.min.js'),
'utf8'
);
let browser;
let page;
runTests();
@ -66,13 +91,47 @@ async function runTests() {
testLangs = testLangs.filter(lang => filterLangs.includes(lang));
}
await Promise.all(testLangs.map(lang => populateTestsForLang(lang)));
const challenges = await Promise.all(
testLangs.map(lang => getChallenges(lang))
);
describe('Check challenges', function() {
before(async function() {
spinner.text = 'Testing';
this.timeout(50000);
liveServer.start({
host: '127.0.0.1',
port: '8080',
root: path.resolve(__dirname, 'stubs'),
mount: [['/js', path.join(clientPath, 'static/js')]],
open: false,
logLevel: 0
});
browser = await puppeteer.launch({
args: ['--no-sandbox']
// dumpio: true
});
global.Worker = createPseudoWorker(await newPageContext(browser));
page = await newPageContext(browser);
await page.setViewport({ width: 300, height: 150 });
});
after(async function() {
this.timeout(30000);
if (browser) {
await browser.close();
}
liveServer.shutdown();
spinner.stop();
});
challenges.forEach(populateTestsForLang);
});
run();
}
async function populateTestsForLang(lang) {
const allChallenges = await getChallengesForLang(lang).then(curriculum =>
async function getChallenges(lang) {
const challenges = await getChallengesForLang(lang).then(curriculum =>
Object.keys(curriculum)
.map(key => curriculum[key].blocks)
.reduce((challengeArray, superBlock) => {
@ -82,28 +141,19 @@ async function populateTestsForLang(lang) {
return [...challengeArray, ...flatten(challengesForBlock)];
}, [])
);
return { lang, challenges };
}
function populateTestsForLang({ lang, challenges }) {
const mongoIds = new MongoIds();
const challengeTitles = new ChallengeTitles();
const validateChallenge = challengeSchemaValidator(lang);
describe(`Check challenges (${lang})`, async function() {
before(async function() {
this.timeout(30000);
global.browser = await puppeteer.launch({ args: ['--no-sandbox'] });
global.page = await browser.newPage();
await page.setViewport({ width: 300, height: 150 });
});
after(async function() {
if (global.browser) {
await browser.close();
}
});
describe(`Check challenges (${lang})`, function() {
this.timeout(5000);
allChallenges.forEach(challenge => {
describe(challenge.title || 'No title', async function() {
challenges.forEach(challenge => {
describe(challenge.title || 'No title', function() {
it('Common checks', function() {
const result = validateChallenge(challenge);
if (result.error) {
@ -140,38 +190,26 @@ async function populateTestsForLang(lang) {
});
});
const { files = [], required = [] } = challenge;
const exts = Array.from(new Set(files.map(({ ext }) => ext)));
const groupedFiles = exts.reduce((result, ext) => {
const file = files.filter(file => file.ext === ext).reduce(
(result, file) => ({
head: result.head + '\n' + file.head,
contents: result.contents + '\n' + file.contents,
tail: result.tail + '\n' + file.tail
}),
{ head: '', contents: '', tail: '' }
);
return {
...result,
[ext]: file
};
}, {});
let { files = [] } = challenge;
let evaluateTest;
if (
challengeType === challengeTypes.modern &&
(groupedFiles.js || groupedFiles.jsx)
if (challengeType === challengeTypes.backend) {
it('Check tests is not implemented.');
return;
} else if (
challengeType === challengeTypes.js ||
challengeType === challengeTypes.bonfire
) {
evaluateTest = evaluateReactReduxTest;
} else if (groupedFiles.html) {
evaluateTest = evaluateHtmlTest;
} else if (groupedFiles.js) {
evaluateTest = evaluateJsTest;
} else if (files.length === 1) {
evaluateTest = evaluateHtmlTest;
} else {
it('Check tests. Unknown file type.');
it('Check tests.', () => {
throw new Error('Seed file should be only the one.');
});
return;
}
files = files.map(createPoly);
it('Test suite must fail on the initial contents', async function() {
this.timeout(20000);
// suppress errors in the console.
@ -180,12 +218,14 @@ async function populateTestsForLang(lang) {
let fails = (await Promise.all(
tests.map(async function(test) {
try {
await evaluateTest({
challengeType,
required,
files: groupedFiles,
test
});
await evaluateTest(
{
...challenge,
files,
test
},
page
);
return false;
} catch (e) {
return true;
@ -207,18 +247,20 @@ async function populateTestsForLang(lang) {
return;
}
describe('Check tests against solutions', async function() {
describe('Check tests against solutions', function() {
solutions.forEach((solution, index) => {
describe(`Solution ${index + 1}`, async function() {
describe(`Solution ${index + 1}`, function() {
tests.forEach(test => {
it(test.text, async function() {
await evaluateTest({
challengeType,
solution,
required,
files: groupedFiles,
test
});
await evaluateTest(
{
...challenge,
files,
solution,
test
},
page
);
});
});
});
@ -229,314 +271,68 @@ async function populateTestsForLang(lang) {
});
}
// Fake Deep Equal dependency
const DeepEqual = (a, b) => JSON.stringify(a) === JSON.stringify(b);
// Hardcode Deep Freeze dependency
const DeepFreeze = o => {
Object.freeze(o);
Object.getOwnPropertyNames(o).forEach(function(prop) {
if (
o.hasOwnProperty(prop) &&
o[prop] !== null &&
(typeof o[prop] === 'object' || typeof o[prop] === 'function') &&
!Object.isFrozen(o[prop])
) {
DeepFreeze(o[prop]);
}
});
return o;
};
function transformSass(solution) {
const fragment = JSDOM.fragment(`<div>${solution}</div>`);
const styleTags = fragment.querySelectorAll('style[type="text/sass"]');
if (styleTags.length > 0) {
styleTags.forEach(styleTag => {
styleTag.innerHTML = Sass.renderSync({ data: styleTag.innerHTML }).css;
styleTag.type = 'text/css';
});
return fragment.children[0].innerHTML;
async function evaluateHtmlTest(
{ solution, required = [], template, files, test },
context
) {
if (solution) {
files[0].contents = solution;
}
return solution;
}
async function evaluateHtmlTest({ solution, required, files, test }) {
const { head = '', contents = '', tail = '' } = files.html;
if (!solution) {
solution = contents;
}
const code = solution;
const loadEnzyme = files[0].ext === 'jsx';
const links = required
.map(({ link, src }) => {
if (link && src) {
throw new Error(`
A required file can not have both a src and a link: src = ${src}, link = ${link}
`);
}
if (src) {
return `<script src='${src}' type='text/javascript'></script>`;
}
if (link) {
return `<link href='${link}' rel='stylesheet' />`;
}
return '';
})
.reduce((head, required) => head.concat(required), '');
const scripts = `
<head>
${links}
</head>
`;
const sandbox = { solution, transformSass };
const context = vm.createContext(sandbox);
vm.runInContext('solution = transformSass(solution);', context, {
timeout: 2000
const { build, sources } = await buildDOMChallenge(files, {
required,
template
});
solution = sandbox.solution;
await preparePageToTest();
await context.reload();
await context.setContent(build);
await page.setContent(
`
<!doctype html>
<html>
${scripts}
${head}
${solution}
${tail}
</html>
`
);
const result = await context.evaluate(
async(sources, testString, loadEnzyme) => {
document.__source = sources && 'index' in sources ? sources['index'] : '';
document.__getUserInput = fileName => sources[fileName];
document.__frameReady = () => {};
document.__loadEnzyme = loadEnzyme;
await document.__initTestFrame();
await runTestInBrowser(code, test.testString);
}
async function preparePageToTest() {
await page.reload();
await page.addScriptTag({
url: 'https://cdnjs.cloudflare.com/ajax/libs/chai/4.2.0/chai.min.js'
});
await page.evaluate(() => {
window.assert = window.chai.assert;
});
await page.evaluate(jQueryScript);
}
async function runTestInBrowser(code, testString) {
const result = await page.evaluate(
async function(code, testString) {
/* eslint-disable no-unused-vars */
// Fake Deep Equal dependency
const DeepEqual = (a, b) => JSON.stringify(a) === JSON.stringify(b);
// Hardcode Deep Freeze dependency
const DeepFreeze = o => {
Object.freeze(o);
Object.getOwnPropertyNames(o).forEach(function(prop) {
if (
o.hasOwnProperty(prop) &&
o[prop] !== null &&
(typeof o[prop] === 'object' || typeof o[prop] === 'function') &&
!Object.isFrozen(o[prop])
) {
DeepFreeze(o[prop]);
}
});
return o;
};
const editor = {
getValue() {
return code;
}
};
/* eslint-enable no-unused-vars */
try {
// eslint-disable-next-line no-eval
const test = eval(testString);
if (typeof test === 'function') {
await test(() => code);
}
} catch (e) {
return {
message: e.message,
isAssertion: e instanceof window.chai.AssertionError
};
const { pass, err } = await document.__runTest(testString);
if (pass) {
return true;
} else {
return { ...err };
}
return true;
},
code,
testString
sources,
test.testString,
loadEnzyme
);
if (result !== true) {
throw result.isAssertion
? new AssertionError(result.message)
: new Error(result.message);
throw AssertionError(result.message);
}
}
async function evaluateJsTest({ solution, files, test }) {
const virtualConsole = new jsdom.VirtualConsole();
const dom = new JSDOM('', { runScripts: 'dangerously', virtualConsole });
const { head = '', contents = '', tail = '' } = files.js;
let scriptString = '';
if (!solution) {
solution = contents;
try {
scriptString =
Babel.transform(head, babelOptions).code +
'\n' +
Babel.transform(contents, babelOptions).code +
'\n' +
Babel.transform(tail, babelOptions).code +
'\n';
} catch (e) {
scriptString = '';
}
} else {
scriptString =
Babel.transform(head, babelOptions).code +
'\n' +
Babel.transform(solution, babelOptions).code +
'\n' +
Babel.transform(tail, babelOptions).code +
'\n';
if (solution) {
files[0].contents = solution;
}
dom.window.require = require;
dom.window.code = solution;
await runTestInJsdom(dom, test.testString, scriptString);
}
const { build, sources } = await buildJSChallenge(files);
const code = sources && 'index' in sources ? sources['index'] : '';
const script = build + '\n' + test.testString;
async function evaluateReactReduxTest({ solution, files, test }) {
let head = '',
tail = '';
if (files.js) {
const { head: headJs = '', tail: tailJs = '' } = files.js;
head += headJs + '\n';
tail += tailJs + '\n';
}
if (files.jsx) {
const { head: headJsx = '', tail: tailJsx = '' } = files.jsx;
head += headJsx + '\n';
tail += tailJsx + '\n';
}
const testWorker = createWorker('test-evaluator');
/* Transpile ALL the code
* (we may use JSX in head or tail or tests, too): */
let scriptString = '';
if (!solution) {
const contents =
(files.js ? files.js.contents || '' : '') +
(files.jsx ? files.jsx.contents || '' : '');
solution = contents;
try {
scriptString =
Babel.transform(head, babelOptions).code +
'\n' +
Babel.transform(contents, babelOptions).code +
'\n' +
Babel.transform(tail, babelOptions).code +
'\n';
} catch (e) {
scriptString = '';
try {
const { pass, err } = await testWorker.execute(
{ script, code, sources },
5000
);
if (!pass) {
throw new AssertionError(err.message);
}
} else {
scriptString =
Babel.transform(head, babelOptions).code +
'\n' +
Babel.transform(solution, babelOptions).code +
'\n' +
Babel.transform(tail, babelOptions).code +
'\n';
}
const code = solution;
const virtualConsole = new jsdom.VirtualConsole();
// Mock DOM document for ReactDOM.render method
const dom = new JSDOM(
`
<!doctype html>
<html>
<body>
<div id="root"><div id="challenge-node"></div>
</body>
</html>
`,
{
runScripts: 'dangerously',
virtualConsole,
url: 'http://localhost'
}
);
const { window } = dom;
const document = window.document;
global.window = window;
global.document = document;
global.navigator = {
userAgent: 'node.js'
};
global.requestAnimationFrame = callback => setTimeout(callback, 0);
global.cancelAnimationFrame = id => clearTimeout(id);
// Provide dependencies, just provide all of them
dom.window.React = require('react');
dom.window.ReactDOM = require('react-dom');
dom.window.PropTypes = require('prop-types');
dom.window.Redux = require('redux');
dom.window.ReduxThunk = require('redux-thunk');
dom.window.ReactRedux = require('react-redux');
dom.window.Enzyme = require('enzyme');
const Adapter16 = require('enzyme-adapter-react-16');
dom.window.Enzyme.configure({ adapter: new Adapter16() });
dom.window.require = require;
dom.window.code = code;
dom.window.editor = {
getValue() {
return code;
}
};
await runTestInJsdom(dom, test.testString, scriptString);
}
async function runTestInJsdom(dom, testString, scriptString = '') {
// jQuery used by tests
jQuery(dom.window);
dom.window.assert = assert;
dom.window.DeepEqual = DeepEqual;
dom.window.DeepFreeze = DeepFreeze;
dom.window.__test = testString;
scriptString += `;
window.__result =
(async () => {
try {
const testResult = eval(__test);
if (typeof testResult === 'function') {
await testResult(() => code);
}
}catch (e) {
window.__error = e;
}
})();`;
const script = new vm.Script(scriptString);
dom.runVMScript(script, { timeout: 5000 });
await dom.window.__result;
if (dom.window.__error) {
throw dom.window.__error;
} finally {
testWorker.killWorker();
}
}

View File

@ -0,0 +1,85 @@
function createPseudoWorker(context) {
class PseudoWorker {
constructor(path) {
this.terminated = false;
this.worker = context.evaluateHandle(path => new Worker(path), path);
this.listenToWorker('onmessage');
this.listenToWorker('onerror');
}
terminate() {
this.terminated = true;
this.worker.then(worker =>
context.evaluate(worker => worker.terminate(), worker)
);
}
listenToWorker(eventName) {
this.worker.then(async worker => {
const producer = await context.evaluateHandle(
(worker, eventName) => {
let callback;
const queue = [];
function send(event) {
if (!queue.length && callback) {
callback();
}
queue.push(event);
}
worker[eventName] = e => send(e);
const resolver = resolve => (callback = resolve);
async function* produce() {
while (true) {
while (queue.length) {
yield queue.shift();
}
await new Promise(resolver);
}
}
return produce();
},
worker,
eventName
);
while (!this.terminated) {
try {
const data = await context.evaluate(
producer =>
producer
.next()
.then(({ value: { data, message } }) => ({ data, message })),
producer
);
if (this[eventName]) {
this[eventName](data);
}
} catch (err) {
break;
}
}
});
}
async postMessage(msg) {
if (this.terminated) {
throw new Error('Worker is terminated.');
}
try {
await this.worker.then(worker =>
worker
.executionContext()
.evaluate((worker, msg) => worker.postMessage(msg), worker, msg)
);
} catch (e) {
if (this.onerror) {
this.onerror({ message: e.message });
}
}
}
}
return PseudoWorker;
}
module.exports = createPseudoWorker;