fix(test): use Puppeteer for HTML tests

This commit is contained in:
Valeriy
2018-12-01 02:47:32 +03:00
committed by Stuart Taylor
parent 9e9944b81a
commit bc3d88605c
3 changed files with 203 additions and 121 deletions

View File

@ -4029,26 +4029,6 @@
"randomfill": "^1.0.3"
}
},
"css": {
"version": "2.2.4",
"resolved": "https://registry.npmjs.org/css/-/css-2.2.4.tgz",
"integrity": "sha512-oUnjmWpy0niI3x/mPL8dVEI1l7MnG3+HHyRPHf+YFSbK+svOhXpmSOcDURUh2aOCgl2grzrOPt1nHLuCVFULLw==",
"dev": true,
"requires": {
"inherits": "^2.0.3",
"source-map": "^0.6.1",
"source-map-resolve": "^0.5.2",
"urix": "^0.1.0"
},
"dependencies": {
"source-map": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
"dev": true
}
}
},
"css-select": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
@ -5288,6 +5268,18 @@
"is-extglob": "^1.0.0"
}
},
"extract-zip": {
"version": "1.6.7",
"resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz",
"integrity": "sha1-qEC0uK9kAyZMjbV/Txp0Mz74H+k=",
"dev": true,
"requires": {
"concat-stream": "1.6.2",
"debug": "2.6.9",
"mkdirp": "0.5.1",
"yauzl": "2.4.1"
}
},
"extsprintf": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
@ -5672,6 +5664,15 @@
}
}
},
"fd-slicer": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz",
"integrity": "sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU=",
"dev": true,
"requires": {
"pend": "~1.2.0"
}
},
"figures": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz",
@ -6258,13 +6259,15 @@
"version": "1.0.0",
"resolved": false,
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
"dev": true
"dev": true,
"optional": true
},
"brace-expansion": {
"version": "1.1.11",
"resolved": false,
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dev": true,
"optional": true,
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@ -6281,19 +6284,22 @@
"version": "1.1.0",
"resolved": false,
"integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=",
"dev": true
"dev": true,
"optional": true
},
"concat-map": {
"version": "0.0.1",
"resolved": false,
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
"dev": true
"dev": true,
"optional": true
},
"console-control-strings": {
"version": "1.1.0",
"resolved": false,
"integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=",
"dev": true
"dev": true,
"optional": true
},
"core-util-is": {
"version": "1.0.2",
@ -6424,7 +6430,8 @@
"version": "2.0.3",
"resolved": false,
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=",
"dev": true
"dev": true,
"optional": true
},
"ini": {
"version": "1.3.5",
@ -6438,6 +6445,7 @@
"resolved": false,
"integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
"dev": true,
"optional": true,
"requires": {
"number-is-nan": "^1.0.0"
}
@ -6454,6 +6462,7 @@
"resolved": false,
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"dev": true,
"optional": true,
"requires": {
"brace-expansion": "^1.1.7"
}
@ -6462,13 +6471,15 @@
"version": "0.0.8",
"resolved": false,
"integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=",
"dev": true
"dev": true,
"optional": true
},
"minipass": {
"version": "2.2.4",
"resolved": false,
"integrity": "sha512-hzXIWWet/BzWhYs2b+u7dRHlruXhwdgvlTMDKC6Cb1U7ps6Ac6yQlR39xsbjWJE377YTCtKwIXIpJ5oP+j5y8g==",
"dev": true,
"optional": true,
"requires": {
"safe-buffer": "^5.1.1",
"yallist": "^3.0.0"
@ -6489,6 +6500,7 @@
"resolved": false,
"integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
"dev": true,
"optional": true,
"requires": {
"minimist": "0.0.8"
}
@ -6577,7 +6589,8 @@
"version": "1.0.1",
"resolved": false,
"integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=",
"dev": true
"dev": true,
"optional": true
},
"object-assign": {
"version": "4.1.1",
@ -6591,6 +6604,7 @@
"resolved": false,
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"dev": true,
"optional": true,
"requires": {
"wrappy": "1"
}
@ -6728,6 +6742,7 @@
"resolved": false,
"integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
"dev": true,
"optional": true,
"requires": {
"code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0",
@ -11833,6 +11848,12 @@
"sha.js": "^2.4.8"
}
},
"pend": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
"integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=",
"dev": true
},
"performance-now": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
@ -12061,6 +12082,12 @@
"integrity": "sha1-+LsmPqG1/Xp2BNJri+Ob13Z4v4o=",
"dev": true
},
"proxy-from-env": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz",
"integrity": "sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4=",
"dev": true
},
"pseudomap": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
@ -12113,6 +12140,45 @@
"integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=",
"dev": true
},
"puppeteer": {
"version": "1.11.0",
"resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.11.0.tgz",
"integrity": "sha512-iG4iMOHixc2EpzqRV+pv7o3GgmU2dNYEMkvKwSaQO/vMZURakwSOn/EYJ6OIRFYOque1qorzIBvrytPIQB3YzQ==",
"dev": true,
"requires": {
"debug": "^4.1.0",
"extract-zip": "^1.6.6",
"https-proxy-agent": "^2.2.1",
"mime": "^2.0.3",
"progress": "^2.0.1",
"proxy-from-env": "^1.0.0",
"rimraf": "^2.6.1",
"ws": "^6.1.0"
},
"dependencies": {
"debug": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.0.tgz",
"integrity": "sha512-heNPJUJIqC+xB6ayLAMHaIrmN9HKa7aQO8MGqKpvCA+uJYVcvR6l5kgdrhRuwPFHU7P5/A1w0BjByPHwpfTDKg==",
"dev": true,
"requires": {
"ms": "^2.1.1"
}
},
"ms": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
"integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==",
"dev": true
},
"progress": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.1.tgz",
"integrity": "sha512-OE+a6vzqazc+K6LxJrX5UPyKFvGnL5CYmq2jFGNIBWHpc4QyE49/YOumcrpQFJpfejmvRtbJzgO1zPmMCqlbBg==",
"dev": true
}
}
},
"q": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz",
@ -12809,30 +12875,6 @@
"integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=",
"dev": true
},
"rework": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/rework/-/rework-1.0.1.tgz",
"integrity": "sha1-MIBqhBNCtUUQqkEQhQzUhTQUSqc=",
"dev": true,
"requires": {
"convert-source-map": "^0.3.3",
"css": "^2.0.0"
},
"dependencies": {
"convert-source-map": {
"version": "0.3.5",
"resolved": "http://registry.npmjs.org/convert-source-map/-/convert-source-map-0.3.5.tgz",
"integrity": "sha1-8dgClQr33SYxof6+BZZVDIarMZA=",
"dev": true
}
}
},
"rework-visit": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/rework-visit/-/rework-visit-1.0.0.tgz",
"integrity": "sha1-mUWygD8hni96ygCtuLyfZA+ELJo=",
"dev": true
},
"right-align": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/right-align/-/right-align-0.1.3.tgz",
@ -14918,6 +14960,15 @@
"requires": {
"camelcase": "^4.1.0"
}
},
"yauzl": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz",
"integrity": "sha1-lSj0QtqxsihOWLQ3m7GU4i4MQAU=",
"dev": true,
"requires": {
"fd-slicer": "~1.0.1"
}
}
}
}

View File

@ -64,14 +64,13 @@
"node-sass": "4.9.4",
"prettier": "^1.13.5",
"prettier-package-json": "^1.6.0",
"puppeteer": "1.11.0",
"react": "^16.5.2",
"react-dom": "^16.5.2",
"react-redux": "^5.0.7",
"readdirp-walk": "^1.6.0",
"redux": "^4.0.1",
"redux-thunk": "^2.3.0",
"rework": "1.0.1",
"rework-visit":"1.0.0",
"rx": "^4.1.0",
"semantic-release": "^15.6.0",
"validator": "^10.4.0"

View File

@ -1,3 +1,4 @@
/* global browser, page */
const { assert, AssertionError } = require('chai');
const Mocha = require('mocha');
@ -8,6 +9,8 @@ require('dotenv').config({ path: path.resolve(__dirname, '../../.env') });
const vm = require('vm');
const puppeteer = require('puppeteer');
const jsdom = require('jsdom');
const jQuery = require('jquery');
const Sass = require('node-sass');
@ -15,9 +18,6 @@ const Babel = require('babel-standalone');
const presetEnv = require('babel-preset-env');
const presetReact = require('babel-preset-react');
const rework = require('rework');
const visit = require('rework-visit');
const { getChallengesForLang } = require('../getChallenges');
const MongoIds = require('./utils/mongoIds');
@ -29,8 +29,12 @@ const { LOCALE: lang = 'english' } = process.env;
const oldRunnerFail = Mocha.Runner.prototype.fail;
Mocha.Runner.prototype.fail = function(test, err) {
// Don't show stacktrace for assertion errors.
if (err.stack && err instanceof AssertionError) {
const assertIndex = err.message.indexOf(': expected');
if (assertIndex !== -1) {
err.message = err.message.slice(0, assertIndex);
}
// Don't show stacktrace for assertion errors.
delete err.stack;
}
return oldRunnerFail.call(this, test, err);
@ -47,7 +51,8 @@ const babelOptions = {
};
const jQueryScript = fs.readFileSync(
path.resolve('./node_modules/jquery/dist/jquery.slim.min.js')
path.resolve('./node_modules/jquery/dist/jquery.slim.min.js'),
'utf8'
);
(async function() {
@ -63,6 +68,18 @@ const jQueryScript = fs.readFileSync(
);
describe('Check challenges tests', async function() {
before(async function() {
this.timeout(30000);
global.browser = await puppeteer.launch({ args: ['--no-sandbox'] });
global.page = await browser.newPage();
await page.setViewport({ width: 300, height: 150 });
});
after(async function() {
if (global.browser) {
await browser.close();
}
});
this.timeout(5000);
allChallenges.forEach(challenge => {
@ -222,10 +239,6 @@ function isPromise(value) {
);
}
function timeout(milliseconds) {
return new Promise(resolve => setTimeout(resolve, milliseconds));
}
function transformSass(solution) {
const fragment = JSDOM.fragment(`<div>${solution}</div>`);
const styleTags = fragment.querySelectorAll('style[type="text/sass"]');
@ -239,61 +252,13 @@ function transformSass(solution) {
return solution;
}
const colors = {
red: 'rgb(255, 0, 0)',
green: 'rgb(0, 255, 0)',
blue: 'rgb(0, 0, 255)',
black: 'rgb(0, 0, 0)',
gray: 'rgb(128, 128, 128)',
yellow: 'rgb(255, 255, 0)'
};
function replaceColorNamesPlugin(style) {
visit(style, declarations => {
declarations.filter(decl => decl.type === 'declaration').forEach(decl => {
if (colors[decl.value]) {
decl.value = colors[decl.value];
}
});
});
}
// JSDOM uses CSSStyleDeclaration, which does not convert color keywords
// to 'rgb()' https://github.com/jsakas/CSSStyleDeclaration/issues/48.
// It's a workaround.
function replaceColorNames(solution) {
const fragment = JSDOM.fragment(`<div>${solution}</div>`);
const styleTags = fragment.querySelectorAll('style');
if (styleTags.length > 0) {
styleTags.forEach(styleTag => {
styleTag.innerHTML = rework(styleTag.innerHTML)
.use(replaceColorNamesPlugin)
.toString();
});
return fragment.children[0].innerHTML;
}
return solution;
}
async function evaluateHtmlTest({
challengeType,
solution,
required,
files,
test
}) {
async function evaluateHtmlTest({ solution, required, files, test }) {
const { head = '', contents = '', tail = '' } = files.html;
if (!solution) {
solution = contents;
}
const code = solution;
const options = {
resources: 'usable',
runScripts: 'dangerously',
virtualConsole: new jsdom.VirtualConsole()
};
const links = required
.map(({ link, src }) => {
if (link && src) {
@ -313,7 +278,6 @@ A required file can not have both a src and a link: src = ${src}, link = ${link}
const scripts = `
<head>
<script>${jQueryScript}</script>
${links}
</head>
`;
@ -324,9 +288,10 @@ A required file can not have both a src and a link: src = ${src}, link = ${link}
timeout: 2000
});
solution = sandbox.solution;
solution = replaceColorNames(solution);
const dom = new JSDOM(
await preparePageToTest();
await page.setContent(
`
<!doctype html>
<html>
@ -335,16 +300,83 @@ A required file can not have both a src and a link: src = ${src}, link = ${link}
${solution}
${tail}
</html>
`,
options
`
);
if (links || challengeType === challengeTypes.modern) {
await timeout(1000);
}
await runTestInBrowser(code, test.testString);
}
dom.window.code = code;
await runTestInJsdom(dom, test.testString);
async function preparePageToTest() {
await page.reload();
await page.addScriptTag({
url: 'https://cdnjs.cloudflare.com/ajax/libs/chai/4.2.0/chai.min.js'
});
await page.evaluate(() => {
window.assert = window.chai.assert;
});
await page.evaluate(jQueryScript);
}
async function runTestInBrowser(code, testString) {
const result = await page.evaluate(
async function(code, testString) {
/* eslint-disable no-unused-vars */
// Fake Deep Equal dependency
const DeepEqual = (a, b) => JSON.stringify(a) === JSON.stringify(b);
// Hardcode Deep Freeze dependency
const DeepFreeze = o => {
Object.freeze(o);
Object.getOwnPropertyNames(o).forEach(function(prop) {
if (
o.hasOwnProperty(prop) &&
o[prop] !== null &&
(typeof o[prop] === 'object' || typeof o[prop] === 'function') &&
!Object.isFrozen(o[prop])
) {
DeepFreeze(o[prop]);
}
});
return o;
};
const editor = {
getValue() {
return code;
}
};
/* eslint-enable no-unused-vars */
const isPromise = value =>
value &&
typeof value.subscribe !== 'function' &&
typeof value.then === 'function';
try {
// eslint-disable-next-line no-eval
const test = eval(testString);
if (typeof test === 'function') {
const __result = test(() => code);
if (isPromise(__result)) {
await __result;
}
}
} catch (e) {
return {
message: e.message,
isAssertion: e instanceof window.chai.AssertionError
};
}
return true;
},
code,
testString
);
if (result !== true) {
throw result.isAssertion
? new AssertionError(result.message)
: new Error(result.message);
}
}
async function evaluateJsTest({ solution, files, test }) {