From bc3d88605cba4705f4cd9ae99cbbc5081061358d Mon Sep 17 00:00:00 2001 From: Valeriy Date: Sat, 1 Dec 2018 02:47:32 +0300 Subject: [PATCH] fix(test): use Puppeteer for HTML tests --- curriculum/package-lock.json | 153 +++++++++++++++++--------- curriculum/package.json | 3 +- curriculum/test/test-challenges.js | 168 +++++++++++++++++------------ 3 files changed, 203 insertions(+), 121 deletions(-) diff --git a/curriculum/package-lock.json b/curriculum/package-lock.json index 93eb90bc1d..9ea67d796f 100644 --- a/curriculum/package-lock.json +++ b/curriculum/package-lock.json @@ -4029,26 +4029,6 @@ "randomfill": "^1.0.3" } }, - "css": { - "version": "2.2.4", - "resolved": "https://registry.npmjs.org/css/-/css-2.2.4.tgz", - "integrity": "sha512-oUnjmWpy0niI3x/mPL8dVEI1l7MnG3+HHyRPHf+YFSbK+svOhXpmSOcDURUh2aOCgl2grzrOPt1nHLuCVFULLw==", - "dev": true, - "requires": { - "inherits": "^2.0.3", - "source-map": "^0.6.1", - "source-map-resolve": "^0.5.2", - "urix": "^0.1.0" - }, - "dependencies": { - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true - } - } - }, "css-select": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", @@ -5288,6 +5268,18 @@ "is-extglob": "^1.0.0" } }, + "extract-zip": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz", + "integrity": "sha1-qEC0uK9kAyZMjbV/Txp0Mz74H+k=", + "dev": true, + "requires": { + "concat-stream": "1.6.2", + "debug": "2.6.9", + "mkdirp": "0.5.1", + "yauzl": "2.4.1" + } + }, "extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", @@ -5672,6 +5664,15 @@ } } }, + "fd-slicer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz", + "integrity": "sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU=", + "dev": true, + "requires": { + "pend": "~1.2.0" + } + }, "figures": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", @@ -6258,13 +6259,15 @@ "version": "1.0.0", "resolved": false, "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "resolved": false, "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -6281,19 +6284,22 @@ "version": "1.1.0", "resolved": false, "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", - "dev": true + "dev": true, + "optional": true }, "concat-map": { "version": "0.0.1", "resolved": false, "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", "resolved": false, "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", - "dev": true + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", @@ -6424,7 +6430,8 @@ "version": "2.0.3", "resolved": false, "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", - "dev": true + "dev": true, + "optional": true }, "ini": { "version": "1.3.5", @@ -6438,6 +6445,7 @@ "resolved": false, "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -6454,6 +6462,7 @@ "resolved": false, "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -6462,13 +6471,15 @@ "version": "0.0.8", "resolved": false, "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", - "dev": true + "dev": true, + "optional": true }, "minipass": { "version": "2.2.4", "resolved": false, "integrity": "sha512-hzXIWWet/BzWhYs2b+u7dRHlruXhwdgvlTMDKC6Cb1U7ps6Ac6yQlR39xsbjWJE377YTCtKwIXIpJ5oP+j5y8g==", "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.1.1", "yallist": "^3.0.0" @@ -6489,6 +6500,7 @@ "resolved": false, "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "dev": true, + "optional": true, "requires": { "minimist": "0.0.8" } @@ -6577,7 +6589,8 @@ "version": "1.0.1", "resolved": false, "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", - "dev": true + "dev": true, + "optional": true }, "object-assign": { "version": "4.1.1", @@ -6591,6 +6604,7 @@ "resolved": false, "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", "dev": true, + "optional": true, "requires": { "wrappy": "1" } @@ -6728,6 +6742,7 @@ "resolved": false, "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -11833,6 +11848,12 @@ "sha.js": "^2.4.8" } }, + "pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=", + "dev": true + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -12061,6 +12082,12 @@ "integrity": "sha1-+LsmPqG1/Xp2BNJri+Ob13Z4v4o=", "dev": true }, + "proxy-from-env": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz", + "integrity": "sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4=", + "dev": true + }, "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -12113,6 +12140,45 @@ "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", "dev": true }, + "puppeteer": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.11.0.tgz", + "integrity": "sha512-iG4iMOHixc2EpzqRV+pv7o3GgmU2dNYEMkvKwSaQO/vMZURakwSOn/EYJ6OIRFYOque1qorzIBvrytPIQB3YzQ==", + "dev": true, + "requires": { + "debug": "^4.1.0", + "extract-zip": "^1.6.6", + "https-proxy-agent": "^2.2.1", + "mime": "^2.0.3", + "progress": "^2.0.1", + "proxy-from-env": "^1.0.0", + "rimraf": "^2.6.1", + "ws": "^6.1.0" + }, + "dependencies": { + "debug": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.0.tgz", + "integrity": "sha512-heNPJUJIqC+xB6ayLAMHaIrmN9HKa7aQO8MGqKpvCA+uJYVcvR6l5kgdrhRuwPFHU7P5/A1w0BjByPHwpfTDKg==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", + "dev": true + }, + "progress": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.1.tgz", + "integrity": "sha512-OE+a6vzqazc+K6LxJrX5UPyKFvGnL5CYmq2jFGNIBWHpc4QyE49/YOumcrpQFJpfejmvRtbJzgO1zPmMCqlbBg==", + "dev": true + } + } + }, "q": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", @@ -12809,30 +12875,6 @@ "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=", "dev": true }, - "rework": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/rework/-/rework-1.0.1.tgz", - "integrity": "sha1-MIBqhBNCtUUQqkEQhQzUhTQUSqc=", - "dev": true, - "requires": { - "convert-source-map": "^0.3.3", - "css": "^2.0.0" - }, - "dependencies": { - "convert-source-map": { - "version": "0.3.5", - "resolved": "http://registry.npmjs.org/convert-source-map/-/convert-source-map-0.3.5.tgz", - "integrity": "sha1-8dgClQr33SYxof6+BZZVDIarMZA=", - "dev": true - } - } - }, - "rework-visit": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/rework-visit/-/rework-visit-1.0.0.tgz", - "integrity": "sha1-mUWygD8hni96ygCtuLyfZA+ELJo=", - "dev": true - }, "right-align": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/right-align/-/right-align-0.1.3.tgz", @@ -14918,6 +14960,15 @@ "requires": { "camelcase": "^4.1.0" } + }, + "yauzl": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz", + "integrity": "sha1-lSj0QtqxsihOWLQ3m7GU4i4MQAU=", + "dev": true, + "requires": { + "fd-slicer": "~1.0.1" + } } } } diff --git a/curriculum/package.json b/curriculum/package.json index 1177645426..8aee43a1a4 100644 --- a/curriculum/package.json +++ b/curriculum/package.json @@ -64,14 +64,13 @@ "node-sass": "4.9.4", "prettier": "^1.13.5", "prettier-package-json": "^1.6.0", + "puppeteer": "1.11.0", "react": "^16.5.2", "react-dom": "^16.5.2", "react-redux": "^5.0.7", "readdirp-walk": "^1.6.0", "redux": "^4.0.1", "redux-thunk": "^2.3.0", - "rework": "1.0.1", - "rework-visit":"1.0.0", "rx": "^4.1.0", "semantic-release": "^15.6.0", "validator": "^10.4.0" diff --git a/curriculum/test/test-challenges.js b/curriculum/test/test-challenges.js index 6b595bb661..6b41fdd10b 100644 --- a/curriculum/test/test-challenges.js +++ b/curriculum/test/test-challenges.js @@ -1,3 +1,4 @@ +/* global browser, page */ const { assert, AssertionError } = require('chai'); const Mocha = require('mocha'); @@ -8,6 +9,8 @@ require('dotenv').config({ path: path.resolve(__dirname, '../../.env') }); const vm = require('vm'); +const puppeteer = require('puppeteer'); + const jsdom = require('jsdom'); const jQuery = require('jquery'); const Sass = require('node-sass'); @@ -15,9 +18,6 @@ const Babel = require('babel-standalone'); const presetEnv = require('babel-preset-env'); const presetReact = require('babel-preset-react'); -const rework = require('rework'); -const visit = require('rework-visit'); - const { getChallengesForLang } = require('../getChallenges'); const MongoIds = require('./utils/mongoIds'); @@ -29,8 +29,12 @@ const { LOCALE: lang = 'english' } = process.env; const oldRunnerFail = Mocha.Runner.prototype.fail; Mocha.Runner.prototype.fail = function(test, err) { - // Don't show stacktrace for assertion errors. if (err.stack && err instanceof AssertionError) { + const assertIndex = err.message.indexOf(': expected'); + if (assertIndex !== -1) { + err.message = err.message.slice(0, assertIndex); + } + // Don't show stacktrace for assertion errors. delete err.stack; } return oldRunnerFail.call(this, test, err); @@ -47,7 +51,8 @@ const babelOptions = { }; const jQueryScript = fs.readFileSync( - path.resolve('./node_modules/jquery/dist/jquery.slim.min.js') + path.resolve('./node_modules/jquery/dist/jquery.slim.min.js'), + 'utf8' ); (async function() { @@ -63,6 +68,18 @@ const jQueryScript = fs.readFileSync( ); describe('Check challenges tests', async function() { + before(async function() { + this.timeout(30000); + global.browser = await puppeteer.launch({ args: ['--no-sandbox'] }); + global.page = await browser.newPage(); + await page.setViewport({ width: 300, height: 150 }); + }); + after(async function() { + if (global.browser) { + await browser.close(); + } + }); + this.timeout(5000); allChallenges.forEach(challenge => { @@ -222,10 +239,6 @@ function isPromise(value) { ); } -function timeout(milliseconds) { - return new Promise(resolve => setTimeout(resolve, milliseconds)); -} - function transformSass(solution) { const fragment = JSDOM.fragment(`
${solution}
`); const styleTags = fragment.querySelectorAll('style[type="text/sass"]'); @@ -239,61 +252,13 @@ function transformSass(solution) { return solution; } -const colors = { - red: 'rgb(255, 0, 0)', - green: 'rgb(0, 255, 0)', - blue: 'rgb(0, 0, 255)', - black: 'rgb(0, 0, 0)', - gray: 'rgb(128, 128, 128)', - yellow: 'rgb(255, 255, 0)' -}; - -function replaceColorNamesPlugin(style) { - visit(style, declarations => { - declarations.filter(decl => decl.type === 'declaration').forEach(decl => { - if (colors[decl.value]) { - decl.value = colors[decl.value]; - } - }); - }); -} - -// JSDOM uses CSSStyleDeclaration, which does not convert color keywords -// to 'rgb()' https://github.com/jsakas/CSSStyleDeclaration/issues/48. -// It's a workaround. -function replaceColorNames(solution) { - const fragment = JSDOM.fragment(`
${solution}
`); - const styleTags = fragment.querySelectorAll('style'); - if (styleTags.length > 0) { - styleTags.forEach(styleTag => { - styleTag.innerHTML = rework(styleTag.innerHTML) - .use(replaceColorNamesPlugin) - .toString(); - }); - return fragment.children[0].innerHTML; - } - return solution; -} - -async function evaluateHtmlTest({ - challengeType, - solution, - required, - files, - test -}) { +async function evaluateHtmlTest({ solution, required, files, test }) { const { head = '', contents = '', tail = '' } = files.html; if (!solution) { solution = contents; } const code = solution; - const options = { - resources: 'usable', - runScripts: 'dangerously', - virtualConsole: new jsdom.VirtualConsole() - }; - const links = required .map(({ link, src }) => { if (link && src) { @@ -313,7 +278,6 @@ A required file can not have both a src and a link: src = ${src}, link = ${link} const scripts = ` - ${links} `; @@ -324,9 +288,10 @@ A required file can not have both a src and a link: src = ${src}, link = ${link} timeout: 2000 }); solution = sandbox.solution; - solution = replaceColorNames(solution); - const dom = new JSDOM( + await preparePageToTest(); + + await page.setContent( ` @@ -335,16 +300,83 @@ A required file can not have both a src and a link: src = ${src}, link = ${link} ${solution} ${tail} - `, - options + ` ); - if (links || challengeType === challengeTypes.modern) { - await timeout(1000); - } + await runTestInBrowser(code, test.testString); +} - dom.window.code = code; - await runTestInJsdom(dom, test.testString); +async function preparePageToTest() { + await page.reload(); + await page.addScriptTag({ + url: 'https://cdnjs.cloudflare.com/ajax/libs/chai/4.2.0/chai.min.js' + }); + await page.evaluate(() => { + window.assert = window.chai.assert; + }); + await page.evaluate(jQueryScript); +} + +async function runTestInBrowser(code, testString) { + const result = await page.evaluate( + async function(code, testString) { + /* eslint-disable no-unused-vars */ + // Fake Deep Equal dependency + const DeepEqual = (a, b) => JSON.stringify(a) === JSON.stringify(b); + + // Hardcode Deep Freeze dependency + const DeepFreeze = o => { + Object.freeze(o); + Object.getOwnPropertyNames(o).forEach(function(prop) { + if ( + o.hasOwnProperty(prop) && + o[prop] !== null && + (typeof o[prop] === 'object' || typeof o[prop] === 'function') && + !Object.isFrozen(o[prop]) + ) { + DeepFreeze(o[prop]); + } + }); + return o; + }; + + const editor = { + getValue() { + return code; + } + }; + /* eslint-enable no-unused-vars */ + + const isPromise = value => + value && + typeof value.subscribe !== 'function' && + typeof value.then === 'function'; + + try { + // eslint-disable-next-line no-eval + const test = eval(testString); + if (typeof test === 'function') { + const __result = test(() => code); + if (isPromise(__result)) { + await __result; + } + } + } catch (e) { + return { + message: e.message, + isAssertion: e instanceof window.chai.AssertionError + }; + } + return true; + }, + code, + testString + ); + if (result !== true) { + throw result.isAssertion + ? new AssertionError(result.message) + : new Error(result.message); + } } async function evaluateJsTest({ solution, files, test }) {