fix(test): use Puppeteer for HTML tests

curriculum/package-lock.json

@@ -4029,26 +4029,6 @@
         "randomfill": "^1.0.3"
       }
     },
-    "css": {
-      "version": "2.2.4",
-      "resolved": "https://registry.npmjs.org/css/-/css-2.2.4.tgz",
-      "integrity": "sha512-oUnjmWpy0niI3x/mPL8dVEI1l7MnG3+HHyRPHf+YFSbK+svOhXpmSOcDURUh2aOCgl2grzrOPt1nHLuCVFULLw==",
-      "dev": true,
-      "requires": {
-        "inherits": "^2.0.3",
-        "source-map": "^0.6.1",
-        "source-map-resolve": "^0.5.2",
-        "urix": "^0.1.0"
-      },
-      "dependencies": {
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
-        }
-      }
-    },
     "css-select": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
@@ -5288,6 +5268,18 @@
         "is-extglob": "^1.0.0"
       }
     },
+    "extract-zip": {
+      "version": "1.6.7",
+      "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz",
+      "integrity": "sha1-qEC0uK9kAyZMjbV/Txp0Mz74H+k=",
+      "dev": true,
+      "requires": {
+        "concat-stream": "1.6.2",
+        "debug": "2.6.9",
+        "mkdirp": "0.5.1",
+        "yauzl": "2.4.1"
+      }
+    },
     "extsprintf": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
@@ -5672,6 +5664,15 @@
         }
       }
     },
+    "fd-slicer": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz",
+      "integrity": "sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU=",
+      "dev": true,
+      "requires": {
+        "pend": "~1.2.0"
+      }
+    },
     "figures": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz",
@@ -6258,13 +6259,15 @@
           "version": "1.0.0",
           "resolved": false,
           "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "brace-expansion": {
           "version": "1.1.11",
           "resolved": false,
           "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
           "dev": true,
+          "optional": true,
           "requires": {
             "balanced-match": "^1.0.0",
             "concat-map": "0.0.1"
@@ -6281,19 +6284,22 @@
           "version": "1.1.0",
           "resolved": false,
           "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=",
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "concat-map": {
           "version": "0.0.1",
           "resolved": false,
           "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "console-control-strings": {
           "version": "1.1.0",
           "resolved": false,
           "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=",
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "core-util-is": {
           "version": "1.0.2",
@@ -6424,7 +6430,8 @@
           "version": "2.0.3",
           "resolved": false,
           "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=",
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "ini": {
           "version": "1.3.5",
@@ -6438,6 +6445,7 @@
           "resolved": false,
           "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
           "dev": true,
+          "optional": true,
           "requires": {
             "number-is-nan": "^1.0.0"
           }
@@ -6454,6 +6462,7 @@
           "resolved": false,
           "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
           "dev": true,
+          "optional": true,
           "requires": {
             "brace-expansion": "^1.1.7"
           }
@@ -6462,13 +6471,15 @@
           "version": "0.0.8",
           "resolved": false,
           "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=",
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "minipass": {
           "version": "2.2.4",
           "resolved": false,
           "integrity": "sha512-hzXIWWet/BzWhYs2b+u7dRHlruXhwdgvlTMDKC6Cb1U7ps6Ac6yQlR39xsbjWJE377YTCtKwIXIpJ5oP+j5y8g==",
           "dev": true,
+          "optional": true,
           "requires": {
             "safe-buffer": "^5.1.1",
             "yallist": "^3.0.0"
@@ -6489,6 +6500,7 @@
           "resolved": false,
           "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
           "dev": true,
+          "optional": true,
           "requires": {
             "minimist": "0.0.8"
           }
@@ -6577,7 +6589,8 @@
           "version": "1.0.1",
           "resolved": false,
           "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=",
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "object-assign": {
           "version": "4.1.1",
@@ -6591,6 +6604,7 @@
           "resolved": false,
           "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
           "dev": true,
+          "optional": true,
           "requires": {
             "wrappy": "1"
           }
@@ -6728,6 +6742,7 @@
           "resolved": false,
           "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
           "dev": true,
+          "optional": true,
           "requires": {
             "code-point-at": "^1.0.0",
             "is-fullwidth-code-point": "^1.0.0",
@@ -11833,6 +11848,12 @@
         "sha.js": "^2.4.8"
       }
     },
+    "pend": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
+      "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=",
+      "dev": true
+    },
     "performance-now": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
@@ -12061,6 +12082,12 @@
       "integrity": "sha1-+LsmPqG1/Xp2BNJri+Ob13Z4v4o=",
       "dev": true
     },
+    "proxy-from-env": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz",
+      "integrity": "sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4=",
+      "dev": true
+    },
     "pseudomap": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
@@ -12113,6 +12140,45 @@
       "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=",
       "dev": true
     },
+    "puppeteer": {
+      "version": "1.11.0",
+      "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.11.0.tgz",
+      "integrity": "sha512-iG4iMOHixc2EpzqRV+pv7o3GgmU2dNYEMkvKwSaQO/vMZURakwSOn/EYJ6OIRFYOque1qorzIBvrytPIQB3YzQ==",
+      "dev": true,
+      "requires": {
+        "debug": "^4.1.0",
+        "extract-zip": "^1.6.6",
+        "https-proxy-agent": "^2.2.1",
+        "mime": "^2.0.3",
+        "progress": "^2.0.1",
+        "proxy-from-env": "^1.0.0",
+        "rimraf": "^2.6.1",
+        "ws": "^6.1.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.0.tgz",
+          "integrity": "sha512-heNPJUJIqC+xB6ayLAMHaIrmN9HKa7aQO8MGqKpvCA+uJYVcvR6l5kgdrhRuwPFHU7P5/A1w0BjByPHwpfTDKg==",
+          "dev": true,
+          "requires": {
+            "ms": "^2.1.1"
+          }
+        },
+        "ms": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
+          "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==",
+          "dev": true
+        },
+        "progress": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.1.tgz",
+          "integrity": "sha512-OE+a6vzqazc+K6LxJrX5UPyKFvGnL5CYmq2jFGNIBWHpc4QyE49/YOumcrpQFJpfejmvRtbJzgO1zPmMCqlbBg==",
+          "dev": true
+        }
+      }
+    },
     "q": {
       "version": "1.5.1",
       "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz",
@@ -12809,30 +12875,6 @@
       "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=",
       "dev": true
     },
-    "rework": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/rework/-/rework-1.0.1.tgz",
-      "integrity": "sha1-MIBqhBNCtUUQqkEQhQzUhTQUSqc=",
-      "dev": true,
-      "requires": {
-        "convert-source-map": "^0.3.3",
-        "css": "^2.0.0"
-      },
-      "dependencies": {
-        "convert-source-map": {
-          "version": "0.3.5",
-          "resolved": "http://registry.npmjs.org/convert-source-map/-/convert-source-map-0.3.5.tgz",
-          "integrity": "sha1-8dgClQr33SYxof6+BZZVDIarMZA=",
-          "dev": true
-        }
-      }
-    },
-    "rework-visit": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/rework-visit/-/rework-visit-1.0.0.tgz",
-      "integrity": "sha1-mUWygD8hni96ygCtuLyfZA+ELJo=",
-      "dev": true
-    },
     "right-align": {
       "version": "0.1.3",
       "resolved": "https://registry.npmjs.org/right-align/-/right-align-0.1.3.tgz",
@@ -14918,6 +14960,15 @@
       "requires": {
         "camelcase": "^4.1.0"
       }
+    },
+    "yauzl": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz",
+      "integrity": "sha1-lSj0QtqxsihOWLQ3m7GU4i4MQAU=",
+      "dev": true,
+      "requires": {
+        "fd-slicer": "~1.0.1"
+      }
     }
   }
 }

curriculum/package.json

@@ -64,14 +64,13 @@
     "node-sass": "4.9.4",
     "prettier": "^1.13.5",
     "prettier-package-json": "^1.6.0",
+    "puppeteer": "1.11.0",
     "react": "^16.5.2",
     "react-dom": "^16.5.2",
     "react-redux": "^5.0.7",
     "readdirp-walk": "^1.6.0",
     "redux": "^4.0.1",
     "redux-thunk": "^2.3.0",
-    "rework": "1.0.1",
-    "rework-visit":"1.0.0",
     "rx": "^4.1.0",
     "semantic-release": "^15.6.0",
     "validator": "^10.4.0"

curriculum/test/test-challenges.js

@@ -1,3 +1,4 @@
+/* global browser, page */
 const { assert, AssertionError } = require('chai');
 const Mocha = require('mocha');
 
@@ -8,6 +9,8 @@ require('dotenv').config({ path: path.resolve(__dirname, '../../.env') });
 
 const vm = require('vm');
 
+const puppeteer = require('puppeteer');
+
 const jsdom = require('jsdom');
 const jQuery = require('jquery');
 const Sass = require('node-sass');
@@ -15,9 +18,6 @@ const Babel = require('babel-standalone');
 const presetEnv = require('babel-preset-env');
 const presetReact = require('babel-preset-react');
 
-const rework = require('rework');
-const visit = require('rework-visit');
-
 const { getChallengesForLang } = require('../getChallenges');
 
 const MongoIds = require('./utils/mongoIds');
@@ -29,8 +29,12 @@ const { LOCALE: lang = 'english' } = process.env;
 
 const oldRunnerFail = Mocha.Runner.prototype.fail;
 Mocha.Runner.prototype.fail = function(test, err) {
-  // Don't show stacktrace for assertion errors.
   if (err.stack && err instanceof AssertionError) {
+    const assertIndex = err.message.indexOf(': expected');
+    if (assertIndex !== -1) {
+      err.message = err.message.slice(0, assertIndex);
+    }
+    // Don't show stacktrace for assertion errors.
     delete err.stack;
   }
   return oldRunnerFail.call(this, test, err);
@@ -47,7 +51,8 @@ const babelOptions = {
 };
 
 const jQueryScript = fs.readFileSync(
-  path.resolve('./node_modules/jquery/dist/jquery.slim.min.js')
+  path.resolve('./node_modules/jquery/dist/jquery.slim.min.js'),
+  'utf8'
 );
 
 (async function() {
@@ -63,6 +68,18 @@ const jQueryScript = fs.readFileSync(
   );
 
   describe('Check challenges tests', async function() {
+    before(async function() {
+      this.timeout(30000);
+      global.browser = await puppeteer.launch({ args: ['--no-sandbox'] });
+      global.page = await browser.newPage();
+      await page.setViewport({ width: 300, height: 150 });
+    });
+    after(async function() {
+      if (global.browser) {
+        await browser.close();
+      }
+    });
+
     this.timeout(5000);
 
     allChallenges.forEach(challenge => {
@@ -222,10 +239,6 @@ function isPromise(value) {
   );
 }
 
-function timeout(milliseconds) {
-  return new Promise(resolve => setTimeout(resolve, milliseconds));
-}
-
 function transformSass(solution) {
   const fragment = JSDOM.fragment(`<div>${solution}</div>`);
   const styleTags = fragment.querySelectorAll('style[type="text/sass"]');
@@ -239,61 +252,13 @@ function transformSass(solution) {
   return solution;
 }
 
-const colors = {
+async function evaluateHtmlTest({ solution, required, files, test }) {
-  red: 'rgb(255, 0, 0)',
-  green: 'rgb(0, 255, 0)',
-  blue: 'rgb(0, 0, 255)',
-  black: 'rgb(0, 0, 0)',
-  gray: 'rgb(128, 128, 128)',
-  yellow: 'rgb(255, 255, 0)'
-};
-
-function replaceColorNamesPlugin(style) {
-  visit(style, declarations => {
-    declarations.filter(decl => decl.type === 'declaration').forEach(decl => {
-      if (colors[decl.value]) {
-        decl.value = colors[decl.value];
-      }
-    });
-  });
-}
-
-// JSDOM uses CSSStyleDeclaration, which does not convert color keywords
-// to 'rgb()' https://github.com/jsakas/CSSStyleDeclaration/issues/48.
-// It's a workaround.
-function replaceColorNames(solution) {
-  const fragment = JSDOM.fragment(`<div>${solution}</div>`);
-  const styleTags = fragment.querySelectorAll('style');
-  if (styleTags.length > 0) {
-    styleTags.forEach(styleTag => {
-      styleTag.innerHTML = rework(styleTag.innerHTML)
-        .use(replaceColorNamesPlugin)
-        .toString();
-    });
-    return fragment.children[0].innerHTML;
-  }
-  return solution;
-}
-
-async function evaluateHtmlTest({
-  challengeType,
-  solution,
-  required,
-  files,
-  test
-}) {
   const { head = '', contents = '', tail = '' } = files.html;
   if (!solution) {
     solution = contents;
   }
   const code = solution;
 
-  const options = {
-    resources: 'usable',
-    runScripts: 'dangerously',
-    virtualConsole: new jsdom.VirtualConsole()
-  };
-
   const links = required
     .map(({ link, src }) => {
       if (link && src) {
@@ -313,7 +278,6 @@ A required file can not have both a src and a link: src = ${src}, link = ${link}
 
   const scripts = `
   <head>
-    <script>${jQueryScript}</script>
     ${links}
   </head>
   `;
@@ -324,9 +288,10 @@ A required file can not have both a src and a link: src = ${src}, link = ${link}
     timeout: 2000
   });
   solution = sandbox.solution;
-  solution = replaceColorNames(solution);
 
-  const dom = new JSDOM(
+  await preparePageToTest();
+
+  await page.setContent(
     `
     <!doctype html>
     <html>
@@ -335,16 +300,83 @@ A required file can not have both a src and a link: src = ${src}, link = ${link}
       ${solution}
       ${tail}
     </html>
-  `,
+  `
-    options
   );
 
-  if (links || challengeType === challengeTypes.modern) {
+  await runTestInBrowser(code, test.testString);
-    await timeout(1000);
+}
-  }
 
-  dom.window.code = code;
+async function preparePageToTest() {
-  await runTestInJsdom(dom, test.testString);
+  await page.reload();
+  await page.addScriptTag({
+    url: 'https://cdnjs.cloudflare.com/ajax/libs/chai/4.2.0/chai.min.js'
+  });
+  await page.evaluate(() => {
+    window.assert = window.chai.assert;
+  });
+  await page.evaluate(jQueryScript);
+}
+
+async function runTestInBrowser(code, testString) {
+  const result = await page.evaluate(
+    async function(code, testString) {
+      /* eslint-disable no-unused-vars */
+      // Fake Deep Equal dependency
+      const DeepEqual = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+
+      // Hardcode Deep Freeze dependency
+      const DeepFreeze = o => {
+        Object.freeze(o);
+        Object.getOwnPropertyNames(o).forEach(function(prop) {
+          if (
+            o.hasOwnProperty(prop) &&
+            o[prop] !== null &&
+            (typeof o[prop] === 'object' || typeof o[prop] === 'function') &&
+            !Object.isFrozen(o[prop])
+          ) {
+            DeepFreeze(o[prop]);
+          }
+        });
+        return o;
+      };
+
+      const editor = {
+        getValue() {
+          return code;
+        }
+      };
+      /* eslint-enable no-unused-vars */
+
+      const isPromise = value =>
+        value &&
+        typeof value.subscribe !== 'function' &&
+        typeof value.then === 'function';
+
+      try {
+        // eslint-disable-next-line no-eval
+        const test = eval(testString);
+        if (typeof test === 'function') {
+          const __result = test(() => code);
+          if (isPromise(__result)) {
+            await __result;
+          }
+        }
+      } catch (e) {
+        return {
+          message: e.message,
+          isAssertion: e instanceof window.chai.AssertionError
+        };
+      }
+      return true;
+    },
+    code,
+    testString
+  );
+  if (result !== true) {
+    throw result.isAssertion
+      ? new AssertionError(result.message)
+      : new Error(result.message);
+  }
 }
 
 async function evaluateJsTest({ solution, files, test }) {