test(curriculum): check tests against solutions (#27716)

curriculum/addAssertsToTapTest.js

@@ -13,6 +13,7 @@ function addAssertsToTapTest(tapTest) {
   const assert = tapTest.assert;
 
   assert.isTrue = createIsAssert(tapTest, v => v === true);
+  assert.isFalse = createIsAssert(tapTest, v => v === false);
   assert.isArray = createIsAssert(tapTest, _.isArray);
   assert.isBoolean = createIsAssert(tapTest, _.isBoolean);
   assert.isString = createIsAssert(tapTest, _.isString);

curriculum/challenges/english/03-front-end-libraries/bootstrap/center-text-with-bootstrap.english.md

@@ -104,7 +104,63 @@ tests:
 ## Solution
 <section id='solution'>
 
-```js
+```html
-// solution required
+<link href="https://fonts.googleapis.com/css?family=Lobster" rel="stylesheet" type="text/css">
+<style>
+  .red-text {
+    color: red;
+  }
+
+  h2 {
+    font-family: Lobster, Monospace;
+  }
+
+  p {
+    font-size: 16px;
+    font-family: Monospace;
+  }
+
+  .thick-green-border {
+    border-color: green;
+    border-width: 10px;
+    border-style: solid;
+    border-radius: 50%;
+  }
+
+  .smaller-image {
+    width: 100px;
+  }
+</style>
+
+<div class="container-fluid">
+  <h2 class="red-text text-center">CatPhotoApp</h2>
+
+  <p>Click here for <a href="#">cat photos</a>.</p>
+
+  <a href="#"><img class="smaller-image thick-green-border" src="https://bit.ly/fcc-relaxing-cat" alt="A cute orange cat lying on its back."></a>
+
+  <img src="https://bit.ly/fcc-running-cats" class="img-responsive" alt="Three kittens running towards the camera.">
+  <p>Things cats love:</p>
+  <ul>
+    <li>cat nip</li>
+    <li>laser pointers</li>
+    <li>lasagna</li>
+  </ul>
+  <p>Top 3 things cats hate:</p>
+  <ol>
+    <li>flea treatment</li>
+    <li>thunder</li>
+    <li>other cats</li>
+  </ol>
+  <form action="/submit-cat-photo">
+    <label><input type="radio" name="indoor-outdoor"> Indoor</label>
+    <label><input type="radio" name="indoor-outdoor"> Outdoor</label>
+    <label><input type="checkbox" name="personality"> Loving</label>
+    <label><input type="checkbox" name="personality"> Lazy</label>
+    <label><input type="checkbox" name="personality"> Crazy</label>
+    <input type="text" placeholder="cat photo URL" required>
+    <button type="submit">Submit</button>
+  </form>
+</div>
 ```
 </section>

curriculum/challenges/english/03-front-end-libraries/jquery/change-text-inside-an-element-using-jquery.english.md

@@ -88,7 +88,34 @@ tests:
 ## Solution
 <section id='solution'>
 
-```js
+```html
-// solution required
+<script>
+  $(document).ready(function() {
+    $("#target1").css("color", "red");
+    $("#target4").html('<em>#target4</em>');
+  });
+</script>
+
+<div class="container-fluid">
+  <h3 class="text-primary text-center">jQuery Playground</h3>
+  <div class="row">
+    <div class="col-xs-6">
+      <h4>#left-well</h4>
+      <div class="well" id="left-well">
+        <button class="btn btn-default target" id="target1">#target1</button>
+        <button class="btn btn-default target" id="target2">#target2</button>
+        <button class="btn btn-default target" id="target3">#target3</button>
+      </div>
+    </div>
+    <div class="col-xs-6">
+      <h4>#right-well</h4>
+      <div class="well" id="right-well">
+        <button class="btn btn-default target" id="target4">#target4</button>
+        <button class="btn btn-default target" id="target5">#target5</button>
+        <button class="btn btn-default target" id="target6">#target6</button>
+      </div>
+    </div>
+  </div>
+</div>
 ```
 </section>

curriculum/challenges/english/03-front-end-libraries/sass/use-for-to-create-a-sass-loop.english.md

@@ -74,7 +74,35 @@ tests:
 ## Solution
 <section id='solution'>
 
-```js
+```html
-// solution required
+<style type='text/sass'>
+
+@for $i from 1 through 5 {
+  .text-#{$i} { font-size: 10px * $i; }
+}
+
+</style>
+
+<p class="text-1">Hello</p>
+<p class="text-2">Hello</p>
+<p class="text-3">Hello</p>
+<p class="text-4">Hello</p>
+<p class="text-5">Hello</p>
+```
+
+```html
+<style type='text/sass'>
+
+@for $i from 1 to 6 {
+  .text-#{$i} { font-size: 10px * $i; }
+}
+
+</style>
+
+<p class="text-1">Hello</p>
+<p class="text-2">Hello</p>
+<p class="text-3">Hello</p>
+<p class="text-4">Hello</p>
+<p class="text-5">Hello</p>
 ```
 </section>

curriculum/challenges/english/04-data-visualization/data-visualization-with-d3/add-classes-with-d3.english.md

@@ -74,7 +74,27 @@ tests:
 ## Solution
 <section id='solution'>
 
-```js
+```html
-.attr("class","bar");
+<style>
+  .bar {
+    width: 25px;
+    height: 100px;
+    display: inline-block;
+    background-color: blue;
+  }
+</style>
+<body>
+  <script>
+    const dataset = [12, 31, 22, 17, 25, 18, 29, 14, 9];
+
+    d3.select("body").selectAll("div")
+      .data(dataset)
+      .enter()
+      .append("div")
+      // Add your code below this line
+      .attr("class","bar");
+      // Add your code above this line
+  </script>
+</body>
 ```
 </section>

curriculum/challenges/english/08-coding-interview-prep/rosetta-code/fractran.english.md

@@ -1,7 +1,7 @@
 ---
 title: Fractran
 id: 5a7dad05be01840e1778a0d1
-challengeType: 3
+challengeType: 5
 ---
 
 ## Description

curriculum/package.json

@@ -45,6 +45,8 @@
     "babel-preset-stage-3": "^6.24.1",
     "babel-standalone": "^6.26.0",
     "browserify": "^16.2.2",
+    "enzyme": "^3.7.0",
+    "enzyme-adapter-react-16": "^1.6.0",
     "eslint": "^4.19.1",
     "eslint-config-freecodecamp": "^1.1.1",
     "eslint-plugin-import": "^2.11.0",
@@ -56,11 +58,20 @@
     "joi-objectid": "^2.0.0",
     "jquery": "^3.3.1",
     "js-yaml": "^3.12.0",
+    "jsdom": "^12.2.0",
     "lint-staged": "^7.2.0",
     "lodash": "^4.17.10",
+    "node-sass": "4.9.4",
     "prettier": "^1.13.5",
     "prettier-package-json": "^1.6.0",
+    "react": "^16.5.2",
+    "react-dom": "^16.5.2",
+    "react-redux": "^5.0.7",
     "readdirp-walk": "^1.6.0",
+    "redux": "^4.0.1",
+    "redux-thunk": "^2.3.0",
+    "rework": "1.0.1",
+    "rework-visit":"1.0.0",
     "rx": "^4.1.0",
     "semantic-release": "^15.6.0",
     "tap-spec": "^5.0.0",

curriculum/test-challenges.js

@@ -5,22 +5,68 @@ const tape = require('tape');
 const { flatten } = require('lodash');
 const vm = require('vm');
 const path = require('path');
+const fs = require('fs');
 require('dotenv').config({ path: path.resolve(__dirname, '../.env') });
 
+const { JSDOM } = require('jsdom');
+const jQuery = require('jquery');
+const Sass = require('node-sass');
+const Babel = require('babel-standalone');
+const presetEnv = require('babel-preset-env');
+const presetReact = require('babel-preset-react');
+
+const rework = require('rework');
+const visit = require('rework-visit');
+
 const { getChallengesForLang } = require('./getChallenges');
 
 const MongoIds = require('./mongoIds');
 const ChallengeTitles = require('./challengeTitles');
 const addAssertsToTapTest = require('./addAssertsToTapTest');
 const { validateChallenge } = require('./schema/challengeSchema');
-
-const { LOCALE: lang } = process.env;
-
 const { challengeTypes } = require('../client/utils/challengeTypes');
 
+const { LOCALE: lang = 'english' } = process.env;
+
 let mongoIds = new MongoIds();
 let challengeTitles = new ChallengeTitles();
 
+const babelOptions = {
+  plugins: ['transform-runtime'],
+  presets: [presetEnv, presetReact]
+};
+
+const jQueryScript = fs.readFileSync(
+  path.resolve('./node_modules/jquery/dist/jquery.slim.min.js')
+);
+
+// Fake Deep Equal dependency
+const DeepEqual = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+
+// Hardcode Deep Freeze dependency
+const DeepFreeze = o => {
+  Object.freeze(o);
+  Object.getOwnPropertyNames(o).forEach(function(prop) {
+    if (
+      o.hasOwnProperty(prop) &&
+      o[prop] !== null &&
+      (typeof o[prop] === 'object' || typeof o[prop] === 'function') &&
+      !Object.isFrozen(o[prop])
+    ) {
+      DeepFreeze(o[prop]);
+    }
+  });
+  return o;
+};
+
+function isPromise(value) {
+  return (
+    value &&
+    typeof value.subscribe !== 'function' &&
+    typeof value.then === 'function'
+  );
+}
+
 function checkSyntax(test, tapTest) {
   try {
     // eslint-disable-next-line
@@ -31,161 +77,299 @@ function checkSyntax(test, tapTest) {
   }
 }
 
-function evaluateHtmlJsTest(
+async function runScript(scriptString, sandbox) {
+  const context = vm.createContext(sandbox);
+  scriptString += `;
+    (async () => {
+      const testResult = eval(test);
+      if (typeof testResult === 'function') {
+        const __result = testResult(() => code);
+        if (isPromise(__result)) {
+          await __result;
+        }
+      }
+    })();`;
+  const script = new vm.Script(scriptString);
+  script.runInContext(context);
+}
+
+function transformSass(solution) {
+  const fragment = JSDOM.fragment(`<div>${solution}</div>`);
+  const styleTags = fragment.querySelectorAll('style[type="text/sass"]');
+  if (styleTags.length > 0) {
+    styleTags.forEach(styleTag => {
+      styleTag.innerHTML = Sass.renderSync({ data: styleTag.innerHTML }).css;
+      styleTag.type = 'text/css';
+    });
+    return fragment.children[0].innerHTML;
+  }
+  return solution;
+}
+
+const colors = {
+  red: 'rgb(255, 0, 0)',
+  green: 'rgb(0, 255, 0)',
+  blue: 'rgb(0, 0, 255)',
+  black: 'rgb(0, 0, 0)',
+  gray: 'rgb(128, 128, 128)',
+  yellow: 'rgb(255, 255, 0)'
+};
+
+function replaceColorNamesPlugin(style) {
+  visit(style, (declarations, node) => {
+    declarations
+      .filter(decl => decl.type === 'declaration')
+      .forEach(decl => {
+        if (colors[decl.value]) {
+          decl.value = colors[decl.value];
+        }
+      });
+  });
+}
+
+// JSDOM uses CSSStyleDeclaration, which does not convert color keywords
+// to 'rgb()' https://github.com/jsakas/CSSStyleDeclaration/issues/48.
+// It's a workaround.
+function replaceColorNames(solution) {
+  const fragment = JSDOM.fragment(`<div>${solution}</div>`);
+  const styleTags = fragment.querySelectorAll('style');
+  if (styleTags.length > 0) {
+    styleTags.forEach(styleTag => {
+      styleTag.innerHTML = rework(styleTag.innerHTML)
+        .use(replaceColorNamesPlugin)
+        .toString();
+    });
+    return fragment.children[0].innerHTML;
+  }
+  return solution;
+
+}
+
+async function evaluateHtmlTest(
+  challengeType,
   solution,
   assert,
+  required,
   files,
   test,
   tapTest
 ) {
-  // Fake Deep Equal dependency
+  try {
-  const DeepEqual = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+    const code = solution;
+    const { head = '', tail = '' } = files.html;
 
-  // Hardcode Deep Freeze dependency
+    const options = {
-  const DeepFreeze = o => {
+      resources: 'usable',
-    Object.freeze(o);
+      runScripts: 'dangerously'
-    Object.getOwnPropertyNames(o).forEach(function(prop) {
+    };
-      if (
-        o.hasOwnProperty(prop) &&
-        o[prop] !== null &&
-        (typeof o[prop] === 'object' || typeof o[prop] === 'function') &&
-        !Object.isFrozen(o[prop])
-      ) {
-        DeepFreeze(o[prop]);
-      }
-    });
-    return o;
-  };
 
-  let sandbox = {
+    const links = required
-    assert,
+      .map(({ link, src }) => {
-    code: solution,
+        if (link && src) {
-    DeepEqual,
+          throw new Error(`
-    DeepFreeze,
+A required file can not have both a src and a link: src = ${src}, link = ${link}
-    test: test.testString
+  `);
-  };
+        }
+        if (src) {
+          return `<script src='${src}' type='text/javascript'></script>`;
+        }
+        if (link) {
+          return `<link href='${link}' rel='stylesheet' />`;
+        }
+        return '';
+      })
+      .reduce((head, required) => head.concat(required), '');
+
+    const scripts = `
+    <head>
+      <script>${jQueryScript}</script>
+      ${links}
+    </head>
+    `;
+
+    solution = transformSass(solution);
+    solution = replaceColorNames(solution);
 
-  if (files.html) {
-    const { head, tail } = files.html;
-    const { JSDOM } = require('jsdom');
     const jsdom = new JSDOM(`
       <!doctype html>
       <html>
+        ${scripts}
         ${head}
         ${solution}
         ${tail}
       </html>
-    `);
+    `, options);
-    const jQuery = require('jquery')(jsdom.window);
-    sandbox = {
-      ...sandbox,
-      window: jsdom.window,
-      document: jsdom.window.document,
-      $: jQuery
-    };
-  }
 
-  let scriptString = '';
+    // jQuery used by tests
-  if (files.js) {
+    jQuery(jsdom.window);
-    const { head, tail } = files.js;
-    scriptString = head + '\n' + solution + '\n' + tail + '\n';
-  }
 
-  try {
+    if (links || challengeType === challengeTypes.modern) {
-    const context = vm.createContext(sandbox);
+      await new Promise(resolve => setTimeout(resolve, 1000));
-    scriptString += `
+    }
-      const testResult = eval(test);
+
+    jsdom.window.assert = assert;
+    jsdom.window.code = code;
+    jsdom.window.DeepEqual = DeepEqual;
+    jsdom.window.DeepFreeze = DeepFreeze;
+    jsdom.window.isPromise = isPromise;
+    jsdom.window.__test = test.testString;
+    const scriptString = `;
+    (async () => {
+      const testResult = eval(__test);
       if (typeof testResult === 'function') {
-        testResult(() => code);
+        const __result = testResult(() => code);
-      }`;
+        if (isPromise(__result)) {
+          await __result;
+        }
+      }
+    })();`;
     const script = new vm.Script(scriptString);
-    script.runInContext(context);
+    jsdom.runVMScript(script);
+    jsdom.window.close();
   } catch (e) {
-    // console.log(scriptString);
-    // console.log(e);
     tapTest.fail(e);
-    // process.exit(1);
   }
 }
 
-function evaluateReactReduxTest() {
+async function evaluateJsTest(
-  /* NOTE: Provide dependencies for React/Redux challenges
+  challengeType,
-               * and configure testing environment
+  solution,
-               */
+  assert,
-  // let React, ReactDOM, Redux, ReduxThunk, ReactRedux, Enzyme, document;
+  required,
+  files,
+  test,
+  tapTest
+) {
 
-  // if (react || redux || reactRedux) {
+  try {
-  //   // Provide dependencies, just provide all of them
+    let sandbox = {
-  //   React = require('react');
+      assert,
-  //   ReactDOM = require('react-dom');
+      code: solution,
-  //   Redux = require('redux');
+      DeepEqual,
-  //   ReduxThunk = require('redux-thunk');
+      DeepFreeze,
-  //   ReactRedux = require('react-redux');
+      isPromise,
-  //   Enzyme = require('enzyme');
+      test: test.testString
-  //   const Adapter15 = require('enzyme-adapter-react-15');
+    };
-  //   Enzyme.configure({ adapter: new Adapter15() });
 
-  //   /* Transpile ALL the code
+    const { head = '', tail = '' } = files.js;
-  //                * (we may use JSX in head or tail or tests, too): */
+    const scriptString = head + '\n' + solution + '\n' + tail + '\n';
-  //   const transform = require('babel-standalone').transform;
-  //   const options = { presets: ['es2015', 'react'] };
 
-  //   head = transform(head, options).code;
+    runScript(scriptString, sandbox);
-  //   solution = transform(solution, options).code;
+  } catch (e) {
-  //   tail = transform(tail, options).code;
+    tapTest.fail(e);
-  //   test = transform(test, options).code;
+  }
 
-  //   const { JSDOM } = require('jsdom');
+}
-  //   // Mock DOM document for ReactDOM.render method
-  //   const jsdom = new JSDOM(`<!doctype html>
-  //                 <html>
-  //                   <body>
-  //                     <div id="challenge-node"></div>
-  //                   </body>
-  //                 </html>
-  //               `);
-  //   const { window } = jsdom;
 
-  //   // Mock DOM for ReactDOM tests
+async function evaluateReactReduxTest(
-  //   document = window.document;
+  challengeType,
-  //   global.window = window;
+  solution,
-  //   global.document = window.document;
+  assert,
-  // }
+  required,
+  files,
+  test,
+  tapTest
+) {
 
-  /* eslint-enable no-unused-vars */
+  try {
+    const code = solution;
+    let sandbox = {
+      assert,
+      code,
+      DeepEqual,
+      DeepFreeze,
+      isPromise
+    };
+    /* Transpile ALL the code
+    * (we may use JSX in head or tail or tests, too): */
+    solution = Babel.transform(solution, babelOptions).code;
+    const testString = Babel.transform(test.testString, babelOptions).code;
 
-  // No support for async tests
+    sandbox = {
-  // const isAsync = s => s.includes('(async () => ');
+      ...sandbox,
+      test: testString
+    };
 
-  // try {
+    let head = '', tail = '';
-  //   if (!isAsync(test.testString)) {
+    if (files.js) {
-  //     const context = vm.createContext(sandbox);
+      const { head: headJs = '', tail: tailJs = '' } = files.js;
-  //     const scriptString =
+      head += Babel.transform(headJs, babelOptions).code + '\n';
-  //       head + '\n' + solution + '\n' + tail + '\n' + `
+      tail += Babel.transform(tailJs, babelOptions).code + '\n';
-  //       const testResult = eval(test);
+    }
-  //       if (typeof testResult === 'function') {
+    if (files.jsx) {
-  //         testResult(() => code);
+      const { head: headJsx = '', tail: tailJsx = '' } = files.jsx;
-  //       }`;
+      head += Babel.transform(headJsx, babelOptions).code + '\n';
-  //     const script = new vm.Script(scriptString);
+      tail += Babel.transform(tailJsx, babelOptions).code + '\n';
-  //     script.runInContext(context);
+    }
-  //   } else {
+
-  //     // For async tests only check syntax
+    const scriptString = head + '\n' + solution + '\n' + tail + '\n';
-  //     // eslint-disable-next-line
+
-  //     new vm.Script(test.testString);
+    // Mock DOM document for ReactDOM.render method
-  //     tapTest.pass(test.text);
+    const jsdom = new JSDOM(`
-  //   }
+      <!doctype html>
-  // } catch (e) {
+      <html>
-  //   console.log(head + '\n' + solution + '\n' + tail + '\n' + test.testString);
+        <body>
-  //   // console.log(e);
+        <div id="root"><div id="challenge-node"></div>
-  //   tapTest.fail(e);
+        </body>
-  //   // process.exit(1);
+      </html>
-  // }
+    `);
+
+    const { window } = jsdom;
+    const document = window.document;
+
+    global.window = window;
+    global.document = document;
+
+    global.navigator = {
+      userAgent: 'node.js'
+    };
+    global.requestAnimationFrame = callback => setTimeout(callback, 0);
+    global.cancelAnimationFrame = id => clearTimeout(id);
+    // copyProps(window, global);
+
+    // Provide dependencies, just provide all of them
+    const React = require('react');
+    const ReactDOM = require('react-dom');
+    const PropTypes = require('prop-types');
+    const Redux = require('redux');
+    const ReduxThunk = require('redux-thunk');
+    const ReactRedux = require('react-redux');
+    const Enzyme = require('enzyme');
+    const Adapter16 = require('enzyme-adapter-react-16');
+    Enzyme.configure({ adapter: new Adapter16() });
+
+    sandbox = {
+      ...sandbox,
+      require,
+      setTimeout,
+      window,
+      document,
+      React,
+      ReactDOM,
+      PropTypes,
+      Redux,
+      ReduxThunk,
+      ReactRedux,
+      Enzyme,
+      editor: {
+        getValue() {
+          return code;
+        }
+      }
+    };
+
+    runScript(scriptString, sandbox);
+    jsdom.window.close();
+  } catch (e) {
+    tapTest.fail(e);
+  }
 }
 
 function createTest({
   title,
   id = '',
   challengeType,
+  required = [],
   tests = [],
   solutions = [],
   files = []
@@ -216,10 +400,11 @@ function createTest({
     !!solution && !noSolution.test(solution)
   ));
 
-  const skipTests = challengeType !== challengeTypes.html &&
+  const skipTests =
+    challengeType !== challengeTypes.html &&
     challengeType !== challengeTypes.js &&
     challengeType !== challengeTypes.bonfire &&
-    challengeType !== challengeTypes.zipline;
+    challengeType !== challengeTypes.modern;
 
   // For problems without a solution, check only the syntax of the tests.
   if (solutions.length === 0 || skipTests) {
@@ -236,8 +421,8 @@ function createTest({
   const groupedFiles = exts.reduce((result, ext) => {
     const file = files.filter(file => file.ext === ext ).reduce(
       (result, file) => ({
-        head: result.head + ';' + file.head,
+        head: result.head + '\n' + file.head,
-        tail: result.tail + ';' + file.tail
+        tail: result.tail + '\n' + file.tail
       }),
       { head: '', tail: '' }
     );
@@ -247,6 +432,17 @@ function createTest({
     };
   }, {});
 
+  let evaluateTest;
+  if (challengeType === challengeTypes.modern &&
+     (groupedFiles.js || groupedFiles.jsx)) {
+    evaluateTest = evaluateReactReduxTest;
+  } else if (groupedFiles.html) {
+    evaluateTest = evaluateHtmlTest;
+  } else if (groupedFiles.js) {
+    evaluateTest = evaluateJsTest;
+  } else {
+    throw new Error(`Unknown challenge type ${title}`);
+  }
   const plan = tests.length * solutions.length;
   return testSuite
     .flatMap(tapTest => {
@@ -254,25 +450,28 @@ function createTest({
       return (
         Observable.just(tapTest)
           .map(addAssertsToTapTest)
-          .doOnNext(assert => {
+          .flatMap(assert =>
-            solutions.forEach(solution => {
+            Observable.from(solutions)
-              tests.forEach(test => {
+              .flatMap(solution =>
-                evaluateHtmlJsTest(
+                Observable.from(tests)
-                  solution,
+                  .flatMap(test => evaluateTest(
-                  assert,
+                    challengeType,
-                  groupedFiles,
+                    solution,
-                  test,
+                    assert,
-                  tapTest
+                    required,
-                );
+                    groupedFiles,
-              });
+                    test,
-            });
+                    tapTest
-          })
+                  )
+                )
+              )
+          )
           .ignoreElements()
       );
     });
 }
 
-Observable.fromPromise(getChallengesForLang(lang || 'english'))
+Observable.fromPromise(getChallengesForLang(lang))
   .flatMap(curriculum => {
     const allChallenges = Object.keys(curriculum)
     .map(key => curriculum[key].blocks)