refactor: abstract out walk in getChallenges

curriculum/getChallenges.js

@@ -1,6 +1,6 @@
 const path = require('path');
 const { findIndex, reduce, toString } = require('lodash');
-const readDirP = require('readdirp-walk');
+const readDirP = require('readdirp');
 const { parseMarkdown } = require('../tools/challenge-md-parser');
 const { parseMD } = require('../tools/challenge-md-parser/mdx');
 const fs = require('fs');
@@ -124,35 +124,46 @@ function getMetaForBlock(block) {
 exports.getChallengesDirForLang = getChallengesDirForLang;
 exports.getMetaForBlock = getMetaForBlock;
 
-exports.getChallengesForLang = function getChallengesForLang(lang) {
+// This recursively walks the directories starting at root, and calls cb for
-  let curriculum = {};
+// each file/directory and only resolves once all the callbacks do.
+const walk = (root, target, options, cb) => {
   return new Promise(resolve => {
     let running = 1;
     function done() {
       if (--running === 0) {
-        resolve(curriculum);
+        resolve(target);
       }
     }
-    readDirP({ root: getChallengesDirForLang(lang) })
+    readDirP(root, options)
       .on('data', file => {
         running++;
-        buildCurriculum(file, curriculum, lang).then(done);
+        cb(file, target).then(done);
       })
       .on('end', done);
   });
 };
 
-async function buildCurriculum(file, curriculum, lang) {
+exports.getChallengesForLang = async function getChallengesForLang(lang) {
-  const { name, depth, path: filePath, stat } = file;
+  const root = getChallengesDirForLang(lang);
-  const createChallenge = createChallengeCreator(challengesDir, lang);
+  // scaffold the curriculum, first set up the superblocks, then recurse into
-  if (depth === 1 && stat.isDirectory()) {
+  // the blocks
-    // extract the superBlock info
+  const curriculum = await walk(
-    const { order, name: superBlock } = superBlockInfo(name);
+    root,
-    curriculum[superBlock] = { superBlock, order, blocks: {} };
+    {},
-    return;
+    { type: 'directories', depth: 1 },
-  }
+    buildSuperBlocks
-  if (depth === 2 && stat.isDirectory()) {
+  );
-    const blockName = getBlockNameFromPath(filePath);
+  const cb = (file, curriculum) => buildChallenges(file, curriculum, lang);
+  // fill the scaffold with the challenges
+  return walk(
+    root,
+    curriculum,
+    { type: 'files', fileFilter: ['*.md', '*.markdown'] },
+    cb
+  );
+};
+
+async function buildBlocks({ basename: blockName }, curriculum, superBlock) {
   const metaPath = path.resolve(
     __dirname,
     `./challenges/_meta/${blockName}/meta.json`
@@ -167,18 +178,24 @@ async function buildCurriculum(file, curriculum, lang) {
 
   if (!isUpcomingChange || process.env.SHOW_UPCOMING_CHANGES === 'true') {
     // add the block to the superBlock
-      const { name: superBlock } = superBlockInfoFromPath(filePath);
     const blockInfo = { meta: blockMeta, challenges: [] };
-      curriculum[superBlock].blocks[name] = blockInfo;
+    curriculum[superBlock].blocks[blockName] = blockInfo;
-    }
-    return;
-  }
-  if (name === 'meta.json' || name === '.DS_Store') {
-    return;
   }
+}
 
-  const block = getBlockNameFromPath(filePath);
+async function buildSuperBlocks({ path, fullPath }, curriculum) {
-  const { name: superBlock } = superBlockInfoFromPath(filePath);
+  const { order, name: superBlock } = superBlockInfo(path);
+  curriculum[superBlock] = { superBlock, order, blocks: {} };
+
+  const cb = (file, curriculum) => buildBlocks(file, curriculum, superBlock);
+  return walk(fullPath, curriculum, { depth: 1, type: 'directories' }, cb);
+}
+
+async function buildChallenges({ path, filePath }, curriculum, lang) {
+  // path is relative to getChallengesDirForLang(lang)
+  const createChallenge = createChallengeCreator(challengesDir, lang);
+  const block = getBlockNameFromPath(path);
+  const { name: superBlock } = superBlockInfoFromPath(path);
   let challengeBlock;
 
   // TODO: this try block and process exit can all go once errors terminate the

curriculum/package-lock.json

@@ -2936,6 +2936,17 @@
           "requires": {
             "is-extglob": "^2.1.1"
           }
+        },
+        "readdirp": {
+          "version": "2.2.1",
+          "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-2.2.1.tgz",
+          "integrity": "sha512-1JU/8q+VgFZyxwrJ+SVIOsh+KywWGpds3NTqikiKpDMZWScmAYyKIgqkO+ARvNWJfXeXR1zxz7aHF4u4CyH6vQ==",
+          "dev": true,
+          "requires": {
+            "graceful-fs": "^4.1.11",
+            "micromatch": "^3.1.10",
+            "readable-stream": "^2.0.2"
+          }
         }
       }
     },
@@ -3291,15 +3302,6 @@
         "whatwg-url": "^7.0.0"
       }
     },
-    "dbly-linked-list": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/dbly-linked-list/-/dbly-linked-list-0.2.0.tgz",
-      "integrity": "sha512-Ool7y15f6JRDs0YKx7Dh9uiTb1jS1SZLNdT3Y2q16DlaEghXbMsmODS/XittjR2xztt1gJUpz7jVxpqAPF8VGg==",
-      "dev": true,
-      "requires": {
-        "lodash.isequal": "^4.5.0"
-      }
-    },
     "debug": {
       "version": "4.3.1",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
@@ -3657,12 +3659,6 @@
         "through": "~2.3.1"
       }
     },
-    "eventemitter3": {
-      "version": "3.1.2",
-      "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz",
-      "integrity": "sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==",
-      "dev": true
-    },
     "expand-brackets": {
       "version": "2.1.4",
       "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz",
@@ -3897,12 +3893,6 @@
         "pend": "~1.2.0"
       }
     },
-    "fifo": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/fifo/-/fifo-2.3.0.tgz",
-      "integrity": "sha1-GC3o3QYyqkfPaBbZvEMjMX1SWdw=",
-      "dev": true
-    },
     "file-uri-to-path": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
@@ -4927,16 +4917,6 @@
       "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
       "dev": true
     },
-    "isflattenable": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/isflattenable/-/isflattenable-1.1.0.tgz",
-      "integrity": "sha512-qr/kk8a8KmevVz0Sp899TdSAHg9ybc1HnFlFq/3ZW9G4a5IQsHz5fVnmq0poFMpCqI9wLPHSH9cCT2R1ctXoIQ==",
-      "dev": true,
-      "requires": {
-        "lodash.isarguments": "^3.1.0",
-        "lodash.isarray": "^3.0.4"
-      }
-    },
     "isobject": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
@@ -4966,15 +4946,6 @@
       "integrity": "sha1-VlSVc6Zrp5Xc9rniJt5fOy027Do=",
       "dev": true
     },
-    "join-deep": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/join-deep/-/join-deep-1.2.1.tgz",
-      "integrity": "sha512-z1rVo21SUcDXtgWvToo8O2RfuMy6/ScnEWzyvVRRGYdWMrLqT0iGw/jkHxZ4BaBJGzTQxYhi0ZonsHlU+oJ8iw==",
-      "dev": true,
-      "requires": {
-        "reduce-deep": "^1.3.1"
-      }
-    },
     "js-tokens": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -5231,24 +5202,6 @@
       "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==",
       "dev": true
     },
-    "lodash.isarguments": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
-      "integrity": "sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=",
-      "dev": true
-    },
-    "lodash.isarray": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/lodash.isarray/-/lodash.isarray-3.0.4.tgz",
-      "integrity": "sha1-eeTriMNqgSKvhvhEqpvNhRtfu1U=",
-      "dev": true
-    },
-    "lodash.isequal": {
-      "version": "4.5.0",
-      "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz",
-      "integrity": "sha1-QVxEePK8wwEgwizhDtMib30+GOA=",
-      "dev": true
-    },
     "lodash.sortby": {
       "version": "4.7.0",
       "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz",
@@ -6584,15 +6537,6 @@
       "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==",
       "dev": true
     },
-    "queue-cb": {
-      "version": "1.1.5",
-      "resolved": "https://registry.npmjs.org/queue-cb/-/queue-cb-1.1.5.tgz",
-      "integrity": "sha512-PKsogFZgSyIYn9hFD401ul8RWJnTtdUw+owgvWwhcevsYwY/UvJIeIziibB61CLp7/wry5EWLHmXzjeXr441Nw==",
-      "dev": true,
-      "requires": {
-        "fifo": "^2.3.0"
-      }
-    },
     "randombytes": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",
@@ -6666,27 +6610,12 @@
       }
     },
     "readdirp": {
-      "version": "2.2.1",
+      "version": "3.5.0",
-      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-2.2.1.tgz",
+      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz",
-      "integrity": "sha512-1JU/8q+VgFZyxwrJ+SVIOsh+KywWGpds3NTqikiKpDMZWScmAYyKIgqkO+ARvNWJfXeXR1zxz7aHF4u4CyH6vQ==",
+      "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==",
       "dev": true,
       "requires": {
-        "graceful-fs": "^4.1.11",
+        "picomatch": "^2.2.1"
-        "micromatch": "^3.1.10",
-        "readable-stream": "^2.0.2"
-      }
-    },
-    "readdirp-walk": {
-      "version": "1.7.0",
-      "resolved": "https://registry.npmjs.org/readdirp-walk/-/readdirp-walk-1.7.0.tgz",
-      "integrity": "sha512-tGAXcAlxTM2I8SfOUniI/43eienxkywW5NuN3Ov1Qpngjnw8l1LcjSTBc6QwOpv9D0MTdPHb/wmPVKExbuYa0g==",
-      "dev": true,
-      "requires": {
-        "eventemitter3": "^3.1.0",
-        "graceful-fs": "^4.1.11",
-        "micromatch": "^3.1.10",
-        "readable-stream": "^2.0.2",
-        "walk-filtered": "^1.13.0"
       }
     },
     "rechoir": {
@@ -6698,15 +6627,6 @@
         "resolve": "^1.1.6"
       }
     },
-    "reduce-deep": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/reduce-deep/-/reduce-deep-1.3.1.tgz",
-      "integrity": "sha512-UJuIfS1er7MwsTQ5fkjgGo2gwM3WynmCg7VJLv/RqIU8xtCqrkFC922tWyD1dxeA+lC5pyUZi6JdIUKI6UubsA==",
-      "dev": true,
-      "requires": {
-        "isflattenable": "^1.1.0"
-      }
-    },
     "regenerate": {
       "version": "1.4.2",
       "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz",
@@ -7478,15 +7398,6 @@
         "tweetnacl": "~0.14.0"
       }
     },
-    "stack-lifo": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/stack-lifo/-/stack-lifo-0.1.6.tgz",
-      "integrity": "sha512-fNXXK6AHbOIExOtJYPb1RlP8OXQr8tlpDNP5I78ZId9uK+MDCcDAkwGWTDACYLXAwOhaKLTYwkoSOihAt+/cLg==",
-      "dev": true,
-      "requires": {
-        "dbly-linked-list": "0.2.0"
-      }
-    },
     "stack-trace": {
       "version": "0.0.10",
       "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz",
@@ -8282,17 +8193,6 @@
         "browser-process-hrtime": "^1.0.0"
       }
     },
-    "walk-filtered": {
-      "version": "1.24.0",
-      "resolved": "https://registry.npmjs.org/walk-filtered/-/walk-filtered-1.24.0.tgz",
-      "integrity": "sha512-z4dwZ5IpNEfWVYEso60fsgrXpym65M+a03vumbnonJCrMll3H8JVy9xrU2RHpVGMr8rwlr5y+9qpr71h0RWdsA==",
-      "dev": true,
-      "requires": {
-        "join-deep": "^1.2.0",
-        "queue-cb": "^1.1.0",
-        "stack-lifo": "^0.1.6"
-      }
-    },
     "web-namespaces": {
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-1.1.4.tgz",

curriculum/package.json

@@ -53,7 +53,7 @@
     "lodash": "^4.17.20",
     "mocha": "8.2.1",
     "puppeteer": "^5.3.1",
-    "readdirp-walk": "^1.7.0",
+    "readdirp": "^3.5.0",
     "rehype": "^11.0.0",
     "rework-visit": "^1.0.0",
     "string-similarity": "^4.0.2",