fix: fixed several issues after move to mono repo (#31)

This PR has the following items:

1. Introduces a couple of fixes to the sweeper script from issues right before the move to the mono repo.  These were related to moving labeler out of the root of the sweeper folder and placing into the pr-tasks sub-folder.

2. Combined two scripts which were being used to update the data manually on Glitch (pr-relations.glitch.me).  The new one-off script (get-pr-relations-data.js) downloads the current data.json and then pulls down applicable Github data which updates the data.json and automatically uploads it back to the Glitch server.

3. In an effort to use the same log file across all current one-off scripts and sweeper.js, the ProcessLog class was modified, so now every log created has the same JSON structure as is uploaded to the Glitch server.  This removed a lot of redundant code across 3 files.

4. During a pair-coding session with @honmanyau, we tweaked the UI for the Dashboard app to give it a more consistent look across all views.

5. Based on feedback from a couple of the moderators using the new Dashboard app, the PR title was added along with some other styling. 

6. Added some environment variables for running dashboard-api and dashboard-client in a development mode.

7.  Added a sample_data.json file as a starter file, so someone does not have to do a full data pull to get things up and running. 

8. Modified Pareto view to only display files with 2 or more PRs to speed up the report.

.gitignore

@@ -92,14 +92,18 @@ typings/
 /coverage
 
 # production
-/build
+dashboard-client/build
 
 # ------------
 # Custom Files
 # ------------
 
 # local-work-logs
-work-logs/*
+sweeper/work-logs/*
 
 # local input files
-input-files/*
+sweeper/input-files/*
+
+dashboard-api/data.json
+dashboard-api/public
+dashboard-api/uploads

dashboard-api/data.js

@@ -0,0 +1,15 @@
+const fs = require('fs');
+
+let data = require('./data.json');
+
+const Container = {
+  data,
+  update(newData) {
+    Container.data = newData;
+  },
+  getData() {
+    return Container.data;
+  }
+};
+
+module.exports = Container;

dashboard-api/package-lock.json

@@ -12,6 +12,11 @@
 				"negotiator": "0.6.1"
 			}
 		},
+		"append-field": {
+			"version": "1.0.0",
+			"resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
+			"integrity": "sha1-HjRA6RXwsSA9I3SOeO3XubW0PlY="
+		},
 		"array-flatten": {
 			"version": "1.1.1",
 			"resolved": "http://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
@@ -34,11 +39,65 @@
 				"type-is": "~1.6.16"
 			}
 		},
+		"buffer-from": {
+			"version": "1.1.1",
+			"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
+			"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
+		},
+		"busboy": {
+			"version": "0.2.14",
+			"resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz",
+			"integrity": "sha1-bCpiLvz0fFe7vh4qnDetNseSVFM=",
+			"requires": {
+				"dicer": "0.2.5",
+				"readable-stream": "1.1.x"
+			}
+		},
 		"bytes": {
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
 			"integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg="
 		},
+		"concat-stream": {
+			"version": "1.6.2",
+			"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
+			"integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
+			"requires": {
+				"buffer-from": "^1.0.0",
+				"inherits": "^2.0.3",
+				"readable-stream": "^2.2.2",
+				"typedarray": "^0.0.6"
+			},
+			"dependencies": {
+				"isarray": {
+					"version": "1.0.0",
+					"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+					"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
+				},
+				"readable-stream": {
+					"version": "2.3.6",
+					"resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
+					"integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
+					"requires": {
+						"core-util-is": "~1.0.0",
+						"inherits": "~2.0.3",
+						"isarray": "~1.0.0",
+						"process-nextick-args": "~2.0.0",
+						"safe-buffer": "~5.1.1",
+						"string_decoder": "~1.1.1",
+						"util-deprecate": "~1.0.1"
+					}
+				},
+				"string_decoder": {
+					"version": "1.1.1",
+					"resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+					"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+					"requires": {
+						"safe-buffer": "~5.1.0"
+					}
+				}
+			}
+		},
 		"content-disposition": {
 			"version": "0.5.2",
 			"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz",
@@ -59,6 +118,32 @@
 			"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
 			"integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw="
 		},
+		"core-util-is": {
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
+			"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
+		},
+		"cross-env": {
+			"version": "5.2.0",
+			"resolved": "https://registry.npmjs.org/cross-env/-/cross-env-5.2.0.tgz",
+			"integrity": "sha512-jtdNFfFW1hB7sMhr/H6rW1Z45LFqyI431m3qU6bFXcQ3Eh7LtBuG3h74o7ohHZ3crrRkkqHlo4jYHFPcjroANg==",
+			"requires": {
+				"cross-spawn": "^6.0.5",
+				"is-windows": "^1.0.0"
+			}
+		},
+		"cross-spawn": {
+			"version": "6.0.5",
+			"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
+			"integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==",
+			"requires": {
+				"nice-try": "^1.0.4",
+				"path-key": "^2.0.1",
+				"semver": "^5.5.0",
+				"shebang-command": "^1.2.0",
+				"which": "^1.2.9"
+			}
+		},
 		"debug": {
 			"version": "2.6.9",
 			"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -77,6 +162,20 @@
 			"resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
 			"integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
 		},
+		"dicer": {
+			"version": "0.2.5",
+			"resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz",
+			"integrity": "sha1-WZbAhrszIYyBLAkL3cCc0S+stw8=",
+			"requires": {
+				"readable-stream": "1.1.x",
+				"streamsearch": "0.1.2"
+			}
+		},
+		"dotenv": {
+			"version": "6.2.0",
+			"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-6.2.0.tgz",
+			"integrity": "sha512-HygQCKUBSFl8wKQZBSemMywRWcEDNidvNbjGVyZu3nbZ8qq9ubiPoGLMdRDpfSrpkkm9BXYFkpKxxFX38o/76w=="
+		},
 		"ee-first": {
 			"version": "1.1.1",
 			"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@ -187,6 +286,21 @@
 			"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.8.0.tgz",
 			"integrity": "sha1-6qM9bd16zo9/b+DJygRA5wZzix4="
 		},
+		"is-windows": {
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz",
+			"integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA=="
+		},
+		"isarray": {
+			"version": "0.0.1",
+			"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
+			"integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
+		},
+		"isexe": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+			"integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA="
+		},
 		"media-typer": {
 			"version": "0.3.0",
 			"resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
@@ -220,16 +334,54 @@
 				"mime-db": "~1.37.0"
 			}
 		},
+		"minimist": {
+			"version": "0.0.8",
+			"resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
+			"integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
+		},
+		"mkdirp": {
+			"version": "0.5.1",
+			"resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
+			"integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
+			"requires": {
+				"minimist": "0.0.8"
+			}
+		},
 		"ms": {
 			"version": "2.0.0",
 			"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
 			"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
 		},
+		"multer": {
+			"version": "1.4.1",
+			"resolved": "https://registry.npmjs.org/multer/-/multer-1.4.1.tgz",
+			"integrity": "sha512-zzOLNRxzszwd+61JFuAo0fxdQfvku12aNJgnla0AQ+hHxFmfc/B7jBVuPr5Rmvu46Jze/iJrFpSOsD7afO8SDw==",
+			"requires": {
+				"append-field": "^1.0.0",
+				"busboy": "^0.2.11",
+				"concat-stream": "^1.5.2",
+				"mkdirp": "^0.5.1",
+				"object-assign": "^4.1.1",
+				"on-finished": "^2.3.0",
+				"type-is": "^1.6.4",
+				"xtend": "^4.0.0"
+			}
+		},
 		"negotiator": {
 			"version": "0.6.1",
 			"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz",
 			"integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk="
 		},
+		"nice-try": {
+			"version": "1.0.5",
+			"resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz",
+			"integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ=="
+		},
+		"object-assign": {
+			"version": "4.1.1",
+			"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
+			"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
+		},
 		"on-finished": {
 			"version": "2.3.0",
 			"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
@@ -243,11 +395,21 @@
 			"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz",
 			"integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M="
 		},
+		"path-key": {
+			"version": "2.0.1",
+			"resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz",
+			"integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A="
+		},
 		"path-to-regexp": {
 			"version": "0.1.7",
 			"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
 			"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
 		},
+		"process-nextick-args": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
+			"integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="
+		},
 		"proxy-addr": {
 			"version": "2.0.4",
 			"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz",
@@ -278,6 +440,17 @@
 				"unpipe": "1.0.0"
 			}
 		},
+		"readable-stream": {
+			"version": "1.1.14",
+			"resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
+			"integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=",
+			"requires": {
+				"core-util-is": "~1.0.0",
+				"inherits": "~2.0.1",
+				"isarray": "0.0.1",
+				"string_decoder": "~0.10.x"
+			}
+		},
 		"safe-buffer": {
 			"version": "5.1.2",
 			"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
@@ -288,6 +461,11 @@
 			"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
 			"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
 		},
+		"semver": {
+			"version": "5.6.0",
+			"resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz",
+			"integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg=="
+		},
 		"send": {
 			"version": "0.16.2",
 			"resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz",
@@ -324,11 +502,34 @@
 			"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz",
 			"integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ=="
 		},
+		"shebang-command": {
+			"version": "1.2.0",
+			"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz",
+			"integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=",
+			"requires": {
+				"shebang-regex": "^1.0.0"
+			}
+		},
+		"shebang-regex": {
+			"version": "1.0.0",
+			"resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz",
+			"integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM="
+		},
 		"statuses": {
 			"version": "1.4.0",
 			"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz",
 			"integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew=="
 		},
+		"streamsearch": {
+			"version": "0.1.2",
+			"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz",
+			"integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo="
+		},
+		"string_decoder": {
+			"version": "0.10.31",
+			"resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
+			"integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
+		},
 		"type-is": {
 			"version": "1.6.16",
 			"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz",
@@ -338,11 +539,21 @@
 				"mime-types": "~2.1.18"
 			}
 		},
+		"typedarray": {
+			"version": "0.0.6",
+			"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
+			"integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
+		},
 		"unpipe": {
 			"version": "1.0.0",
 			"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
 			"integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
 		},
+		"util-deprecate": {
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
+			"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
+		},
 		"utils-merge": {
 			"version": "1.0.1",
 			"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
@@ -352,6 +563,19 @@
 			"version": "1.1.2",
 			"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
 			"integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw="
+		},
+		"which": {
+			"version": "1.3.1",
+			"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
+			"integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
+			"requires": {
+				"isexe": "^2.0.0"
+			}
+		},
+		"xtend": {
+			"version": "4.0.1",
+			"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
+			"integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68="
 		}
 	}
 }

dashboard-api/package.json

@@ -3,10 +3,14 @@
   "private": true,
   "main": "server.js",
   "scripts": {
-    "start": "node server.js"
+    "start": "node server.js",
+    "develop": "cross-env PORT=3001 node server.js"
   },
   "dependencies": {
-    "express": "^4.16.3"
+    "cross-env": "^5.2.0",
+    "dotenv": "^6.2.0",
+    "express": "^4.16.3",
+    "multer": "^1.4.1"
   },
   "devDependencies": {},
   "license": "BSD-3-Clause"

dashboard-api/routes/catchAll.js

@@ -1,4 +1,3 @@
-const { indices, prs } = require('../data.json');
 const router = require('express').Router();
 
 router.get('/', (request, response) => {

dashboard-api/routes/getCurrData.js

@@ -1,8 +1,9 @@
-const data = require('../data.json');
 const router = require('express').Router();
 
+const container = require ('../data');
+
 router.get('/', (request, response) => {
-  response.json(data);
+  response.json(container.data);
 });
 
 module.exports = router;

dashboard-api/routes/index.js

@@ -4,5 +4,6 @@ const pr = require('./pr');
 const search = require('./search');
 const info = require('./info');
 const getCurrData = require('./getCurrData');
+const upload = require('./upload');
 
-module.exports = { catchAll, pareto, pr, search, info, getCurrData };
+module.exports = { catchAll, pareto, pr, search, info, getCurrData, upload };

dashboard-api/routes/info.js

@@ -1,9 +1,14 @@
-const { prs, startTime } = require('../data.json');
 const router = require('express').Router();
+const fs = require('fs');
+const path = require('path');
+
+const container = require ('../data');
 
-const firstPR = prs[0].number;
-const lastPR = prs[prs.length - 1].number;
 router.get('/', (request, response) => {
+  const { prs, startTime } = container.data;
+  const firstPR = prs[0].number;
+  const lastPR = prs[prs.length - 1].number;
+
   response.json({
     ok: true,
     lastUpdate: startTime,

dashboard-api/routes/pareto.js

@@ -1,30 +1,34 @@
 const router = require('express').Router();
 
-const { indices, prs } = require('../data.json');
+const container = require ('../data');
-
-const reportObj = prs.reduce((obj, pr) => {
-  const { number, filenames, username } = pr;
-  
-  filenames.forEach((filename) => {
-    if (obj[filename]) {
-      const { count, prs } = obj[filename];
-      obj[filename] = { count: count + 1, prs: prs.concat({ number, username } ) };
-    }
-    else {
-      obj[filename] = { count: 1, prs: [ { number, username } ] };
-    }
-  });
-  return obj;
-}, {});
-const pareto = Object.keys(reportObj)
-  .map((filename) => {
-    const { count, prs } = reportObj[filename];
-    return { filename, count, prs };
-  })
-  .sort((a, b) => b.count - a.count);
-
 
 router.get('/', (reqeust, response) => {
+  const { indices, prs } = container.data;
+
+  const reportObj = prs.reduce((obj, pr) => {
+    const { number, filenames, username, title } = pr;
+
+    filenames.forEach((filename) => {
+      if (obj[filename]) {
+        const { count, prs } = obj[filename];
+        obj[filename] = { count: count + 1, prs: prs.concat({ number, username, title } ) };
+      }
+      else {
+        obj[filename] = { count: 1, prs: [ { number, username, title } ] };
+      }
+    });
+    return obj;
+  }, {});
+  const pareto = Object.keys(reportObj)
+    .reduce((arr, filename) => {
+      const { count, prs } = reportObj[filename];
+      if (count > 1) {
+        arr.push({ filename, count, prs });
+      }
+      return arr;
+    }, [])
+    .sort((a, b) => b.count - a.count);
+
   response.json({ ok: true, pareto });  
 });
 

dashboard-api/routes/pr.js

@@ -1,12 +1,14 @@
-const { indices, prs } = require('../data.json');
 const router = require('express').Router();
 
+const container = require('../data');
+
 router.get('/:number', (request, response) => {
+  const { indices, prs } = container.data;
   const { number: refNumber } = request.params;
   const index = indices[refNumber];
 
-  if (!index) {
+  if (!index && index !== 0) {
-    response.json({ ok: true, message: 'Not a valid PR #.', results: [] });
+    response.json({ ok: true, message: 'Unable to find that open PR #.', results: [] });
     return;
   }
 
@@ -14,20 +16,20 @@ router.get('/:number', (request, response) => {
   const results = [];
   const { filenames: refFilenames } = pr;
 
-  prs.forEach(({ number, filenames, username }) => {
+  prs.forEach(({ number, filenames, username, title }) => {
     if (number != refNumber) {
       const matchedFilenames = filenames.filter((filename) => {
         return refFilenames.includes(filename);
       });
 
       if (matchedFilenames.length) {
-        results.push({ number, filenames: matchedFilenames, username });
+        results.push({ number, filenames: matchedFilenames, username, title });
       }
     }
   });
 
   if (!results.length) {
-    response.json({ ok: true, message: 'No matching results.', results: [] });
+    response.json({ ok: true, message: `No other open PRs with at least one filename which PR #${refNumber} has.`, results: [] });
     return;
   }
 

dashboard-api/routes/search.js

@@ -1,19 +1,22 @@
-
-const { indices, prs } = require('../data.json');
 const router = require('express').Router();
 
+const container = require ('../data');
+
 router.get('/', (request, response) => {
+  const { indices, prs } = container.data;
   const value  = request.query.value;
 
   if (value) {
     const filesFound = {};
 
-    prs.forEach(({ number, filenames }) => {
+    prs.forEach(({ number, filenames, username, title }) => {
       filenames.forEach((filename) => {
         if (filename.toLowerCase().includes(value.toLowerCase())) {
           const prObj = {
             number,
-            fileCount: prs[indices[number]].filenames.length
+            fileCount: prs[indices[number]].filenames.length,
+            username,
+            title
           };
 
           if (filesFound.hasOwnProperty(filename)) {

dashboard-api/routes/upload.js

@@ -0,0 +1,43 @@
+require('dotenv').config();
+const fs = require('fs');
+const path = require('path');
+const multer  = require('multer');
+const router = require('express').Router();
+
+const container = require('../data');
+
+const upload = multer({ dest: 'uploads' });
+
+router.post('/', upload.single('file'), (request, response) => {
+  const secret = process.env.UPLOAD_SECRET;
+  const { password } = request.query;
+  if (!secret) {
+    console.log('Environment variable for upload secret has not been set!');
+  }
+
+  if (!!secret && password === secret) {
+    const { file: { path: filePath } } = request;
+    const uploaded = path.resolve(__dirname, '../' + filePath);
+    const dest = path.resolve(__dirname, '../data.json');
+    const data = JSON.parse(fs.readFileSync(uploaded));
+    const { indices, prs } = data;
+    const dataOK = Object.keys(data).every((key) => {
+      return !!data[key];
+    });
+    const lengthsMatch = Object.keys(indices).length === prs.length;
+
+    if (dataOK && lengthsMatch) {
+      container.update(data);
+      fs.renameSync(uploaded, dest);
+    }
+    else {
+      const logPath = path.resolve(__dirname, '../log.txt');
+      const errorMsg = `Upload failed with ${uploaded}, dataOK: ${dataOK}, lengthsMatch: ${lengthsMatch}.`;
+      fs.appendFileSync(logPath, errorMsg);
+    }
+
+    response.send(dest);
+  }
+});
+
+module.exports = router;

dashboard-api/sample.env

@@ -0,0 +1,10 @@
+# Environment Config
+
+# store your secrets and config variables in here
+# only invited collaborators will be able to see your .env values
+
+# reference these in your code with process.env.UPLOAD_SECRET
+
+UPLOAD_SECRET='replace with upload secret'
+
+# note: .env is a shell file so there can't be spaces around =

dashboard-api/server.js

@@ -2,7 +2,7 @@ const fs = require('fs');
 const express = require('express');
 
 const app = express();
-const { catchAll, pareto, pr, search, info, getCurrData } = require('./routes');
+const { catchAll, pareto, pr, search, info, getCurrData, upload } = require('./routes');
 
 app.use(express.static('public'));
 app.use((request, response, next) => {
@@ -12,13 +12,12 @@ app.use((request, response, next) => {
   next();
 });
 
-app.get('/', (request, response) => response.sendFile(__dirname + '/views/index.html'));
-
 app.use('/pr', pr);
 app.use('/search', search);
 app.use('/pareto', pareto);
 app.use('/info', info);
 app.use('/getCurrData', getCurrData);
+app.use('/upload', upload);
 app.use('*', catchAll);
 
 const listener = app.listen(process.env.PORT, () => {

dashboard-client/package-lock.json

@@ -5530,13 +5530,11 @@
         },
         "balanced-match": {
           "version": "1.0.0",
-          "bundled": true,
+          "bundled": true
-          "optional": true
         },
         "brace-expansion": {
           "version": "1.1.11",
           "bundled": true,
-          "optional": true,
           "requires": {
             "balanced-match": "^1.0.0",
             "concat-map": "0.0.1"
@@ -5549,18 +5547,15 @@
         },
         "code-point-at": {
           "version": "1.1.0",
-          "bundled": true,
+          "bundled": true
-          "optional": true
         },
         "concat-map": {
           "version": "0.0.1",
-          "bundled": true,
+          "bundled": true
-          "optional": true
         },
         "console-control-strings": {
           "version": "1.1.0",
-          "bundled": true,
+          "bundled": true
-          "optional": true
         },
         "core-util-is": {
           "version": "1.0.2",
@@ -5663,8 +5658,7 @@
         },
         "inherits": {
           "version": "2.0.3",
-          "bundled": true,
+          "bundled": true
-          "optional": true
         },
         "ini": {
           "version": "1.3.5",
@@ -5674,7 +5668,6 @@
         "is-fullwidth-code-point": {
           "version": "1.0.0",
           "bundled": true,
-          "optional": true,
           "requires": {
             "number-is-nan": "^1.0.0"
           }
@@ -5687,20 +5680,17 @@
         "minimatch": {
           "version": "3.0.4",
           "bundled": true,
-          "optional": true,
           "requires": {
             "brace-expansion": "^1.1.7"
           }
         },
         "minimist": {
           "version": "0.0.8",
-          "bundled": true,
+          "bundled": true
-          "optional": true
         },
         "minipass": {
           "version": "2.2.4",
           "bundled": true,
-          "optional": true,
           "requires": {
             "safe-buffer": "^5.1.1",
             "yallist": "^3.0.0"
@@ -5717,7 +5707,6 @@
         "mkdirp": {
           "version": "0.5.1",
           "bundled": true,
-          "optional": true,
           "requires": {
             "minimist": "0.0.8"
           }
@@ -5790,8 +5779,7 @@
         },
         "number-is-nan": {
           "version": "1.0.1",
-          "bundled": true,
+          "bundled": true
-          "optional": true
         },
         "object-assign": {
           "version": "4.1.1",
@@ -5801,7 +5789,6 @@
         "once": {
           "version": "1.4.0",
           "bundled": true,
-          "optional": true,
           "requires": {
             "wrappy": "1"
           }
@@ -5907,7 +5894,6 @@
         "string-width": {
           "version": "1.0.2",
           "bundled": true,
-          "optional": true,
           "requires": {
             "code-point-at": "^1.0.0",
             "is-fullwidth-code-point": "^1.0.0",

dashboard-client/package.json

@@ -15,5 +15,11 @@
   },
   "eslintConfig": {
     "extends": "react-app"
-  }
+  },
+  "browserslist": [
+    ">0.2%",
+    "not dead",
+    "not ie <= 11",
+    "not op_mini all"
+  ]
 }

dashboard-client/public/index.html

@@ -14,12 +14,11 @@
       Notice the use of %PUBLIC_URL% in the tags above.
       It will be replaced with the URL of the `public` folder during the build.
       Only files inside the `public` folder can be referenced from the HTML.
-
       Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
       work correctly both with client-side routing and a non-root public URL.
       Learn how to configure a non-root public URL by running `npm run build`.
     -->
-    <title>React App</title>
+    <title>freeCodeCamp Moderator Tools</title>
   </head>
   <body>
     <noscript>
@@ -29,10 +28,8 @@
     <!--
       This HTML file is a template.
       If you open it directly in the browser, you will see an empty page.
-
       You can add webfonts, meta tags, or analytics to this file.
       The build step will place the bundled scripts into the <body> tag.
-
       To begin the development, run `npm start` or `yarn start`.
       To create a production bundle, use `npm run build` or `yarn build`.
     -->

dashboard-client/src/App.js

@@ -6,63 +6,86 @@ import Search from './components/Search';
 import Pareto from './components/Pareto';
 import Footer from './components/Footer';
 
+import { ENDPOINT_INFO } from './constants';
+
+console.log(ENDPOINT_INFO);
 const PageContainer = styled.div`
-  display: flex;
+   display: flex;
-  flex-direction: column;
+   flex-direction: column;
-  justify-content: center;
+   justify-content: center;
-  align-items: center;
+   align-items: center;
 `;
 
 const Container = styled.div`
-  display: flex;
+   display: flex;
-  flex-direction: column;
+   flex-direction: column;
-  justify-content: center;
+   justify-content: center;
-  align-items: center;
+   align-items: center;
-  max-width: 960px;
+   max-width: 960px;
-  width: 90vw;
+   width: 90vw;
-  padding: 15px;
+   padding: 15px;
-  border-radius: 4px;
+   border-radius: 4px;
-  box-shadow: 0 0 4px 0 #777;
+   box-shadow: 0 0 4px 0 #777;
 `;
 
 const Title = styled.h1`
-  display: flex;
+   display: flex;
-  justify-content: center;
+   justify-content: center;
-  align-items: center;
+   align-items: center;
-  background: ${({ theme }) => theme.primary};
+   background: ${({ theme }) => theme.primary};
-  color: white;
+   color: white;
-  width: 100%;
+   width: 100%;
-  padding: 3px;
+   padding: 3px;
 `;
 
 const imgStyle = {
-  paddingRight: '20px',
+   paddingRight: '20px',
-  paddingTop: '6px'
+   paddingTop: '6px'
 };
 
 class App extends Component {
-  state = {
+   state = {
-    view: 'search'
+     view: 'search',
-  };
+     footerInfo: null
+   };
 
-  handleViewChange = ( { target: { id } }) => {
+   updateInfo() {
-    const view = id.replace('tabs-', '');
+     fetch(ENDPOINT_INFO)
-    this.setState((prevState) => ({ ...this.clearObj, view }));
+     .then((response) => response.json())
-  }
+     .then(({ ok, numPRs, prRange, lastUpdate }) => {
+       if (ok) {
+       const footerInfo = { numPRs, prRange, lastUpdate };
+         this.setState((prevState) => ({ footerInfo }));
+       }
+     })
+     .catch(() => {
+       // do nothing
+     });
+   }
 
-  render() {
+   handleViewChange = ( { target: { id } }) => {
-    const { handleViewChange, state: { view } } = this;
+     const view = id.replace('tabs-', '');
-    return (
+     this.setState((prevState) => ({ ...this.clearObj, view }));
-      <PageContainer>
+     this.updateInfo();
-        <Title><img style={imgStyle} src="https://discourse-user-assets.s3.dualstack.us-east-1.amazonaws.com/original/3X/e/d/ed1c70bda321aaeee9e6c20ab650ce8bc34899fa.svg" alt="Free Code Camp Logo" /> Moderator Tools</Title>
+   }
-        <Tabs view={view} onViewChange={handleViewChange}/>
+
-        <Container>
+   componentDidMount() {
-          { view === 'search' && <Search /> }
+     this.updateInfo();
-          { view === 'reports' && <Pareto /> }
+   }
-        </Container>
+
-        <Footer />
+   render() {
-      </PageContainer>
+     const { handleViewChange, state: { view, footerInfo } } = this;
-    );
+     return (
-  }
+       <PageContainer>
+         <Title><img style={imgStyle} src="https://discourse-user-assets.s3.dualstack.us-east-1.amazonaws.com/original/3X/e/d/ed1c70bda321aaeee9e6c20ab650ce8bc34899fa.svg" alt="Free Code Camp Logo" /> Moderator Tools</Title>
+         <Tabs view={view} onViewChange={handleViewChange}/>
+         <Container>
+           { view === 'search' && <Search /> }
+           { view === 'reports' && <Pareto /> }
+         </Container>
+         { footerInfo && <Footer footerInfo={footerInfo}/> }
+       </PageContainer>
+     );
+   }
 }
 export default App;

dashboard-client/src/components/FilenameResults.js

@@ -1,41 +1,47 @@
 import React from 'react';
 import styled from 'styled-components';
 
+import ListItem from './ListItem';
+import FullWidthDiv from './FullWidthDiv';
+import Result from './Result';
+
 const List = styled.div`
   margin: 5px;
   display: flex;
-  flex-wrap: wrap;
+  flex-direction: column;
 `;
 
-const ListItem = styled.div`
+const filenameTitle = { fontWeight: '600' };
-  padding: 0 5px;
-`;
 
 const FilenameResults = ({ searchValue, results }) => {
   const elements = results.map((result) => {
     const { filename, prs: prObjects } = result;
-    const prs = prObjects.map(({ number }, index) => {
+    const prs = prObjects.map(({ number, username, title }, index) => {
-      const prUrl = `https://github.com/freeCodeCamp/freeCodeCamp/pull/${number}`;
+      return (
-      return <ListItem key={`${filename}-${index}`}>
+        <ListItem
-        • <a href={prUrl} rel="noopener noreferrer" target="_blank">{number}</a>
+          number={number}
-      </ListItem>;
+          username={username}
+          prTitle={title}
+        />
+      );
     });
 
+    const fileOnMaster = `https://github.com/freeCodeCamp/freeCodeCamp/blob/master/${filename}`;
     return (
-      <div key={filename}>
+      <Result key={filename}>
-        {filename}
+        <span style={filenameTitle}>{filename}</span> <a href={fileOnMaster} rel="noopener noreferrer" target="_blank">(File on Master)</a>
         <List>
           {prs}
         </List>
-      </div>
+      </Result>
     );
   });
 
   return (
-    <div>
+    <FullWidthDiv>
       {results.length ? <h3>Results for: {searchValue}</h3> : null}
       {elements}
-    </div>
+    </FullWidthDiv>
   );
 };
 

dashboard-client/src/components/Footer.js

@@ -1,52 +1,32 @@
-import React, { Component } from 'react';
+import React from 'react';
 import styled from 'styled-components';
 
-import { ENDPOINT_INFO } from '../constants';
-
 const Container = styled.div`
-  margin-top: 5px;
+   margin-top: 5px;
-  text-align: center;
+   text-align: center;
 `;
 
 const Info = styled.div`
-  font-size: 14px;
+   font-size: 14px;
-  padding: 2px;
+   padding: 2px;
 `;
 
-class Footer extends Component {
+const Footer = (props) => {
-  state = {
-    numPRs: null,
-    prRange: null,
-    lastUpdate: null
-  }
 
-  componentDidMount() {
+   const localTime = (lastUpdate) => {
-    fetch(ENDPOINT_INFO)
+     const newTime = new Date(lastUpdate);
-      .then((response) => response.json())
+     return newTime.toLocaleString();
-      .then(({ ok, numPRs, prRange, lastUpdate }) => {
+   }
-        if (ok) {
-          this.setState((prevState) => ({ numPRs, prRange, lastUpdate }));
-        }
-      })
-      .catch(() => {
-        // do nothing
-      });
-  }
 
-  localTime(lastUpdate) {
+   const { footerInfo: { numPRs, prRange, lastUpdate } } = props;
-    const newTime = new Date(lastUpdate);
+   return (
-    return newTime.toLocaleString();
+     lastUpdate &&
-  }
+     <Container>
+       <Info>Last Update: {localTime(lastUpdate)}</Info>
+       <Info># of open PRs: {numPRs} ({prRange})</Info>
+     </Container>
+   );
 
-  render() {
-    const { numPRs, prRange, lastUpdate } = this.state;
-    return (
-      <Container>
-        <Info>Last Update: {this.localTime(lastUpdate)}</Info>
-        <Info># of open PRs: {numPRs} ({prRange})</Info>
-      </Container>
-    );
-  }
 };
 
 export default Footer;

dashboard-client/src/components/FullWidthDiv.js

@@ -0,0 +1,7 @@
+import styled from 'styled-components';
+
+const FullWidthDiv = styled.div`
+  width: 100%;
+`;
+
+export default FullWidthDiv;

dashboard-client/src/components/ListItem.js

@@ -0,0 +1,28 @@
+import React from 'react';
+import styled from 'styled-components';
+
+const Container = styled.div`
+  display: flex;
+  justify-content: space-between;
+  flex-direction: row;
+  overflow: hidden;
+`;
+
+const prNumStyle = { flex: 1 };
+const usernameStyle = { flex: 1 };
+const titleStyle = { flex: 3 };
+
+const ListItem = ({ number, username, prTitle: title }) => {
+  const prUrl = `https://github.com/freeCodeCamp/freeCodeCamp/pull/${number}`;
+  return (
+    <Container>
+      <a style={prNumStyle} href={prUrl} rel="noopener noreferrer" target="_blank">
+        #{number}
+      </a>
+      <span style={usernameStyle}>{username}</span>
+      <span style={titleStyle}>{title}</span>
+    </Container>
+  );
+};
+
+export default ListItem;

dashboard-client/src/components/Pareto.js

@@ -1,30 +1,18 @@
 import React from 'react';
 import styled from 'styled-components';
 
+import ListItem from './ListItem';
+import FullWidthDiv from './FullWidthDiv';
+import Result from './Result';
 import { ENDPOINT_PARETO } from '../constants';
 
-const Result = styled.div`
-  word-wrap: break-word;
-  margin: 10px 0;
-  &:nth-child(odd) {
-    background: #eee;
-  }
-  padding: 3px;
-`;
-
 const List = styled.div`
   margin: 5px;
   display: flex;
-  flex-wrap: wrap;
+  flex-direction: column;
-`;
-
-const ListItem = styled.a`
-  flex-basis: 33%;
-  overflow: hidden;
 `;
 
 const detailsStyle = { padding: '3px' };
-
 const filenameTitle = { fontWeight: '600' };
 
 class Pareto extends React.Component {
@@ -53,17 +41,19 @@ class Pareto extends React.Component {
     const { data } = this.state;
     const elements = data.map((entry) => {
       const { filename, count, prs } = entry;
-      const prsList = prs.map(({ number, username }) => {
+      const prsList = prs.map(({ number, username, title }) => {
-        const prUrl = `https://github.com/freeCodeCamp/freeCodeCamp/pull/${number}`;
         return (
-          <ListItem href={prUrl} rel="noopener noreferrer" target="_blank">
+          <ListItem
-            #{number} <span>({username})</span>
+            number={number}
-          </ListItem>
+            username={username}
+            prTitle={title}
+          />
         )
       });
+      const fileOnMaster = `https://github.com/freeCodeCamp/freeCodeCamp/blob/master/${filename}`;
       return (
         <Result key={filename}>
-          <span style={filenameTitle}>{filename}</span><br />
+          <span style={filenameTitle}>{filename}</span> <a href={fileOnMaster} rel="noopener noreferrer" target="_blank">(File on Master)</a><br />
           <details style={detailsStyle}>
             <summary># of PRs: {count}</summary>
             <List>{prsList}</List>
@@ -73,9 +63,9 @@ class Pareto extends React.Component {
     });
 
     return (
-      <div>
+      <FullWidthDiv>
         {data.length ? elements : 'Report Loading...'}
-      </div>
+      </FullWidthDiv>
     );
   }
 }

dashboard-client/src/components/PrResults.js

@@ -1,9 +1,9 @@
 import React from 'react';
 import styled from 'styled-components';
 
-const Container = styled.div`
+import ListItem from './ListItem';
-  margin-bottom: 15px;
+import FullWidthDiv from './FullWidthDiv';
-`;
+import Result from './Result';
 
 const List = styled.ul`
   margin: 3px;
@@ -11,33 +11,35 @@ const List = styled.ul`
 
 const PrResults = ({ searchValue, results }) => {
   const elements = results.map((result, idx) => {
-    const { number, filenames, username } = result;
+    const { number, filenames, username, title } = result;
     const files = filenames.map((filename, index) => {
-      return <li key={`${number}-${index}`}>{filename}</li>;
+      const fileOnMaster = `https://github.com/freeCodeCamp/freeCodeCamp/blob/master/${filename}`;
+      return (
+        <li key={`${number}-${index}`}>
+          {filename} <a href={fileOnMaster} rel="noopener noreferrer" target="_blank">(File on Master)</a>
+        </li>
+      );
     });
-    const prUrl = `https://github.com/freeCodeCamp/freeCodeCamp/pull/${number}`
 
     return (
-      <Container key={`${number}-${idx}`}>
+      <Result key={`${number}-${idx}`}>
-        {!Number(number)
+        <ListItem
-          ? number
+          number={number}
-          : <>
+          username={username}
-              <a href={prUrl} rel="noopener noreferrer" target="_blank">{number}</a>
+          prTitle={title}
-              <span>&nbsp;{username}</span>
+        />
-            </>
-        }
         <List>
           {files}
         </List>
-      </Container>
+      </Result>
     );
   });
 
   return (
-    <div>
+    <FullWidthDiv style={{width: '100%'}}>
       {results.length ? <h3>Results for PR# {searchValue}</h3> : null}
       {elements}
-    </div>
+    </FullWidthDiv>
   );
 };
 

dashboard-client/src/components/Result.js

@@ -0,0 +1,12 @@
+import styled from 'styled-components';
+
+const Result = styled.div`
+  border: 1px solid #aaa;
+  margin: 10px 0;
+  &:nth-child(odd) {
+    background: #eee;
+  }
+  padding: 3px;
+`;
+
+export default Result;

dashboard-client/src/constants/index.js

@@ -1,5 +1,5 @@
 const API_HOST = !!process.env.REACT_APP_DEV ?
-  'http://localhost:3001/' :
+  'http://localhost:3001' :
   'https://pr-relations.glitch.me';
 const ENDPOINT_INFO = API_HOST + '/info';
 const ENDPOINT_PARETO = API_HOST + '/pareto';

dashboard-client/src/index.css

@@ -21,6 +21,11 @@ a {
   color: #006400;
 }
 
+a:visited {
+  text-decoration: none;
+  color: purple;
+}
+
 code {
   font-family: source-code-pro, Menlo, Monaco, Consolas, "Courier New",
     monospace;

sweeper/get-prs/index.js

@@ -87,7 +87,8 @@ const getUserInput = async (rangeType = '') => {
 
 const getPRs = async (totalPRs, firstPR, lastPR, prPropsToGet) => {
   const getPRsBar = new _cliProgress.Bar({
-    format: `Part 1 of 2: Retrieving PRs (${firstPR}-${lastPR}) [{bar}] {percentage}% - {duration_formatted}`
+    format: `Retrieve PRs (${firstPR}-${lastPR}) [{bar}] {percentage}% | Elapsed Time: {duration_formatted} | ETA: {eta_formatted}`,
+    etaBuffer: 50
   }, _cliProgress.Presets.shades_classic);
   getPRsBar.start(totalPRs, 0);
   let openPRs = await paginate(octokit.pullRequests.list, octokit, firstPR, lastPR, prPropsToGet, getPRsBar);

sweeper/one-off-scripts/add-comment-on-frontmatter-issues.js

@@ -84,10 +84,11 @@ const guideFolderChecks = async (number, prFiles, user) => {
 };
 
 (async () => {
-  const { firstPR, lastPR } = await getUserInput();
+  const { totalPRs, firstPR, lastPR } = await getUserInput();
-  log.setFirstLast({ firstPR, lastPR });
+  // log.setFirstLast({ firstPR, lastPR });
+  console.log(firstPR, lastPR);
   const prPropsToGet = ['number', 'labels', 'user'];
-  const { openPRs } = await getPRs(firstPR, lastPR, prPropsToGet);
+  const { openPRs } = await getPRs(totalPRs, firstPR, lastPR, prPropsToGet);
 
   if (openPRs.length) {
     savePrData(openPRs, firstPR, lastPR);
@@ -103,7 +104,7 @@ const guideFolderChecks = async (number, prFiles, user) => {
       const labelsAdded = await labeler(number, prFiles, currentLabels, guideFolderErrorsComment);
       const labelLogVal = labelsAdded.length ? labelsAdded : 'none added';
 
-      log.add(number, { comment: commentLogVal, labels: labelLogVal });
+      log.add(number, { number, comment: commentLogVal, labels: labelLogVal });
       await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500);
     }
   }

sweeper/one-off-scripts/add-test-locally-label.js

@@ -18,10 +18,9 @@ octokit.authenticate(octokitAuth);
 const log = new ProcessingLog('all-locally-tested-labels');
 
 (async () => {
-  const { firstPR, lastPR } = await getUserInput();
+  const { totalPRs, firstPR, lastPR } = await getUserInput();
-  log.setFirstLast({ firstPR, lastPR });
   const prPropsToGet = ['number', 'labels'];
-  const { openPRs } = await getPRs(firstPR, lastPR, prPropsToGet);
+  const { openPRs } = await getPRs(totalPRs, firstPR, lastPR, prPropsToGet);
 
   if (openPRs.length) {
     savePrData(openPRs, firstPR, lastPR);
@@ -45,7 +44,7 @@ const log = new ProcessingLog('all-locally-tested-labels');
         }
       }
       else {
-        log.add(number, { labels: 'none added' });
+        log.add(number, { number, labels: 'none added' });
       }
     }
   }

sweeper/one-off-scripts/close-open-specific-failures.js

@@ -27,17 +27,16 @@ const getUserInput = async () => {
 .then(async (prs) => {
   const firstPR = prs[0].number;
   const lastPR = prs[prs.length - 1].number;
-  log.setFirstLast({ firstPR, lastPR });
+  for (let { number, errorDesc } of prs) {
-  for (let { number, data: { errorDesc } } of prs) {
     if (errorDesc !== "unknown error") {
-      log.add(number, { closedOpened: true, errorDesc });
+      log.add(number, { number, closedOpened: true, errorDesc });
       if (process.env.PRODUCTION_RUN === 'true') {
          await closeOpen(number);
          await rateLimiter(90000);
       }
     }
     else {
-      log.add(number, { closedOpened: false, errorDesc });
+      log.add(number, { number, closedOpened: false, errorDesc });
     }
   }
 })

sweeper/one-off-scripts/find-failures.js

@@ -22,10 +22,9 @@ const log = new ProcessingLog('find-failures-script');
 const errorsToFind = require(path.resolve(__dirname, '../input-files/failuresToFind.json'));
 
 (async () => {
-  const { firstPR, lastPR } = await getUserInput();
+  const { totalPRs, firstPR, lastPR } = await getUserInput();
-  log.setFirstLast({ firstPR, lastPR });
   const prPropsToGet = ['number', 'labels', 'head'];
-  const { openPRs } = await getPRs(firstPR, lastPR, prPropsToGet);
+  const { openPRs } = await getPRs(totalPRs, firstPR, lastPR, prPropsToGet);
 
   if (openPRs.length) {
     savePrData(openPRs, firstPR, lastPR);
@@ -58,7 +57,7 @@ const errorsToFind = require(path.resolve(__dirname, '../input-files/failuresToF
               }
             }
             const errorDesc = error ? error : 'unknown error';
-            log.add(number, { errorDesc, buildLog: travisLogUrl });
+            log.add(number, { number, errorDesc, buildLog: travisLogUrl });
           }
         }
       }

sweeper/one-off-scripts/get-changes-for-pr-relations-data.js

@@ -1,134 +0,0 @@
-require('dotenv').config({ path: '../.env' });
-const formatDate = require('date-fns/format');
-const path = require('path');
-const fs = require('fs');
-const _cliProgress = require('cli-progress');
-const fetch = require('node-fetch');
-
-const { saveToFile } = require('../utils/save-to-file');
-
-class Log {
-  constructor() {
-    this._startTime = null;
-    this._finishTime = null;
-    this._elapsedTime = null;
-    this._prsArr = [];
-    this._indicesObj = {};
-    this._logfile = path.resolve(__dirname, `../work-logs/pr-relations.json`);
-  }
-
-  export() {
-    const log = {
-      startTime: this._startTime,
-      finishTime: this._finishTime,
-      elapsedTime: this._elapsedTime,
-      indices: this._indicesObj,
-      prs: this._prsArr
-    };
-    saveToFile(this._logfile, JSON.stringify(log))
-  }
-
-  getPrRange() {
-    const first = this._prsArr[0].number;
-    const last = this._prsArr[this._prsArr.length -1].number;
-    return [first, last];
-  }
-
-  add(prNum, props) {
-    this._prsArr.push(props);
-    this._indicesObj[prNum] = this._prsArr.length -1;
-  }
-
-  start() {
-    this._startTime = new Date();
-    this.export();
-  }
-
-  finish() {
-    this._finishTime = new Date();
-    const minutesElapsed = (this._finishTime - this._startTime) / 1000 / 60;
-    this._elapsedTime =  minutesElapsed.toFixed(2) + ' mins';
-    this.export();
-    this.changeFilename(this.getPrRange());
-  }
-
-  changeFilename( [first, last] ) {
-    const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss');
-    const newFilename = path.resolve(__dirname,`../work-logs/pr-relations_${first}-${last}_${now}.json`);
-    fs.rename(this._logfile, newFilename, function(err) {
-      if (err) {
-        throw('ERROR: ' + err);
-      }
-    });
-  }
-};
-
-const getExistingData = async () => {
-  const url = `https://pr-relations.glitch.me/getCurrData`;
-  const response = await fetch(url);
-  const data = await response.json();
-  return data;
-};
-
-const { owner, repo, octokitConfig, octokitAuth } = require('../constants');
-const octokit = require('@octokit/rest')(octokitConfig);
-
-const { getPRs, getUserInput } = require('../get-prs');
-const { rateLimiter, savePrData } = require('../utils');
-
-octokit.authenticate(octokitAuth);
-
-const log = new Log();
-(async () => {
-  const { totalPRs, firstPR, lastPR } = await getUserInput('all');
-  log.start();
-  const prPropsToGet = ['number', 'user', 'updated_at', 'files'];
-  const { openPRs } = await getPRs(totalPRs, firstPR, lastPR, prPropsToGet);
-
-  if (openPRs.length) {
-   const { indices: oldIndices, prs: oldPRs } = await getExistingData();
-
-    const getFilesBar = new _cliProgress.Bar({
-      format: `Part 2 of 2: Adding/Updating PRs [{bar}] {percentage}% | {value}/{total} | {duration_formatted}`
-    }, _cliProgress.Presets.shades_classic);
-    getFilesBar.start(openPRs.length, 0);
-
-    let newOrUpdated = '';
-    for (let count in openPRs) {
-      let { number, updated_at, user: { login: username } } = openPRs[count];
-      let oldUpdated_at;
-      let oldPrData = oldPRs[oldIndices[number]];
-      if (oldPrData) {
-        oldUpdated_at = oldPrData.updated_at;
-      }
-
-      if (!oldIndices.hasOwnProperty(number) || updated_at > oldUpdated_at) {
-        newOrUpdated += `PR #${number} was new or needed updating\n`;
-        const { data: prFiles } = await octokit.pullRequests.listFiles({ owner, repo, number });
-        const filenames = prFiles.map(({ filename }) => filename);
-        log.add(number, { number, updated_at, username, filenames });
-        if (+count > 3000 ) {
-          await rateLimiter(1400);
-        }
-      }
-      else {
-        let { username: oldUsername, filenames: oldFilenames } = oldPrData;
-        log.add(number, { number, updated_at: oldUpdated_at, username: oldUsername, filenames: oldFilenames });
-      }
-      if (+count % 10 === 0) {
-        getFilesBar.update(+count);
-      }
-    }
-    getFilesBar.update(openPRs.length);
-    getFilesBar.stop();
-    console.log(newOrUpdated);
-  }
-})()
-.then(() => {
-  log.finish();
-  console.log('Finished retrieving pr-relations data');
-})
-.catch(err => {
-  log.finish();
-  console.log(err)
-})

sweeper/one-off-scripts/get-pr-relations-data.js

@@ -1,108 +1,99 @@
-require('dotenv').config({ path: '../.env' });
-const formatDate = require('date-fns/format');
 const path = require('path');
+// require('dotenv').config({ path: '../.env' });
+require('dotenv').config();
+const formatDate = require('date-fns/format');
 const fs = require('fs');
 const _cliProgress = require('cli-progress');
+const fetch = require('node-fetch');
+const FormData = require('form-data');
 
-const { saveToFile } = require('../utils/save-to-file');
+const HOST = process.env.NODE_DEV
+  ? process.env.LOCAL_HOST
+  : process.env.GLITCH_API_URL;
 
-class Log {
+const getExistingData = async () => {
-  constructor() {
+  const url = `${HOST}/getCurrData`;
-    this._startTime = null;
+  const response = await fetch(url);
-    this._finishTime = null;
+  const data = await response.json();
-    this._elapsedTime = null;
+  return data ? data : { indices: {}, prs: [] };
-    this._prsArr = [];
-    this._indicesObj = {};
-    this._logfile = path.resolve(__dirname, `../work-logs/pr-relations.json`);
-  }
-
-  export() {
-    const log = {
-      startTime: this._startTime,
-      finishTime: this._finishTime,
-      elapsedTime: this._elapsedTime,
-      indices: this._indicesObj,
-      prs: this._prsArr
-    };
-    saveToFile(this._logfile, JSON.stringify(log))
-  }
-
-  getPrRange() {
-    const first = this._prsArr[0].number;
-    const last = this._prsArr[this._prsArr.length -1].number;
-    return [first, last];
-    // return [null, null]
-  }
-
-  add(prNum, props) {
-    this._prsArr.push(props);
-    this._indicesObj[prNum] = this._prsArr.length -1;
-  }
-
-  start() {
-    this._startTime = new Date();
-    this.export();
-  }
-
-  finish() {
-    this._finishTime = new Date();
-    const minutesElapsed = (this._finishTime - this._startTime) / 1000 / 60;
-    this._elapsedTime =  minutesElapsed.toFixed(2) + ' mins';
-    this.export();
-    this.changeFilename(this.getPrRange());
-  }
-
-  changeFilename( [first, last] ) {
-    const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss');
-    const newFilename = path.resolve(__dirname,`../work-logs/pr-relations_${first}-${last}_${now}.json`);
-    fs.rename(this._logfile, newFilename, function(err) {
-      if (err) {
-        throw('ERROR: ' + err);
-      }
-    });
-  }
 };
 
 const { owner, repo, octokitConfig, octokitAuth } = require('../constants');
-
 const octokit = require('@octokit/rest')(octokitConfig);
 
 const { getPRs, getUserInput } = require('../get-prs');
-const { rateLimiter, savePrData } = require('../utils');
+const { rateLimiter, savePrData, ProcessingLog } = require('../utils');
 
 octokit.authenticate(octokitAuth);
 
-const log = new Log();
+const log = new ProcessingLog('pr-relations');
 (async () => {
   const { totalPRs, firstPR, lastPR } = await getUserInput('all');
-  const prPropsToGet = ['number', 'user', 'updated_at', 'files'];
+  log.start();
+  const { indices: oldIndices, prs: oldPRs } = await getExistingData();
+  if (!oldPRs.length) {
+     console.log('No existing PRs data found, so it will take a while to download PRs/filenames data.');
+  }
+  const prPropsToGet = ['number', 'user', 'title', 'updated_at', 'files'];
   const { openPRs } = await getPRs(totalPRs, firstPR, lastPR, prPropsToGet);
 
   if (openPRs.length) {
-    log.start();
     const getFilesBar = new _cliProgress.Bar({
-      format: `Part 2 of 2: Retrieving filenames [{bar}] {percentage}% | {value}/{total}`
+      format: `Update PRs [{bar}] {percentage}% | {value}/{total} | Time Elapsed: {duration_formatted} | ETA: {eta_formatted}`,
+      etaBuffer: 50
     }, _cliProgress.Presets.shades_classic);
     getFilesBar.start(openPRs.length, 0);
+
+    let prsUpdated = '';
     for (let count in openPRs) {
-      let { number, updated_at, user: { login: username } } = openPRs[count];
+      let { number, updated_at, title, user: { login: username } } = openPRs[count];
-      const { data: prFiles } = await octokit.pullRequests.listFiles({ owner, repo, number });
+      let oldUpdated_at;
-      const filenames = prFiles.map(({ filename }) => filename);
+      let oldPrData = oldPRs[oldIndices[number]];
-      log.add(number, { number, updated_at, username, filenames });
+      if (oldPrData) {
-      if (+count > 3000 ) {
+        oldUpdated_at = oldPrData.updated_at;
-        await rateLimiter(1500);
       }
+      if (!oldIndices.hasOwnProperty(number) || updated_at > oldUpdated_at) {
+        const { data: prFiles } = await octokit.pullRequests.listFiles({ owner, repo, number });
+        const filenames = prFiles.map(({ filename }) => filename);
+        log.add(number, { number, updated_at, title, username, filenames });
+        if (+count > 3000 ) {
+          await rateLimiter(1400);
+        }
+        prsUpdated += `PR #${number} added or updated\n`;
+      }
+      else {
+        let { username: oldUsername, title: oldTitle, filenames: oldFilenames } = oldPrData;
+        log.add(number, { number, updated_at: oldUpdated_at, username: oldUsername, title: oldTitle, filenames: oldFilenames });
+      }
+
       if (+count % 10 === 0) {
         getFilesBar.update(+count);
       }
     }
     getFilesBar.update(openPRs.length);
     getFilesBar.stop();
+    console.log(prsUpdated);
+  }
+  else {
+    throw 'There were no open PRs received from Github';
   }
 })()
-.then(() => {
+.then(async () => {
   log.finish();
-  console.log('Finished retrieving pr-relations data');
+  console.log('Finished retrieving Dashboard data');
+
+  const formData = new FormData();
+  formData.append('file', fs.createReadStream(log._logfile));
+  const result = await fetch(`${HOST}/upload?password=${process.env.GLITCH_UPLOAD_SECRET}`, {
+    method: 'POST',
+    body: formData
+  })
+  .then(() => {
+    console.log(`Finished uploading data for ${HOST}`);
+  })
+  .catch((err) => {
+    console.log(err);
+  });
 })
 .catch(err => {
   log.finish();

sweeper/one-off-scripts/get-unknown-repo-prs.js

@@ -0,0 +1,45 @@
+const path = require('path');
+require('dotenv').config({ path: '../.env' });
+const formatDate = require('date-fns/format');
+const fs = require('fs');
+const _cliProgress = require('cli-progress');
+
+const { owner, repo, octokitConfig, octokitAuth } = require('../constants');
+const octokit = require('@octokit/rest')(octokitConfig);
+
+const { getPRs, getUserInput } = require('../get-prs');
+const { ProcessingLog, rateLimiter } = require('../utils');
+
+octokit.authenticate(octokitAuth);
+
+const log = new ProcessingLog('unknown-repo-prs-with-merge-conflicts');
+log.start();
+(async () => {
+  const { totalPRs, firstPR, lastPR } = await getUserInput('all');
+  const prPropsToGet = ['number', 'user', 'head'];
+  const { openPRs } = await getPRs(totalPRs, firstPR, lastPR, prPropsToGet);
+  if (openPRs.length) {
+    for (let count in openPRs) {
+      let { number, head: { repo: headRepo } } = openPRs[count];
+      if (headRepo === null) {
+        const { data: { mergeable, mergeable_state } } = await octokit.pullRequests.get({ owner, repo, number });
+        if (mergeable_state === 'dirty' || mergeable_state === 'unknown') {
+          log.add(number, { number, mergeable_state });
+          console.log(`[${number} (mergeable_state: ${mergeable_state})](https://github.com/freeCodeCamp/freeCodeCamp/pull/${number})`);
+        }
+        await rateLimiter(1000);
+      }
+    }
+  }
+  else {
+    throw 'There were no open PRs received from Github';
+  }
+})()
+.then(async () => {
+  log.finish();
+  console.log('Finished finding unknown repo PRs with merge conflicts');
+})
+.catch(err => {
+  log.finish();
+  console.log(err)
+})

sweeper/package.json

@@ -8,6 +8,7 @@
     "date-fns": "^1.29.0",
     "dedent": "^0.7.0",
     "dotenv": "^6.1.0",
+    "form-data": "^2.3.3",
     "gray-matter": "^4.0.1",
     "lodash": "^4.17.11",
     "path": "^0.12.7",

sweeper/pr-tasks/labeler.js

@@ -1,4 +1,5 @@
 const { validLabels } = require('../validation');
+
 const { addLabels } = require('./add-labels');
 const { rateLimiter } = require('../utils');
 
@@ -13,7 +14,6 @@ const labeler = async (number, prFiles, currentLabels, guideFolderErrorsComment)
     const filenameReplacement = filename.replace(/^curriculum\/challenges\//, 'curriculum\/');
     const regex = /^(docs|curriculum|guide)(?:\/)(arabic|chinese|portuguese|russian|spanish)?\/?/
     const [ _, articleType, language ] = filenameReplacement.match(regex) || []; // need an array to pass to labelsAdder
-
     if (articleType && validLabels[articleType]) {
       labelsToAdd[validLabels[articleType]] = 1
     }

sweeper/sample.env

@@ -5,3 +5,6 @@ REPOSITORY_OWNER='freeCodeCamp-rocks'
 REPOSITORY='freeCodeCamp'
 RATELIMIT_INTERVAL=1500
 PRODUCTION_RUN=false
+LOCAL_HOST='http://localhost:3001'
+GLITCH_API_URL='https://pr-relations.glitch.me'
+GLITCH_UPLOAD_SECRET='replace this with secret code'

sweeper/sweeper.js

@@ -14,7 +14,7 @@ const { owner, repo, octokitConfig, octokitAuth } = require('./constants');
 const octokit = require('@octokit/rest')(octokitConfig);
 
 const { getPRs, getUserInput } = require('./get-prs');
-const { guideFolderChecks } = require('./validation');
+ const { guideFolderChecks } = require('./validation');
 const { savePrData, ProcessingLog, rateLimiter } = require('./utils');
 const { labeler } = require('./pr-tasks');
 
@@ -25,13 +25,11 @@ const log = new ProcessingLog('sweeper');
 log.start();
 console.log('Sweeper started...');
 (async () => {
-  const { firstPR, lastPR } = await getUserInput();
+  const { totalPRs, firstPR, lastPR } = await getUserInput();
-  log.setFirstLast({ firstPR, lastPR });
   const prPropsToGet = ['number', 'labels', 'user'];
-  const { openPRs } = await getPRs(firstPR, lastPR, prPropsToGet);
+  const { openPRs } = await getPRs(totalPRs, firstPR, lastPR, prPropsToGet);
 
   if (openPRs.length) {
-    savePrData(openPRs, firstPR, lastPR);
     console.log('Processing PRs...');
     for (let count in openPRs) {
       let { number, labels: currentLabels, user: { login: username } } = openPRs[count];
@@ -43,7 +41,7 @@ console.log('Sweeper started...');
       const labelsAdded = await labeler(number, prFiles, currentLabels, guideFolderErrorsComment);
       const labelLogVal = labelsAdded.length ? labelsAdded : 'none added';
 
-      log.add(number, { comment: commentLogVal, labels: labelLogVal });
+      log.add(number, { number, comment: commentLogVal, labels: labelLogVal });
       await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500);
     }
   }

sweeper/utils/processing-log.js

@@ -8,10 +8,11 @@ const { saveToFile } = require('./save-to-file');
 class ProcessingLog {
   constructor(script) {
     this._script = script;
-    this._start = null;
+    this._startTime = null;
-    this._finish = null;
+    this._finishTime = null;
     this._elapsedTime = null;
-    this._prs = {};
+    this._prs = [];
+    this._indicesObj = {};
     this._prCount = null;
     this._logfile = path.resolve(__dirname, `../work-logs/data-for_${this.getRunType()}_${this._script}.json`);
   }
@@ -21,51 +22,59 @@ class ProcessingLog {
   }
 
   export() {
-    let sortedPRs = Object.keys(this._prs)
-     .sort((a, b) => a - b)
-     .map(num => ({ number: num, data: this._prs[num] }));
-
     const log = {
-      start: this._start,
+      startTime: this._startTime,
-      finish: this._finish,
+      finishTime: this._finishTime,
       elapsedTime: this._elapsedTime,
-      prCount: sortedPRs.length,
+      prCount: this._prs.length,
       firstPR: this._firstPR,
       lastPR: this._lastPR,
-      prs: sortedPRs
+      indices: this._indicesObj,
+      prs: this._prs
     };
     saveToFile(this._logfile, JSON.stringify(log))
   }
 
   add(prNum, props) {
-    this._prs[prNum] = props;
+    this._prs.push(props);
+    this._indicesObj[prNum] = this._prs.length -1;
   }
 
-  setFirstLast({ firstPR, lastPR }) {
+  getPrRange() {
-    this._firstPR = firstPR;
+    if (this._prs.length) {
-    this._lastPR = lastPR;
+      const first = this._prs[0].number;
+      const last = this._prs[this._prs.length -1].number;
+      return [first, last];
+    }
+    console.log('Current log file does not contain any PRs');
+    return [null, null];
   }
 
   start() {
-    this._start = new Date();
+    this._startTime = new Date();
+    this.export();
   }
 
   finish() {
-    this._finish = new Date();
+    this._finishTime = new Date();
-    const minutesElapsed = (this._finish - this._start) / 1000 / 60;
+    const minutesElapsed = (this._finishTime - this._startTime) / 1000 / 60;
     this._elapsedTime =  minutesElapsed.toFixed(2) + ' mins';
+    let [ first, last ] = this.getPrRange();
+    this._firstPR = first;
+    this._lastPR = last;
     this.export();
     this.changeFilename();
   }
 
   changeFilename() {
     const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss');
-    const newFilename = path.resolve(__dirname,`../work-logs/${this.getRunType()}_${this._script}_${this._firstPR}-${this._lastPR}_${now}.json`);
+    const finalFilename = `${this.getRunType()}_${this._script}_${this._firstPR}-${this._lastPR}_${now}.json`;
-    fs.rename(this._logfile, newFilename, function(err) {
+    const newFilename = path.resolve(__dirname,`../work-logs/${finalFilename}`);
-      if (err) {
+    fs.renameSync(this._logfile, newFilename);
-        throw(err);
+    if (!fs.existsSync(newFilename)) {
-      }
+      throw `File rename unsuccessful.`;
-    });
+    }
+    this._logfile = newFilename;
   }
 };
 

sweeper/validation/guide-folder-checks/index.js

@@ -1,6 +1,6 @@
 const fetch = require('node-fetch');
 
-const { addComment } = require('../../pr-tasks');
+const { addComment } = require('../../pr-tasks/add-comment');
 const { rateLimiter } = require('../../utils');
 const { createErrorMsg } = require('./create-error-msg');
 const { checkPath } = require('./check-path');

sweeper/validation/index.js

@@ -1,4 +1,3 @@
 const { validLabels } = require('./valid-labels');
 const { guideFolderChecks } = require('./guide-folder-checks');
-
 module.exports = { validLabels, guideFolderChecks };