feat: improved logging functionality

one-off-scripts/add-comment-on-frontmatter-issues.js

@@ -3,79 +3,6 @@ This is a one-off script to run on all open PRs to add
 a comment and "status: needs update" label to any PR with guide articles which
 have frontmatter issues.
 *//************ log.js **************/
-const path = require('path');
-const fs = require('fs');
-
-const { saveToFile } = require('../utils/save-to-file');
-
-class Log {
-  constructor() {
-    this._timeStamp = null;
-    this._prsArr = [];
-    this._indicesObj = {};
-    this._logfile = path.resolve(__dirname, `../work-logs/pr-relations.json`);
-  }
-
-  export() {
-    const log = {
-      timeStamp: this._timeStamp,
-      indices: this._indicesObj,
-      prs: this._prsArr
-    };
-    saveToFile(this._logfile, JSON.stringify(log))
-  }
-
-  add(prNum, props) {
-    this._prsArr.push(props);
-    this._indicesObj[prNum] = this._prsArr.length -1;
-  }
-
-  finish() {
-    this._timeStamp = new Date();
-    this.export();
-  }
-};
-
-module.exports = { Log };
-
-/*********** getData.js ***************/
-require('dotenv').config({ path: '../.env' });
-const { owner, repo, octokitConfig, octokitAuth } = require('../constants');
-
-const octokit = require('@octokit/rest')(octokitConfig);
-
-const { getPRs, getUserInput } = require('../get-prs');
-const { rateLimiter, savePrData } = require('../utils');
-const { Log } = require('./log');
-
-octokit.authenticate(octokitAuth);
-
-const log = new Log();
-
-(async () => {
-  const { firstPR, lastPR } = await getUserInput();
-  const prPropsToGet = ['number', 'user','files'];
-  const { openPRs } = await getPRs(firstPR, lastPR, prPropsToGet);
-
-  if (openPRs.length) {
-    console.log('Getting files...');
-    for (let count in openPRs) {
-      let { number, user: { login: username } } = openPRs[count];
-      const { data: prFiles } = await octokit.pullRequests.listFiles({ owner, repo, number });
-      const filenames = prFiles.map(({ filename }) => filename);
-      log.add(number, { number, username, filenames });
-      await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500);
-    }
-  }
-})()
-.then(() => {
-  log.finish();
-  console.log('Script completed');
-})
-.catch(err => {
-  log.finish();
-  console.log(err)
-})
 
 require('dotenv').config({ path: '../.env' });
 const fetch = require('node-fetch');
@@ -101,7 +28,7 @@ const allowedLangDirNames = [
 
 octokit.authenticate(octokitAuth);
 
-const log = new ProcessingLog();
+const log = new ProcessingLog('all-frontmatter-checks');
 
 const labeler = async (number, prFiles, currentLabels, guideFolderErrorsComment) => {
   const labelsToAdd = {}; // holds potential labels to add based on file path
@@ -167,18 +94,16 @@ const guideFolderChecks = async (number, prFiles, user) => {
     console.log('Starting frontmatter checks process...');
     for (let count in openPRs) {
       let { number, labels: currentLabels, user: { login: username } } = openPRs[count];
-      log.add(number, 'comment');
-      log.add(number, 'labels');
-
       const { data: prFiles } = await octokit.pullRequests.listFiles({ owner, repo, number });
 
       const guideFolderErrorsComment = await guideFolderChecks(number, prFiles, username);
       const commentLogVal = guideFolderErrorsComment ? guideFolderErrorsComment : 'none';
-      log.update(number, 'comment', commentLogVal)
 
       const labelsAdded = await labeler(number, prFiles, currentLabels, guideFolderErrorsComment);
       const labelLogVal = labelsAdded.length ? labelsAdded : 'none added';
-      log.update(number, 'labels', labelLogVal);
+
+      log.update(number, { comment: commentLogVal, labels: labelLogVal });
+      await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500);
     }
   }
 })()

one-off-scripts/add-test-locally-label.js

@@ -15,7 +15,7 @@ const { rateLimiter, savePrData, ProcessingLog } = require('../utils');
 
 octokit.authenticate(octokitAuth);
 
-const log = new ProcessingLog();
+const log = new ProcessingLog('all-locally-tested-labels');
 
 (async () => {
   const { firstPR, lastPR } = await getUserInput();
@@ -28,7 +28,6 @@ const log = new ProcessingLog();
     console.log('Starting labeling process...');
     for (let count in openPRs) {
       let { number, labels } = openPRs[count];
-      log.add(number, 'labels');
       const labelsToAdd = {}; // holds potential labels to add based on file path
       const existingLabels = labels.map(({ name }) => name);
       if (existingLabels.includes('scope: curriculum')) {
@@ -38,14 +37,14 @@ const log = new ProcessingLog();
       /* this next section only adds needed labels which are NOT currently on the PR. */
       const newLabels = Object.keys(labelsToAdd).filter(label => !existingLabels.includes(label));
       if (newLabels.length) {
-        log.update(number, 'labels', newLabels);
+        log.add(number, { labels: newLabels });
         if (process.env.PRODUCTION_RUN === 'true') {
           addLabels(number, newLabels, log);
           await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500);
         }
       }
       else {
-        log.update(number, 'labels', 'none added');
+        log.add(number, { labels: 'none added' });
       }
     }
   }

sweeper.js

@@ -1,40 +1,50 @@
+/*
+This script was originally created to iterate over all open PRs to label and comment
+on specific PR errors (i.e. guide related filenmame syntax and frontmatter).
+
+Since the first run which covered over 10,000+ PRs, it is curently ran every couple of days
+for just the most recent PRs.
+
+To run the script for a specific range (i.e. label and comment on guide errors),
+run `node sweeper.js range startingPrNumber endingPrNumber`
+*/
+
 const { owner, repo, octokitConfig, octokitAuth } = require('./constants');
 
 const octokit = require('@octokit/rest')(octokitConfig);
 
 const { getPRs, getUserInput } = require('./get-prs');
 const { guideFolderChecks } = require('./validation');
-const { savePrData, ProcessingLog } = require('./utils');
+const { savePrData, ProcessingLog, rateLimiter } = require('./utils');
 const { labeler } = require('./labeler');
 
 octokit.authenticate(octokitAuth);
 
-const log = new ProcessingLog();
+const log = new ProcessingLog('sweeper');
 
+log.start();
 console.log('Sweeper started...');
 (async () => {
   const { firstPR, lastPR } = await getUserInput();
+  log.setFirstLast({ firstPR, lastPR });
   const prPropsToGet = ['number', 'labels', 'user'];
   const { openPRs } = await getPRs(firstPR, lastPR, prPropsToGet);
 
   if (openPRs.length) {
     savePrData(openPRs, firstPR, lastPR);
-    log.start();
     console.log('Processing PRs...');
     for (let count in openPRs) {
       let { number, labels: currentLabels, user: { login: username } } = openPRs[count];
-      log.add(number, 'labels');
-      log.add(number, 'comment');
-
       const { data: prFiles } = await octokit.pullRequests.listFiles({ owner, repo, number });
 
       const guideFolderErrorsComment = await guideFolderChecks(number, prFiles, username);
       const commentLogVal = guideFolderErrorsComment ? guideFolderErrorsComment : 'none';
-      log.update(number, 'comment', commentLogVal)
 
       const labelsAdded = await labeler(number, prFiles, currentLabels, guideFolderErrorsComment);
       const labelLogVal = labelsAdded.length ? labelsAdded : 'none added';
-      log.update(number, 'labels', labelLogVal);
+
+      log.add(number, { comment: commentLogVal, labels: labelLogVal });
+      await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500);
     }
   }
 })()

utils/processing-log.js

@@ -1,45 +1,49 @@
+require('dotenv').config();
+const formatDate = require('date-fns/format');
 const path = require('path');
 const fs = require('fs');
 
 const { saveToFile } = require('./save-to-file');
 
 class ProcessingLog {
-  constructor() {
+  constructor(script) {
+    this._script = script;
     this._start = null;
-    this._lastUpdate = null;
-    this._lastPRlogged = null;
     this._finish = null;
+    this._elapsedTime = null;
     this._prs = {};
-    this._logfile = path.resolve(__dirname, `../work-logs/${this.getRunType()}_open-prs-processed.json`);
+    this._prCount = null;
+    this._logfile = path.resolve(__dirname, `../work-logs/data-for_${this.getRunType()}_${this._script}.json`);
   }
 
   getRunType() {
     return process.env.PRODUCTION_RUN === 'true' ? 'production' : 'test';
   }
 
-  import() {
-    return JSON.parse(fs.readFileSync(this._logfile, 'utf8'));
-  }
-
   export() {
     let sortedPRs = Object.keys(this._prs)
      .sort((a, b) => a - b)
-     .map(num => ({ [num]: this._prs[num] }));
+     .map(num => ({ number: num, data: this._prs[num] }));
+
     const log = {
       start: this._start,
       finish: this._finish,
+      elapsedTime: this._elapsedTime,
+      prCount: sortedPRs.length,
+      firstPR: this._firstPR,
+      lastPR: this._lastPR,
       prs: sortedPRs
     };
     saveToFile(this._logfile, JSON.stringify(log))
   }
 
-  add(prNum, prop) {
+  add(prNum, props) {
-    this._prs[prNum] = {};
+    this._prs[prNum] = props;
-    this._prs[prNum][prop] = null;
   }
 
-  update(prNum, prop, value) {
+  setFirstLast({ firstPR, lastPR }) {
-    this._prs[prNum][prop] = value;
+    this._firstPR = firstPR;
+    this._lastPR = lastPR;
   }
 
   start() {
@@ -48,7 +52,20 @@ class ProcessingLog {
 
   finish() {
     this._finish = new Date();
+    const minutesElapsed = (this._finish - this._start) / 1000 / 60;
+    this._elapsedTime =  minutesElapsed.toFixed(2) + ' mins';
     this.export();
+    this.changeFilename();
+  }
+
+  changeFilename() {
+    const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss');
+    const newFilename = path.resolve(__dirname,`../work-logs/${this.getRunType()}_${this._script}_${this._firstPR}-${this._lastPR}_${now}.json`);
+    fs.rename(this._logfile, newFilename, function(err) {
+      if (err) {
+        throw(err);
+      }
+    });
   }
 };
 

utils/save-pr-data.js

@@ -5,7 +5,7 @@ const { saveToFile } = require('./save-to-file');
 
 const savePrData = (openPRs, firstPR, lastPR) => {
   const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss');
-  const filename = path.resolve(__dirname, `../work-logs/openprs_${firstPR}-${lastPR}_${now}.json`);
+  const filename = path.resolve(__dirname, `../work-logs/data-for-openprs_${firstPR}-${lastPR}_${now}.json`);
   console.log(`# of PRs Retrieved: ${openPRs.length}`);
   console.log(`PR Range: ${firstPR} - ${lastPR}`);
   saveToFile(filename, JSON.stringify(openPRs));