feat: added 2 progress bars for sweeper

get-prs/index.js

@@ -1,13 +1,14 @@
 require('dotenv').config();
+const _cliProgress = require('cli-progress');
 
 const { owner, repo, octokitConfig, octokitAuth } = require('../constants');
 
 const octokit = require('@octokit/rest')(octokitConfig);
-const { getRange } = require('./pr-stats');
+const { getRange, getCount } = require('./pr-stats');
 
 octokit.authenticate(octokitAuth);
 
-const paginate = async function paginate (method, octokit, firstPR, lastPR, prPropsToGet) {
+const paginate = async function paginate (method, octokit, firstPR, lastPR, prPropsToGet, progressBar) {
 
   const prFilter = (prs, first, last, prPropsToGet) => {
     const filtered = [];
@@ -41,6 +42,7 @@ const paginate = async function paginate (method, octokit, firstPR, lastPR, prPr
     response = await octokit.getNextPage(response);
     let dataFiltered = prFilter(response.data, firstPR, lastPR, prPropsToGet);
     data = data.concat(dataFiltered);
+    progressBar.increment(dataFiltered.length);
   }
   return data;
 };
@@ -48,7 +50,6 @@ const paginate = async function paginate (method, octokit, firstPR, lastPR, prPr
 const getUserInput = async () => {
   let [ n, f, type, start, end ] = process.argv;
   let [ firstPR, lastPR ] = await getRange().then(data => data);
-
   if (type !== 'all' && type !== 'range') {
     throw `Please specify either all or range for 1st arg.`;
   }
@@ -70,12 +71,19 @@ const getUserInput = async () => {
     }
     lastPR = end;
   }
-  return {firstPR, lastPR};
+  const totalPRs = await getCount().then(data => data);
+  return {totalPRs, firstPR, lastPR};
 };
 
-const getPRs = async (firstPR, lastPR, prPropsToGet) => {
+const getPRs = async (totalPRs, firstPR, lastPR, prPropsToGet) => {
-  console.log(`Retrieving PRs (#${firstPR} thru #${lastPR})`);
+  const getPRsBar = new _cliProgress.Bar({
-  let openPRs = await paginate(octokit.pullRequests.list, octokit, firstPR, lastPR, prPropsToGet);
+    format: `Part 1 of 2: Retrieving PRs (${firstPR}-${lastPR}) [{bar}] {percentage}%`
+  }, _cliProgress.Presets.shades_classic);
+  getPRsBar.start(totalPRs, 0);
+  let openPRs = await paginate(octokit.pullRequests.list, octokit, firstPR, lastPR, prPropsToGet, getPRsBar);
+  getPRsBar.update(totalPRs);
+  getPRsBar.stop();
+  console.log(`# of PRs retrieved: ${openPRs.length}`);
   return { firstPR, lastPR, openPRs };
 }
 

get-prs/pr-stats.js

@@ -13,11 +13,14 @@ const page = 1;
 const per_page = 1;
 
 const getCount = async () => {
-  const { data } = await octokit.search.issues({
+  const { data: { total_count: count } } = await octokit.search.issues({
     q: `repo:${owner}/${repo}+is:open+type:pr+base:master`,
     sort: 'created', order: 'asc', page: 1, per_page: 1
+  })
+  .catch((err) => {
+    console.log(err);
   });
-  return data.length;
+  return count;
 };
 
 const getFirst = async () => {

one-off-scripts/get-pr-relations-data.js

@@ -0,0 +1,112 @@
+require('dotenv').config({ path: '../.env' });
+const formatDate = require('date-fns/format');
+
+const path = require('path');
+const fs = require('fs');
+const _cliProgress = require('cli-progress');
+
+const { saveToFile } = require('../utils/save-to-file');
+
+class Log {
+  constructor() {
+    this._startTime = null;
+    this._finishTime = null;
+    this._elapsedTime = null;
+    this._prsArr = [];
+    this._indicesObj = {};
+    this._logfile = path.resolve(__dirname, `../work-logs/pr-relations.json`);
+  }
+
+  export() {
+    const log = {
+      startTime: this._startTime,
+      finishTime: this._finishTime,
+      elapsedTime: this._elapsedTime,
+      indices: this._indicesObj,
+      prs: this._prsArr
+    };
+    saveToFile(this._logfile, JSON.stringify(log))
+  }
+
+  getPrRange() {
+    const first = this._prsArr[0].number;
+    const last = this._prsArr[this._prsArr.length -1].number;
+    return [first, last];
+  }
+
+  add(prNum, props) {
+    this._prsArr.push(props);
+    this._indicesObj[prNum] = this._prsArr.length -1;
+  }
+
+  start() {
+    this._startTime = new Date();
+    this.export();
+  }
+
+  finish() {
+    this._finishTime = new Date();
+    const minutesElapsed = (this._finishTime - this._startTime) / 1000 / 60;
+    this._elapsedTime =  minutesElapsed.toFixed(2) + ' mins';
+    this.export();
+    this.changeFilename(this.getPrRange());
+  }
+
+  changeFilename( [first, last] ) {
+    const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss');
+    const newFilename = path.resolve(__dirname,`./work-logs/pr-relations_${first}-${last}_${now}.json`);
+    fs.rename(this._logfile, newFilename, function(err) {
+      if (err) {
+        throw('ERROR: ' + err);
+      }
+    });
+  }
+};
+
+const { owner, repo, octokitConfig, octokitAuth } = require('../constants');
+
+const octokit = require('@octokit/rest')(octokitConfig);
+
+const { getPRs, getUserInput } = require('../get-prs');
+const { rateLimiter, savePrData } = require('../utils');
+
+octokit.authenticate(octokitAuth);
+
+const log = new Log();
+(async () => {
+  const { totalPRs, firstPR, lastPR } = await getUserInput();
+  const prPropsToGet = ['number', 'user', 'files'];
+  const { openPRs } = await getPRs(totalPRs, firstPR, lastPR, prPropsToGet);
+
+  if (openPRs.length) {
+    log.start();
+    let numFilenameRequests = 0;
+    const getFilesBar = new _cliProgress.Bar({
+      format: `Part 2 of 2: Retrieving filenames [{bar}] {percentage}% | {value}/{total}`
+    }, _cliProgress.Presets.shades_classic);
+    getFilesBar.start(openPRs.length, 0);
+    for (let count in openPRs) {
+      let { number, user: { login: username } } = openPRs[count];
+      const { data: prFiles } = await octokit.pullRequests.listFiles({ owner, repo, number });
+      const filenames = prFiles.map(({ filename }) => filename);
+      log.add(number, { number, username, filenames });
+      if (numFilenameRequests > 3000 ) {
+        await rateLimiter(1200);
+      }
+      if (numFilenameRequests % 10 === 0) {
+        getFilesBar.update(numFilenameRequests);
+      }
+      numFilenameRequests++;
+    }
+    getFilesBar.update(openPRs.length);
+    getFilesBar.stop();
+  }
+})()
+.then(() => {
+  log.finish();
+  console.log('Finished retrieving pr-relations data');
+})
+.catch(err => {
+  log.finish();
+  console.log(err)
+})