From dcf513cb198ea39b428360f4a353ea645121a22e Mon Sep 17 00:00:00 2001 From: Randell Dawson Date: Mon, 3 Dec 2018 01:57:13 -0800 Subject: [PATCH] feat: added 2 progress bars for sweeper --- get-prs/index.js | 22 +++-- get-prs/pr-stats.js | 7 +- one-off-scripts/get-pr-relations-data.js | 112 +++++++++++++++++++++++ 3 files changed, 132 insertions(+), 9 deletions(-) create mode 100644 one-off-scripts/get-pr-relations-data.js diff --git a/get-prs/index.js b/get-prs/index.js index 14ef0b347d..f4b1699814 100644 --- a/get-prs/index.js +++ b/get-prs/index.js @@ -1,13 +1,14 @@ require('dotenv').config(); +const _cliProgress = require('cli-progress'); const { owner, repo, octokitConfig, octokitAuth } = require('../constants'); const octokit = require('@octokit/rest')(octokitConfig); -const { getRange } = require('./pr-stats'); +const { getRange, getCount } = require('./pr-stats'); octokit.authenticate(octokitAuth); -const paginate = async function paginate (method, octokit, firstPR, lastPR, prPropsToGet) { +const paginate = async function paginate (method, octokit, firstPR, lastPR, prPropsToGet, progressBar) { const prFilter = (prs, first, last, prPropsToGet) => { const filtered = []; @@ -41,6 +42,7 @@ const paginate = async function paginate (method, octokit, firstPR, lastPR, prPr response = await octokit.getNextPage(response); let dataFiltered = prFilter(response.data, firstPR, lastPR, prPropsToGet); data = data.concat(dataFiltered); + progressBar.increment(dataFiltered.length); } return data; }; @@ -48,7 +50,6 @@ const paginate = async function paginate (method, octokit, firstPR, lastPR, prPr const getUserInput = async () => { let [ n, f, type, start, end ] = process.argv; let [ firstPR, lastPR ] = await getRange().then(data => data); - if (type !== 'all' && type !== 'range') { throw `Please specify either all or range for 1st arg.`; } @@ -70,12 +71,19 @@ const getUserInput = async () => { } lastPR = end; } - return {firstPR, lastPR}; + const totalPRs = await getCount().then(data => data); + return {totalPRs, firstPR, lastPR}; }; -const getPRs = async (firstPR, lastPR, prPropsToGet) => { - console.log(`Retrieving PRs (#${firstPR} thru #${lastPR})`); - let openPRs = await paginate(octokit.pullRequests.list, octokit, firstPR, lastPR, prPropsToGet); +const getPRs = async (totalPRs, firstPR, lastPR, prPropsToGet) => { + const getPRsBar = new _cliProgress.Bar({ + format: `Part 1 of 2: Retrieving PRs (${firstPR}-${lastPR}) [{bar}] {percentage}%` + }, _cliProgress.Presets.shades_classic); + getPRsBar.start(totalPRs, 0); + let openPRs = await paginate(octokit.pullRequests.list, octokit, firstPR, lastPR, prPropsToGet, getPRsBar); + getPRsBar.update(totalPRs); + getPRsBar.stop(); + console.log(`# of PRs retrieved: ${openPRs.length}`); return { firstPR, lastPR, openPRs }; } diff --git a/get-prs/pr-stats.js b/get-prs/pr-stats.js index 07d43f49ab..435c9235da 100644 --- a/get-prs/pr-stats.js +++ b/get-prs/pr-stats.js @@ -13,11 +13,14 @@ const page = 1; const per_page = 1; const getCount = async () => { - const { data } = await octokit.search.issues({ + const { data: { total_count: count } } = await octokit.search.issues({ q: `repo:${owner}/${repo}+is:open+type:pr+base:master`, sort: 'created', order: 'asc', page: 1, per_page: 1 + }) + .catch((err) => { + console.log(err); }); - return data.length; + return count; }; const getFirst = async () => { diff --git a/one-off-scripts/get-pr-relations-data.js b/one-off-scripts/get-pr-relations-data.js new file mode 100644 index 0000000000..116ae0cadb --- /dev/null +++ b/one-off-scripts/get-pr-relations-data.js @@ -0,0 +1,112 @@ +require('dotenv').config({ path: '../.env' }); +const formatDate = require('date-fns/format'); + +const path = require('path'); +const fs = require('fs'); +const _cliProgress = require('cli-progress'); + +const { saveToFile } = require('../utils/save-to-file'); + +class Log { + constructor() { + this._startTime = null; + this._finishTime = null; + this._elapsedTime = null; + this._prsArr = []; + this._indicesObj = {}; + this._logfile = path.resolve(__dirname, `../work-logs/pr-relations.json`); + } + + export() { + const log = { + startTime: this._startTime, + finishTime: this._finishTime, + elapsedTime: this._elapsedTime, + indices: this._indicesObj, + prs: this._prsArr + }; + saveToFile(this._logfile, JSON.stringify(log)) + } + + getPrRange() { + const first = this._prsArr[0].number; + const last = this._prsArr[this._prsArr.length -1].number; + return [first, last]; + } + + add(prNum, props) { + this._prsArr.push(props); + this._indicesObj[prNum] = this._prsArr.length -1; + } + + start() { + this._startTime = new Date(); + this.export(); + } + + finish() { + this._finishTime = new Date(); + const minutesElapsed = (this._finishTime - this._startTime) / 1000 / 60; + this._elapsedTime = minutesElapsed.toFixed(2) + ' mins'; + this.export(); + this.changeFilename(this.getPrRange()); + } + + changeFilename( [first, last] ) { + const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss'); + const newFilename = path.resolve(__dirname,`./work-logs/pr-relations_${first}-${last}_${now}.json`); + fs.rename(this._logfile, newFilename, function(err) { + if (err) { + throw('ERROR: ' + err); + } + }); + } +}; + +const { owner, repo, octokitConfig, octokitAuth } = require('../constants'); + +const octokit = require('@octokit/rest')(octokitConfig); + +const { getPRs, getUserInput } = require('../get-prs'); +const { rateLimiter, savePrData } = require('../utils'); + +octokit.authenticate(octokitAuth); + +const log = new Log(); +(async () => { + const { totalPRs, firstPR, lastPR } = await getUserInput(); + const prPropsToGet = ['number', 'user', 'files']; + const { openPRs } = await getPRs(totalPRs, firstPR, lastPR, prPropsToGet); + + if (openPRs.length) { + log.start(); + let numFilenameRequests = 0; + const getFilesBar = new _cliProgress.Bar({ + format: `Part 2 of 2: Retrieving filenames [{bar}] {percentage}% | {value}/{total}` + }, _cliProgress.Presets.shades_classic); + getFilesBar.start(openPRs.length, 0); + for (let count in openPRs) { + let { number, user: { login: username } } = openPRs[count]; + const { data: prFiles } = await octokit.pullRequests.listFiles({ owner, repo, number }); + const filenames = prFiles.map(({ filename }) => filename); + log.add(number, { number, username, filenames }); + if (numFilenameRequests > 3000 ) { + await rateLimiter(1200); + } + if (numFilenameRequests % 10 === 0) { + getFilesBar.update(numFilenameRequests); + } + numFilenameRequests++; + } + getFilesBar.update(openPRs.length); + getFilesBar.stop(); + } +})() +.then(() => { + log.finish(); + console.log('Finished retrieving pr-relations data'); +}) +.catch(err => { + log.finish(); + console.log(err) +})