diff --git a/package.json b/package.json index 3b197779e1..7ec11549eb 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,8 @@ "description": "Tools to help maintain freecodecamp.org's Open Source Codebase on GitHub", "private": true, "scripts": { - "postinstall": "lerna bootstrap" + "bootstrap": "lerna bootstrap", + "postinstall": "npm run bootstrap" }, "devDependencies": { "lerna": "^3.5.1" diff --git a/sweeper/get-prs/index.js b/sweeper/get-prs/index.js index f4b1699814..733b8cdaea 100644 --- a/sweeper/get-prs/index.js +++ b/sweeper/get-prs/index.js @@ -47,13 +47,22 @@ const paginate = async function paginate (method, octokit, firstPR, lastPR, prPr return data; }; -const getUserInput = async () => { - let [ n, f, type, start, end ] = process.argv; - let [ firstPR, lastPR ] = await getRange().then(data => data); - if (type !== 'all' && type !== 'range') { - throw `Please specify either all or range for 1st arg.`; +const getUserInput = async (rangeType = '') => { + let data, firstPR, lastPR; + if (rangeType === 'all') { + data = await getRange().then(data => data); + firstPR = data[0]; + lastPR = data[1]; } - if (type === 'range') { + else { + let [ n, f, type, start, end ] = process.argv; + data = await getRange().then(data => data); + firstPR = data[0]; + lastPR = data[1]; + if (type !== 'all' && type !== 'range') { + throw `Please specify either all or range for 1st arg.`; + } + if (type === 'range') { start = parseInt(start); end = parseInt(end); if (!start || !end) { @@ -70,14 +79,15 @@ const getUserInput = async () => { throw `Ending PR # can not be greater than last open PR # (${lastPR})`; } lastPR = end; + } } const totalPRs = await getCount().then(data => data); - return {totalPRs, firstPR, lastPR}; + return { totalPRs, firstPR, lastPR }; }; const getPRs = async (totalPRs, firstPR, lastPR, prPropsToGet) => { const getPRsBar = new _cliProgress.Bar({ - format: `Part 1 of 2: Retrieving PRs (${firstPR}-${lastPR}) [{bar}] {percentage}%` + format: `Part 1 of 2: Retrieving PRs (${firstPR}-${lastPR}) [{bar}] {percentage}% - {duration_formatted}` }, _cliProgress.Presets.shades_classic); getPRsBar.start(totalPRs, 0); let openPRs = await paginate(octokit.pullRequests.list, octokit, firstPR, lastPR, prPropsToGet, getPRsBar); diff --git a/sweeper/one-off-scripts/get-changes-for-pr-relations-data.js b/sweeper/one-off-scripts/get-changes-for-pr-relations-data.js new file mode 100644 index 0000000000..a7efa40286 --- /dev/null +++ b/sweeper/one-off-scripts/get-changes-for-pr-relations-data.js @@ -0,0 +1,134 @@ +require('dotenv').config({ path: '../.env' }); +const formatDate = require('date-fns/format'); +const path = require('path'); +const fs = require('fs'); +const _cliProgress = require('cli-progress'); +const fetch = require('node-fetch'); + +const { saveToFile } = require('../utils/save-to-file'); + +class Log { + constructor() { + this._startTime = null; + this._finishTime = null; + this._elapsedTime = null; + this._prsArr = []; + this._indicesObj = {}; + this._logfile = path.resolve(__dirname, `../work-logs/pr-relations.json`); + } + + export() { + const log = { + startTime: this._startTime, + finishTime: this._finishTime, + elapsedTime: this._elapsedTime, + indices: this._indicesObj, + prs: this._prsArr + }; + saveToFile(this._logfile, JSON.stringify(log)) + } + + getPrRange() { + const first = this._prsArr[0].number; + const last = this._prsArr[this._prsArr.length -1].number; + return [first, last]; + } + + add(prNum, props) { + this._prsArr.push(props); + this._indicesObj[prNum] = this._prsArr.length -1; + } + + start() { + this._startTime = new Date(); + this.export(); + } + + finish() { + this._finishTime = new Date(); + const minutesElapsed = (this._finishTime - this._startTime) / 1000 / 60; + this._elapsedTime = minutesElapsed.toFixed(2) + ' mins'; + this.export(); + this.changeFilename(this.getPrRange()); + } + + changeFilename( [first, last] ) { + const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss'); + const newFilename = path.resolve(__dirname,`../work-logs/pr-relations_${first}-${last}_${now}.json`); + fs.rename(this._logfile, newFilename, function(err) { + if (err) { + throw('ERROR: ' + err); + } + }); + } +}; + +const getExistingData = async () => { + const url = `https://pr-relations.glitch.me/getCurrData`; + const response = await fetch(url); + const data = await response.json(); + return data; +}; + +const { owner, repo, octokitConfig, octokitAuth } = require('../constants'); +const octokit = require('@octokit/rest')(octokitConfig); + +const { getPRs, getUserInput } = require('../get-prs'); +const { rateLimiter, savePrData } = require('../utils'); + +octokit.authenticate(octokitAuth); + +const log = new Log(); +(async () => { + const { totalPRs, firstPR, lastPR } = await getUserInput('all'); + log.start(); + const prPropsToGet = ['number', 'user', 'updated_at', 'files']; + const { openPRs } = await getPRs(totalPRs, firstPR, lastPR, prPropsToGet); + + if (openPRs.length) { + const { indices: oldIndices, prs: oldPRs } = await getExistingData(); + + const getFilesBar = new _cliProgress.Bar({ + format: `Part 2 of 2: Adding/Updating PRs [{bar}] {percentage}% | {value}/{total} | {duration_formatted}` + }, _cliProgress.Presets.shades_classic); + getFilesBar.start(openPRs.length, 0); + + let newOrUpdated = ''; + for (let count in openPRs) { + let { number, updated_at, user: { login: username } } = openPRs[count]; + let oldUpdated_at; + let oldPrData = oldPRs[oldIndices[number]]; + if (oldPrData) { + oldUpdated_at = oldPrData.updated_at; + } + + if (!oldIndices.hasOwnProperty(number) || updated_at > oldUpdated_at) { + newOrUpdated += `PR #${number} was new or needed updating\n`; + const { data: prFiles } = await octokit.pullRequests.listFiles({ owner, repo, number }); + const filenames = prFiles.map(({ filename }) => filename); + log.add(number, { number, updated_at, username, filenames }); + if (+count > 3000 ) { + await rateLimiter(1400); + } + } + else { + let { username: oldUsername, filenames: oldFilenames } = oldPrData; + log.add(number, { number, updated_at: oldUpdated_at, username: oldUsername, filenames: oldFilenames }); + } + if (+count % 10 === 0) { + getFilesBar.update(+count); + } + } + getFilesBar.update(openPRs.length); + getFilesBar.stop(); + console.log(newOrUpdated); + } +})() +.then(() => { + log.finish(); + console.log('Finished retrieving pr-relations data'); +}) +.catch(err => { + log.finish(); + console.log(err) +}) diff --git a/sweeper/one-off-scripts/get-pr-relations-data.js b/sweeper/one-off-scripts/get-pr-relations-data.js index a1250140b7..bca7d963a3 100644 --- a/sweeper/one-off-scripts/get-pr-relations-data.js +++ b/sweeper/one-off-scripts/get-pr-relations-data.js @@ -1,6 +1,5 @@ require('dotenv').config({ path: '../.env' }); const formatDate = require('date-fns/format'); - const path = require('path'); const fs = require('fs'); const _cliProgress = require('cli-progress'); @@ -32,6 +31,7 @@ class Log { const first = this._prsArr[0].number; const last = this._prsArr[this._prsArr.length -1].number; return [first, last]; + // return [null, null] } add(prNum, props) { @@ -74,29 +74,27 @@ octokit.authenticate(octokitAuth); const log = new Log(); (async () => { - const { totalPRs, firstPR, lastPR } = await getUserInput(); - const prPropsToGet = ['number', 'user', 'files']; + const { totalPRs, firstPR, lastPR } = await getUserInput('all'); + const prPropsToGet = ['number', 'user', 'updated_at', 'files']; const { openPRs } = await getPRs(totalPRs, firstPR, lastPR, prPropsToGet); if (openPRs.length) { log.start(); - let numFilenameRequests = 0; const getFilesBar = new _cliProgress.Bar({ format: `Part 2 of 2: Retrieving filenames [{bar}] {percentage}% | {value}/{total}` }, _cliProgress.Presets.shades_classic); getFilesBar.start(openPRs.length, 0); for (let count in openPRs) { - let { number, user: { login: username } } = openPRs[count]; + let { number, updated_at, user: { login: username } } = openPRs[count]; const { data: prFiles } = await octokit.pullRequests.listFiles({ owner, repo, number }); const filenames = prFiles.map(({ filename }) => filename); - log.add(number, { number, username, filenames }); - if (numFilenameRequests > 3000 ) { - await rateLimiter(1250); + log.add(number, { number, updated_at, username, filenames }); + if (+count > 3000 ) { + await rateLimiter(1500); } - if (numFilenameRequests % 10 === 0) { - getFilesBar.update(numFilenameRequests); + if (+count % 10 === 0) { + getFilesBar.update(+count); } - numFilenameRequests++; } getFilesBar.update(openPRs.length); getFilesBar.stop();