diff --git a/one-off-scripts/add-comment-on-frontmatter-issues.js b/one-off-scripts/add-comment-on-frontmatter-issues.js index ae897399a6..52474cfe88 100644 --- a/one-off-scripts/add-comment-on-frontmatter-issues.js +++ b/one-off-scripts/add-comment-on-frontmatter-issues.js @@ -3,79 +3,6 @@ This is a one-off script to run on all open PRs to add a comment and "status: needs update" label to any PR with guide articles which have frontmatter issues. *//************ log.js **************/ -const path = require('path'); -const fs = require('fs'); - -const { saveToFile } = require('../utils/save-to-file'); - -class Log { - constructor() { - this._timeStamp = null; - this._prsArr = []; - this._indicesObj = {}; - this._logfile = path.resolve(__dirname, `../work-logs/pr-relations.json`); - } - - export() { - const log = { - timeStamp: this._timeStamp, - indices: this._indicesObj, - prs: this._prsArr - }; - saveToFile(this._logfile, JSON.stringify(log)) - } - - add(prNum, props) { - this._prsArr.push(props); - this._indicesObj[prNum] = this._prsArr.length -1; - } - - finish() { - this._timeStamp = new Date(); - this.export(); - } -}; - -module.exports = { Log }; - -/*********** getData.js ***************/ -require('dotenv').config({ path: '../.env' }); -const { owner, repo, octokitConfig, octokitAuth } = require('../constants'); - -const octokit = require('@octokit/rest')(octokitConfig); - -const { getPRs, getUserInput } = require('../get-prs'); -const { rateLimiter, savePrData } = require('../utils'); -const { Log } = require('./log'); - -octokit.authenticate(octokitAuth); - -const log = new Log(); - -(async () => { - const { firstPR, lastPR } = await getUserInput(); - const prPropsToGet = ['number', 'user','files']; - const { openPRs } = await getPRs(firstPR, lastPR, prPropsToGet); - - if (openPRs.length) { - console.log('Getting files...'); - for (let count in openPRs) { - let { number, user: { login: username } } = openPRs[count]; - const { data: prFiles } = await octokit.pullRequests.listFiles({ owner, repo, number }); - const filenames = prFiles.map(({ filename }) => filename); - log.add(number, { number, username, filenames }); - await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500); - } - } -})() -.then(() => { - log.finish(); - console.log('Script completed'); -}) -.catch(err => { - log.finish(); - console.log(err) -}) require('dotenv').config({ path: '../.env' }); const fetch = require('node-fetch'); @@ -101,7 +28,7 @@ const allowedLangDirNames = [ octokit.authenticate(octokitAuth); -const log = new ProcessingLog(); +const log = new ProcessingLog('all-frontmatter-checks'); const labeler = async (number, prFiles, currentLabels, guideFolderErrorsComment) => { const labelsToAdd = {}; // holds potential labels to add based on file path @@ -167,18 +94,16 @@ const guideFolderChecks = async (number, prFiles, user) => { console.log('Starting frontmatter checks process...'); for (let count in openPRs) { let { number, labels: currentLabels, user: { login: username } } = openPRs[count]; - log.add(number, 'comment'); - log.add(number, 'labels'); - const { data: prFiles } = await octokit.pullRequests.listFiles({ owner, repo, number }); const guideFolderErrorsComment = await guideFolderChecks(number, prFiles, username); const commentLogVal = guideFolderErrorsComment ? guideFolderErrorsComment : 'none'; - log.update(number, 'comment', commentLogVal) const labelsAdded = await labeler(number, prFiles, currentLabels, guideFolderErrorsComment); const labelLogVal = labelsAdded.length ? labelsAdded : 'none added'; - log.update(number, 'labels', labelLogVal); + + log.update(number, { comment: commentLogVal, labels: labelLogVal }); + await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500); } } })() diff --git a/one-off-scripts/add-test-locally-label.js b/one-off-scripts/add-test-locally-label.js index 5cab97089a..c4e04e1a3c 100644 --- a/one-off-scripts/add-test-locally-label.js +++ b/one-off-scripts/add-test-locally-label.js @@ -15,7 +15,7 @@ const { rateLimiter, savePrData, ProcessingLog } = require('../utils'); octokit.authenticate(octokitAuth); -const log = new ProcessingLog(); +const log = new ProcessingLog('all-locally-tested-labels'); (async () => { const { firstPR, lastPR } = await getUserInput(); @@ -28,7 +28,6 @@ const log = new ProcessingLog(); console.log('Starting labeling process...'); for (let count in openPRs) { let { number, labels } = openPRs[count]; - log.add(number, 'labels'); const labelsToAdd = {}; // holds potential labels to add based on file path const existingLabels = labels.map(({ name }) => name); if (existingLabels.includes('scope: curriculum')) { @@ -38,14 +37,14 @@ const log = new ProcessingLog(); /* this next section only adds needed labels which are NOT currently on the PR. */ const newLabels = Object.keys(labelsToAdd).filter(label => !existingLabels.includes(label)); if (newLabels.length) { - log.update(number, 'labels', newLabels); + log.add(number, { labels: newLabels }); if (process.env.PRODUCTION_RUN === 'true') { addLabels(number, newLabels, log); await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500); } } else { - log.update(number, 'labels', 'none added'); + log.add(number, { labels: 'none added' }); } } } diff --git a/sweeper.js b/sweeper.js index 79688667f0..0ba6f9569a 100644 --- a/sweeper.js +++ b/sweeper.js @@ -1,40 +1,50 @@ +/* +This script was originally created to iterate over all open PRs to label and comment +on specific PR errors (i.e. guide related filenmame syntax and frontmatter). + +Since the first run which covered over 10,000+ PRs, it is curently ran every couple of days +for just the most recent PRs. + +To run the script for a specific range (i.e. label and comment on guide errors), +run `node sweeper.js range startingPrNumber endingPrNumber` +*/ + const { owner, repo, octokitConfig, octokitAuth } = require('./constants'); const octokit = require('@octokit/rest')(octokitConfig); const { getPRs, getUserInput } = require('./get-prs'); const { guideFolderChecks } = require('./validation'); -const { savePrData, ProcessingLog } = require('./utils'); +const { savePrData, ProcessingLog, rateLimiter } = require('./utils'); const { labeler } = require('./labeler'); octokit.authenticate(octokitAuth); -const log = new ProcessingLog(); +const log = new ProcessingLog('sweeper'); +log.start(); console.log('Sweeper started...'); (async () => { const { firstPR, lastPR } = await getUserInput(); + log.setFirstLast({ firstPR, lastPR }); const prPropsToGet = ['number', 'labels', 'user']; const { openPRs } = await getPRs(firstPR, lastPR, prPropsToGet); if (openPRs.length) { savePrData(openPRs, firstPR, lastPR); - log.start(); console.log('Processing PRs...'); for (let count in openPRs) { let { number, labels: currentLabels, user: { login: username } } = openPRs[count]; - log.add(number, 'labels'); - log.add(number, 'comment'); - const { data: prFiles } = await octokit.pullRequests.listFiles({ owner, repo, number }); const guideFolderErrorsComment = await guideFolderChecks(number, prFiles, username); const commentLogVal = guideFolderErrorsComment ? guideFolderErrorsComment : 'none'; - log.update(number, 'comment', commentLogVal) const labelsAdded = await labeler(number, prFiles, currentLabels, guideFolderErrorsComment); const labelLogVal = labelsAdded.length ? labelsAdded : 'none added'; - log.update(number, 'labels', labelLogVal); + + log.add(number, { comment: commentLogVal, labels: labelLogVal }); + await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500); } } })() diff --git a/utils/processing-log.js b/utils/processing-log.js index ee8bfefb0d..cce788d6ca 100644 --- a/utils/processing-log.js +++ b/utils/processing-log.js @@ -1,45 +1,49 @@ +require('dotenv').config(); +const formatDate = require('date-fns/format'); const path = require('path'); const fs = require('fs'); const { saveToFile } = require('./save-to-file'); class ProcessingLog { - constructor() { + constructor(script) { + this._script = script; this._start = null; - this._lastUpdate = null; - this._lastPRlogged = null; this._finish = null; + this._elapsedTime = null; this._prs = {}; - this._logfile = path.resolve(__dirname, `../work-logs/${this.getRunType()}_open-prs-processed.json`); + this._prCount = null; + this._logfile = path.resolve(__dirname, `../work-logs/data-for_${this.getRunType()}_${this._script}.json`); } getRunType() { return process.env.PRODUCTION_RUN === 'true' ? 'production' : 'test'; } - import() { - return JSON.parse(fs.readFileSync(this._logfile, 'utf8')); - } - export() { let sortedPRs = Object.keys(this._prs) .sort((a, b) => a - b) - .map(num => ({ [num]: this._prs[num] })); + .map(num => ({ number: num, data: this._prs[num] })); + const log = { start: this._start, finish: this._finish, + elapsedTime: this._elapsedTime, + prCount: sortedPRs.length, + firstPR: this._firstPR, + lastPR: this._lastPR, prs: sortedPRs }; saveToFile(this._logfile, JSON.stringify(log)) } - add(prNum, prop) { - this._prs[prNum] = {}; - this._prs[prNum][prop] = null; + add(prNum, props) { + this._prs[prNum] = props; } - update(prNum, prop, value) { - this._prs[prNum][prop] = value; + setFirstLast({ firstPR, lastPR }) { + this._firstPR = firstPR; + this._lastPR = lastPR; } start() { @@ -48,7 +52,20 @@ class ProcessingLog { finish() { this._finish = new Date(); + const minutesElapsed = (this._finish - this._start) / 1000 / 60; + this._elapsedTime = minutesElapsed.toFixed(2) + ' mins'; this.export(); + this.changeFilename(); + } + + changeFilename() { + const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss'); + const newFilename = path.resolve(__dirname,`../work-logs/${this.getRunType()}_${this._script}_${this._firstPR}-${this._lastPR}_${now}.json`); + fs.rename(this._logfile, newFilename, function(err) { + if (err) { + throw(err); + } + }); } }; diff --git a/utils/save-pr-data.js b/utils/save-pr-data.js index f7fe00231d..6906016095 100644 --- a/utils/save-pr-data.js +++ b/utils/save-pr-data.js @@ -5,7 +5,7 @@ const { saveToFile } = require('./save-to-file'); const savePrData = (openPRs, firstPR, lastPR) => { const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss'); - const filename = path.resolve(__dirname, `../work-logs/openprs_${firstPR}-${lastPR}_${now}.json`); + const filename = path.resolve(__dirname, `../work-logs/data-for-openprs_${firstPR}-${lastPR}_${now}.json`); console.log(`# of PRs Retrieved: ${openPRs.length}`); console.log(`PR Range: ${firstPR} - ${lastPR}`); saveToFile(filename, JSON.stringify(openPRs));