gaspersic/freeCodeCamp
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat: improved logging functionality

Browse Source
This commit is contained in:
Randell Dawson
2018-11-22 23:38:58 -08:00
committed by mrugesh mohapatra
parent 545324ea46
commit b5b6faed30
5 changed files with 57 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
one-off-scripts/add-comment-on-frontmatter-issues.js
View File

@ -3,79 +3,6 @@ This is a one-off script to run on all open PRs to add
a comment and "status: needs update" label to any PR with guide articles which
have frontmatter issues.
*//************ log.js **************/
const path = require('path');
const fs = require('fs');
const { saveToFile } = require('../utils/save-to-file');
class Log {
constructor() {
this._timeStamp = null;
this._prsArr = [];
this._indicesObj = {};
this._logfile = path.resolve(__dirname, `../work-logs/pr-relations.json`);
}
export() {
const log = {
timeStamp: this._timeStamp,
indices: this._indicesObj,
prs: this._prsArr
};
saveToFile(this._logfile, JSON.stringify(log))
}
add(prNum, props) {
this._prsArr.push(props);
this._indicesObj[prNum] = this._prsArr.length -1;
}
finish() {
this._timeStamp = new Date();
this.export();
}
};
module.exports = { Log };
/*********** getData.js ***************/
require('dotenv').config({ path: '../.env' });
const { owner, repo, octokitConfig, octokitAuth } = require('../constants');
const octokit = require('@octokit/rest')(octokitConfig);
const { getPRs, getUserInput } = require('../get-prs');
const { rateLimiter, savePrData } = require('../utils');
const { Log } = require('./log');
octokit.authenticate(octokitAuth);
const log = new Log();
(async () => {
const { firstPR, lastPR } = await getUserInput();
const prPropsToGet = ['number', 'user','files'];
const { openPRs } = await getPRs(firstPR, lastPR, prPropsToGet);
if (openPRs.length) {
console.log('Getting files...');
for (let count in openPRs) {
let { number, user: { login: username } } = openPRs[count];
const { data: prFiles } = await octokit.pullRequests.listFiles({ owner, repo, number });
const filenames = prFiles.map(({ filename }) => filename);
log.add(number, { number, username, filenames });
await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500);
}
}
})()
.then(() => {
log.finish();
console.log('Script completed');
})
.catch(err => {
log.finish();
console.log(err)
})
require('dotenv').config({ path: '../.env' });
const fetch = require('node-fetch');
@ -101,7 +28,7 @@ const allowedLangDirNames = [
octokit.authenticate(octokitAuth);
const log = new ProcessingLog();
const log = new ProcessingLog('all-frontmatter-checks');
const labeler = async (number, prFiles, currentLabels, guideFolderErrorsComment) => {
const labelsToAdd = {}; // holds potential labels to add based on file path
@ -167,18 +94,16 @@ const guideFolderChecks = async (number, prFiles, user) => {
console.log('Starting frontmatter checks process...');
for (let count in openPRs) {
let { number, labels: currentLabels, user: { login: username } } = openPRs[count];
log.add(number, 'comment');
log.add(number, 'labels');
const { data: prFiles } = await octokit.pullRequests.listFiles({ owner, repo, number });
const guideFolderErrorsComment = await guideFolderChecks(number, prFiles, username);
const commentLogVal = guideFolderErrorsComment ? guideFolderErrorsComment : 'none';
log.update(number, 'comment', commentLogVal)
const labelsAdded = await labeler(number, prFiles, currentLabels, guideFolderErrorsComment);
const labelLogVal = labelsAdded.length ? labelsAdded : 'none added';
log.update(number, 'labels', labelLogVal);
log.update(number, { comment: commentLogVal, labels: labelLogVal });
await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500);
}
}
})()

7
one-off-scripts/add-test-locally-label.js
View File

@ -15,7 +15,7 @@ const { rateLimiter, savePrData, ProcessingLog } = require('../utils');
octokit.authenticate(octokitAuth);
const log = new ProcessingLog();
const log = new ProcessingLog('all-locally-tested-labels');
(async () => {
const { firstPR, lastPR } = await getUserInput();
@ -28,7 +28,6 @@ const log = new ProcessingLog();
console.log('Starting labeling process...');
for (let count in openPRs) {
let { number, labels } = openPRs[count];
log.add(number, 'labels');
const labelsToAdd = {}; // holds potential labels to add based on file path
const existingLabels = labels.map(({ name }) => name);
if (existingLabels.includes('scope: curriculum')) {
@ -38,14 +37,14 @@ const log = new ProcessingLog();
/* this next section only adds needed labels which are NOT currently on the PR. */
const newLabels = Object.keys(labelsToAdd).filter(label => !existingLabels.includes(label));
if (newLabels.length) {
log.update(number, 'labels', newLabels);
log.add(number, { labels: newLabels });
if (process.env.PRODUCTION_RUN === 'true') {
addLabels(number, newLabels, log);
await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500);
}
}
else {
log.update(number, 'labels', 'none added');
log.add(number, { labels: 'none added' });
}
}
}

26
sweeper.js
View File

@ -1,40 +1,50 @@
/*
This script was originally created to iterate over all open PRs to label and comment
on specific PR errors (i.e. guide related filenmame syntax and frontmatter).
Since the first run which covered over 10,000+ PRs, it is curently ran every couple of days
for just the most recent PRs.
To run the script for a specific range (i.e. label and comment on guide errors),
run `node sweeper.js range startingPrNumber endingPrNumber`
*/
const { owner, repo, octokitConfig, octokitAuth } = require('./constants');
const octokit = require('@octokit/rest')(octokitConfig);
const { getPRs, getUserInput } = require('./get-prs');
const { guideFolderChecks } = require('./validation');
const { savePrData, ProcessingLog } = require('./utils');
const { savePrData, ProcessingLog, rateLimiter } = require('./utils');
const { labeler } = require('./labeler');
octokit.authenticate(octokitAuth);
const log = new ProcessingLog();
const log = new ProcessingLog('sweeper');
log.start();
console.log('Sweeper started...');
(async () => {
const { firstPR, lastPR } = await getUserInput();
log.setFirstLast({ firstPR, lastPR });
const prPropsToGet = ['number', 'labels', 'user'];
const { openPRs } = await getPRs(firstPR, lastPR, prPropsToGet);
if (openPRs.length) {
savePrData(openPRs, firstPR, lastPR);
log.start();
console.log('Processing PRs...');
for (let count in openPRs) {
let { number, labels: currentLabels, user: { login: username } } = openPRs[count];
log.add(number, 'labels');
log.add(number, 'comment');
const { data: prFiles } = await octokit.pullRequests.listFiles({ owner, repo, number });
const guideFolderErrorsComment = await guideFolderChecks(number, prFiles, username);
const commentLogVal = guideFolderErrorsComment ? guideFolderErrorsComment : 'none';
log.update(number, 'comment', commentLogVal)
const labelsAdded = await labeler(number, prFiles, currentLabels, guideFolderErrorsComment);
const labelLogVal = labelsAdded.length ? labelsAdded : 'none added';
log.update(number, 'labels', labelLogVal);
log.add(number, { comment: commentLogVal, labels: labelLogVal });
await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500);
}
}
})()

45
utils/processing-log.js
View File

@ -1,45 +1,49 @@
require('dotenv').config();
const formatDate = require('date-fns/format');
const path = require('path');
const fs = require('fs');
const { saveToFile } = require('./save-to-file');
class ProcessingLog {
constructor() {
constructor(script) {
this._script = script;
this._start = null;
this._lastUpdate = null;
this._lastPRlogged = null;
this._finish = null;
this._elapsedTime = null;
this._prs = {};
this._logfile = path.resolve(__dirname, `../work-logs/${this.getRunType()}_open-prs-processed.json`);
this._prCount = null;
this._logfile = path.resolve(__dirname, `../work-logs/data-for_${this.getRunType()}_${this._script}.json`);
}
getRunType() {
return process.env.PRODUCTION_RUN === 'true' ? 'production' : 'test';
}
import() {
return JSON.parse(fs.readFileSync(this._logfile, 'utf8'));
}
export() {
let sortedPRs = Object.keys(this._prs)
.sort((a, b) => a - b)
.map(num => ({ [num]: this._prs[num] }));
.map(num => ({ number: num, data: this._prs[num] }));
const log = {
start: this._start,
finish: this._finish,
elapsedTime: this._elapsedTime,
prCount: sortedPRs.length,
firstPR: this._firstPR,
lastPR: this._lastPR,
prs: sortedPRs
};
saveToFile(this._logfile, JSON.stringify(log))
}
add(prNum, prop) {
this._prs[prNum] = {};
this._prs[prNum][prop] = null;
add(prNum, props) {
this._prs[prNum] = props;
}
update(prNum, prop, value) {
this._prs[prNum][prop] = value;
setFirstLast({ firstPR, lastPR }) {
this._firstPR = firstPR;
this._lastPR = lastPR;
}
start() {
@ -48,7 +52,20 @@ class ProcessingLog {
finish() {
this._finish = new Date();
const minutesElapsed = (this._finish - this._start) / 1000 / 60;
this._elapsedTime = minutesElapsed.toFixed(2) + ' mins';
this.export();
this.changeFilename();
}
changeFilename() {
const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss');
const newFilename = path.resolve(__dirname,`../work-logs/${this.getRunType()}_${this._script}_${this._firstPR}-${this._lastPR}_${now}.json`);
fs.rename(this._logfile, newFilename, function(err) {
if (err) {
throw(err);
}
});
}
};

2
utils/save-pr-data.js
View File

@ -5,7 +5,7 @@ const { saveToFile } = require('./save-to-file');
const savePrData = (openPRs, firstPR, lastPR) => {
const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss');
const filename = path.resolve(__dirname, `../work-logs/openprs_${firstPR}-${lastPR}_${now}.json`);
const filename = path.resolve(__dirname, `../work-logs/data-for-openprs_${firstPR}-${lastPR}_${now}.json`);
console.log(`# of PRs Retrieved: ${openPRs.length}`);
console.log(`PR Range: ${firstPR} - ${lastPR}`);
saveToFile(filename, JSON.stringify(openPRs));