gaspersic/freeCodeCamp
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix: labler and getOpenPrs working

Browse Source
This commit is contained in:
Randell Dawson
2018-11-07 20:12:04 -08:00
parent a70cf738c5
commit 74989b28bc
5 changed files with 283 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
addLabels.js
View File

@ -6,16 +6,17 @@ const octokit = require('@octokit/rest')(octokitConfig);
octokit.authenticate(octokitAuth);
const addLabels = (number, labels, log) => {
octokit.issues.addLabels({ owner, repo, number, labels })
.then(() => {
log.update(number, true);
console.log(`PR #${number} added ${JSON.stringify(labels)}\n`);
})
.catch((err) => {
console.log(`PR #${number} had an error when trying to label with ${JSON.stringify(labels)}\n`);
console.log(err)
log.finish()
})
console.log(`PR #${number} added ${JSON.stringify(labels)}\n`);
// octokit.issues.addLabels({ owner, repo, number, labels })
// .then(() => {
// log.update(number, true);
// console.log(`PR #${number} added ${JSON.stringify(labels)}\n`);
// })
// .catch((err) => {
// console.log(`PR #${number} had an error when trying to label with ${JSON.stringify(labels)}\n`);
// console.log(err)
// log.finish()
// })
};
exports.addLabels = addLabels;

59
getOpenPrs.js
View File

@ -1,4 +1,5 @@
require('dotenv').config();
const formatDate = require('date-fns/format');
const { owner, repo } = require('./constants');
const fs = require('fs');
const { saveToFile } = require('./fileFunctions');
@ -9,37 +10,39 @@ const { getOpenPrRange } = require('./openPrStats');
octokit.authenticate(octokitAuth);
(async () => {
let [ n, f, startingPR ] = process.argv;
const getPrRange = async () => {
let [ n, f, type, start, end ] = process.argv;
let [ firstPR, lastPR ] = await getOpenPrRange().then(data => data);
if (startingPR) {
startingPR = parseInt(startingPR);
if (startingPR < firstPR) {
if (type !== 'all' && type !== 'range') {
throw `Please specify either all or range for 1st arg.`;
}
if (type === 'range') {
start = parseInt(start);
end = parseInt(end);
if (!start || !end) {
throw `Please specify both a starting PR # (2nd arg) and ending PR # (3rd arg).`;
}
if (start > end) {
throw `Starting PR # must be less than or equal to end PR #.`;
}
if (start < firstPR) {
throw `Starting PR # can not be less than first open PR # (${firstPR})`;
}
firstPR = startingPR
firstPR = start
if (end > lastPR) {
throw `Ending PR # can not be greater than last open PR # (${lastPR})`;
}
lastPR = end;
}
return {firstPR, lastPR};
};
const methodProps = {
owner, repo, state: 'open',
base: 'master', sort: 'created',
direction: 'asc', page: 1, per_page: 100
};
const getOpenPrs = async (firstPR, lastPR, prPropsToGet) => {
console.log(`Retrieving PRs (#${firstPR} thru #${lastPR})`);
let openPRs = await paginate(octokit.pullRequests.getAll, octokit, firstPR, lastPR, prPropsToGet);
return { firstPR, lastPR, openPRs };
}
console.log(`Retrieving PRs (starting at #${firstPR}) ...`);
const openPRs = await paginate(octokit.pullRequests.getAll, methodProps, octokit);
if (startingPR) {
openPRs = openPRs.filter(pr => pr.number >= startingPR);
}
return { firstPR, lastPR, openPRs }
})()
.then(data => {
console.log(`# of PRs Retrieved: ${data.openPRs.length}`);
console.log(`PR Range: ${data.firstPR} - ${data.lastPR}`);
saveToFile('data/open-prs.json', JSON.stringify(data));
})
.catch(err => {
console.log(err);
})
exports.getOpenPrs = getOpenPrs;
exports.getPrRange = getPrRange;

82
labelOpenPrs.js
View File

@ -1,9 +1,11 @@
require('dotenv').config();
const { owner, repo, fccBaseUrl, prBaseUrl } = require('./constants');
const fs = require('fs');
const formatDate = require('date-fns/format');
const { saveToFile, openJSONFile } = require('./fileFunctions');
const { octokitConfig, octokitAuth } = require('./octokitConfig');
const octokit = require('@octokit/rest')(octokitConfig);
const { getOpenPrs, getPrRange } = require('./getOpenPrs');
const { validLabels } = require('./validLabels');
const { addLabels } = require('./addLabels');
@ -22,43 +24,51 @@ const labelsAdder = (number, existingLabels, labelsToAdd, log) => {
const { PrProcessingLog } = require('./prProcessingLog');
const log = new PrProcessingLog();
(async () => {
let [ n, f, fileName ] = process.argv;
const openPRs = openJSONFile(fileName);
log.start();
const { openPRs: prs } = openPRs;
const prPropsToGet = ['number', 'labels'];
console.log('Starting labeling process ...');
const maxCount = 50;
let count = 0
let interval = setInterval(async () => {
const { number, labels } = prs[count];
if (count < maxCount ) {
const { data: prFiles } = await octokit.pullRequests.getFiles({ owner, repo, number });
log.add(number)
const existingLabels = labels.map(({ name }) => name);
/* holds potential labels to add based on file path */
const labelsToAdd = {};
prFiles.forEach(({ filename }) => {
/* remove '/challenges' from filename so variable second (below) will be the language */
const filenameReplacement = filename.replace(/^curriculum\/challenges\//, 'curriculum\/');
const regex = /^(docs|curriculum|guide)(?:\/)(arabic|chinese|portuguese|russian|spanish)?\/?/
const [ _, first, second ] = filenameReplacement.match(regex) || []; // need an array to pass to labelsAdder
if (first && validLabels[first]) { labelsToAdd[validLabels[first]] = 1 }
if (second && validLabels[second]) { labelsToAdd[validLabels[second]] = 1 }
})
labelsAdder(number, existingLabels, labelsToAdd, log);
}
else {
clearInterval(interval);
interval = null;
log.export();
}
if (count % 25 === 0) {
log.export();
}
count++;
}, 1500);
(async () => {
const { firstPR, lastPR } = await getPrRange();
const { openPRs } = await getOpenPrs(firstPR, lastPR, prPropsToGet);
if (openPRs.length) {
console.log(`# of PRs Retrieved: ${openPRs.length}`);
console.log(`PR Range: ${firstPR} - ${lastPR}`);
const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss');
const fileName = `data/openprs_${firstPR}-${lastPR}_${now}.json`;
saveToFile(fileName, JSON.stringify(openPRs));
console.log(`Data saved in file: ${fileName}`);
log.start();
console.log('Starting labeling process...');
let count = 0;
const maxCount = openPRs.length;
let interval = setInterval(async () => {
if (count < maxCount ) {
let { number, labels } = openPRs[count];
const { data: prFiles } = await octokit.pullRequests.getFiles({ owner, repo, number });
log.add(number)
const existingLabels = labels.map(({ name }) => name);
const labelsToAdd = {}; // holds potential labels to add based on file path
prFiles.forEach(({ filename }) => {
/* remove '/challenges' from filename so variable second (below) will be the language */
const filenameReplacement = filename.replace(/^curriculum\/challenges\//, 'curriculum\/');
const regex = /^(docs|curriculum|guide)(?:\/)(arabic|chinese|portuguese|russian|spanish)?\/?/
const [ _, first, second ] = filenameReplacement.match(regex) || []; // need an array to pass to labelsAdder
if (first && validLabels[first]) { labelsToAdd[validLabels[first]] = 1 }
if (second && validLabels[second]) { labelsToAdd[validLabels[second]] = 1 }
})
labelsAdder(number, existingLabels, labelsToAdd, log);
}
else {
clearInterval(interval);
interval = null;
log.export();
}
if (count % 25 === 0) { log.export() }
count++;
}, 1000);
}
})()
.then(() => {
log.finish();

158
package-lock.json generated
View File

@ -1,3 +1,159 @@
{
"lockfileVersion": 1
"requires": true,
"lockfileVersion": 1,
"dependencies": {
"@octokit/rest": {
"version": "15.16.1",
"resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-15.16.1.tgz",
"integrity": "sha512-86RGoibm8AJ3ZBlM0NdMI932wEZ/bdo2eEppHtliEYwJT9hsy5qt+i9HA+T8CC90r4atoFQDrY7mDLBcAjp9ow==",
"requires": {
"before-after-hook": "^1.1.0",
"btoa-lite": "^1.0.0",
"debug": "^3.1.0",
"http-proxy-agent": "^2.1.0",
"https-proxy-agent": "^2.2.0",
"lodash": "^4.17.4",
"node-fetch": "^2.1.1",
"universal-user-agent": "^2.0.0",
"url-template": "^2.0.8"
}
},
"agent-base": {
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz",
"integrity": "sha512-JVwXMr9nHYTUXsBFKUqhJwvlcYU/blreOEUkhNR2eXZIvwd+c+o5V4MgDPKWnMS/56awN3TRzIP+KoPn+roQtg==",
"requires": {
"es6-promisify": "^5.0.0"
}
},
"before-after-hook": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-1.2.0.tgz",
"integrity": "sha512-wI3QtdLppHNkmM1VgRVLCrlWCKk/YexlPicYbXPs4eYdd1InrUCTFsx5bX1iUQzzMsoRXXPpM1r+p7JEJJydag=="
},
"btoa-lite": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/btoa-lite/-/btoa-lite-1.0.0.tgz",
"integrity": "sha1-M3dm2hWAEhD92VbCLpxokaudAzc="
},
"date-fns": {
"version": "1.29.0",
"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-1.29.0.tgz",
"integrity": "sha512-lbTXWZ6M20cWH8N9S6afb0SBm6tMk+uUg6z3MqHPKE9atmsY3kJkTm8vKe93izJ2B2+q5MV990sM2CHgtAZaOw=="
},
"debug": {
"version": "3.2.6",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
"integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
"requires": {
"ms": "^2.1.1"
}
},
"dotenv": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-6.1.0.tgz",
"integrity": "sha512-/veDn2ztgRlB7gKmE3i9f6CmDIyXAy6d5nBq+whO9SLX+Zs1sXEgFLPi+aSuWqUuusMfbi84fT8j34fs1HaYUw=="
},
"es6-promise": {
"version": "4.2.5",
"resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.5.tgz",
"integrity": "sha512-n6wvpdE43VFtJq+lUDYDBFUwV8TZbuGXLV4D6wKafg13ldznKsyEvatubnmUe31zcvelSzOHF+XbaT+Bl9ObDg=="
},
"es6-promisify": {
"version": "5.0.0",
"resolved": "http://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz",
"integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=",
"requires": {
"es6-promise": "^4.0.3"
}
},
"http-proxy-agent": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz",
"integrity": "sha512-qwHbBLV7WviBl0rQsOzH6o5lwyOIvwp/BdFnvVxXORldu5TmjFfjzBcWUWS5kWAZhmv+JtiDhSuQCp4sBfbIgg==",
"requires": {
"agent-base": "4",
"debug": "3.1.0"
},
"dependencies": {
"debug": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
"integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
"requires": {
"ms": "2.0.0"
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
}
}
},
"https-proxy-agent": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz",
"integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==",
"requires": {
"agent-base": "^4.1.0",
"debug": "^3.1.0"
}
},
"lodash": {
"version": "4.17.11",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
"integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
},
"macos-release": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/macos-release/-/macos-release-1.1.0.tgz",
"integrity": "sha512-mmLbumEYMi5nXReB9js3WGsB8UE6cDBWyIO62Z4DNx6GbRhDxHNjA1MlzSpJ2S2KM1wyiPRA0d19uHWYYvMHjA=="
},
"ms": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
"integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
},
"node-fetch": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.2.1.tgz",
"integrity": "sha512-ObXBpNCD3A/vYQiQtEWl7DuqjAXjfptYFuGHLdPl5U19/6kJuZV+8uMHLrkj3wJrJoyfg4nhgyFixZdaZoAiEQ=="
},
"os-name": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/os-name/-/os-name-2.0.1.tgz",
"integrity": "sha1-uaOGNhwXrjohc27wWZQFyajF3F4=",
"requires": {
"macos-release": "^1.0.0",
"win-release": "^1.0.0"
}
},
"semver": {
"version": "5.6.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz",
"integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg=="
},
"universal-user-agent": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-2.0.1.tgz",
"integrity": "sha512-vz+heWVydO0iyYAa65VHD7WZkYzhl7BeNVy4i54p4TF8OMiLSXdbuQe4hm+fmWAsL+rVibaQHXfhvkw3c1Ws2w==",
"requires": {
"os-name": "^2.0.1"
}
},
"url-template": {
"version": "2.0.8",
"resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz",
"integrity": "sha1-/FZaPMy/93MMd19WQflVV5FDnyE="
},
"win-release": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/win-release/-/win-release-1.1.1.tgz",
"integrity": "sha1-X6VeAr58qTTt/BJmVjLoSbcuUgk=",
"requires": {
"semver": "^5.0.1"
}
}
}
}

51
paginate.js
View File

@ -1,14 +1,39 @@
exports.paginate = async function paginate (method, methodProps, octokit, maxPages = 10000) {
let response = await method(methodProps);
let { data } = response;
let pageNum = 1;
while (octokit.hasNextPage(response) && pageNum < maxPages) {
response = await octokit.getNextPage(response);
data = data.concat(response.data);
if (pageNum % 5 === 0) {
console.log(`page #${pageNum}`);
const { owner, repo } = require('./constants');
const paginate = async function paginate (method, octokit, firstPR, lastPR, prPropsToGet) {
const minMaxFilter = (prs, first, last, prPropsToGet) => {
return prs.reduce((filtered, pr) => {
if (pr.number >= first && pr.number <= last) {
const propsObj = prPropsToGet.reduce((obj, prop) => {
obj[prop] = pr[prop];
return obj;
} ,{});
filtered = [propsObj, ...filtered];
}
pageNum++;
}
return data;
}
return filtered;
}, []);
};
const includesPRNum = (prs, prNum) => prs.some(({ number }) => number >= prNum);
const methodProps = {
owner, repo, state: 'open',
base: 'master', sort: 'created',
direction: 'asc', page: 1, per_page: 100
};
let done = false; // will be true when lastPR is seen paginated results
let response = await method(methodProps);
let { data } = response;
data = minMaxFilter(data, firstPR, lastPR, prPropsToGet);
done = includesPRNum(data, lastPR);
while (octokit.hasNextPage(response) && !done) {
response = await octokit.getNextPage(response);
let dataFiltered = minMaxFilter(response.data, firstPR, lastPR, prPropsToGet);
data = data.concat(dataFiltered);
done = includesPRNum(dataFiltered, lastPR);
}
return data;
};
exports.paginate = paginate;