diff --git a/addLabels.js b/addLabels.js index 6889799d24..22d2aaaccc 100644 --- a/addLabels.js +++ b/addLabels.js @@ -6,16 +6,17 @@ const octokit = require('@octokit/rest')(octokitConfig); octokit.authenticate(octokitAuth); const addLabels = (number, labels, log) => { - octokit.issues.addLabels({ owner, repo, number, labels }) - .then(() => { - log.update(number, true); - console.log(`PR #${number} added ${JSON.stringify(labels)}\n`); - }) - .catch((err) => { - console.log(`PR #${number} had an error when trying to label with ${JSON.stringify(labels)}\n`); - console.log(err) - log.finish() - }) + console.log(`PR #${number} added ${JSON.stringify(labels)}\n`); + // octokit.issues.addLabels({ owner, repo, number, labels }) + // .then(() => { + // log.update(number, true); + // console.log(`PR #${number} added ${JSON.stringify(labels)}\n`); + // }) + // .catch((err) => { + // console.log(`PR #${number} had an error when trying to label with ${JSON.stringify(labels)}\n`); + // console.log(err) + // log.finish() + // }) }; exports.addLabels = addLabels; diff --git a/getOpenPrs.js b/getOpenPrs.js index a4c6e21675..dce3ff9d3f 100644 --- a/getOpenPrs.js +++ b/getOpenPrs.js @@ -1,4 +1,5 @@ require('dotenv').config(); +const formatDate = require('date-fns/format'); const { owner, repo } = require('./constants'); const fs = require('fs'); const { saveToFile } = require('./fileFunctions'); @@ -9,37 +10,39 @@ const { getOpenPrRange } = require('./openPrStats'); octokit.authenticate(octokitAuth); -(async () => { - let [ n, f, startingPR ] = process.argv; - +const getPrRange = async () => { + let [ n, f, type, start, end ] = process.argv; let [ firstPR, lastPR ] = await getOpenPrRange().then(data => data); - if (startingPR) { - startingPR = parseInt(startingPR); - if (startingPR < firstPR) { + + if (type !== 'all' && type !== 'range') { + throw `Please specify either all or range for 1st arg.`; + } + if (type === 'range') { + start = parseInt(start); + end = parseInt(end); + if (!start || !end) { + throw `Please specify both a starting PR # (2nd arg) and ending PR # (3rd arg).`; + } + if (start > end) { + throw `Starting PR # must be less than or equal to end PR #.`; + } + if (start < firstPR) { throw `Starting PR # can not be less than first open PR # (${firstPR})`; } - firstPR = startingPR + firstPR = start + if (end > lastPR) { + throw `Ending PR # can not be greater than last open PR # (${lastPR})`; + } + lastPR = end; } + return {firstPR, lastPR}; +}; - const methodProps = { - owner, repo, state: 'open', - base: 'master', sort: 'created', - direction: 'asc', page: 1, per_page: 100 - }; +const getOpenPrs = async (firstPR, lastPR, prPropsToGet) => { + console.log(`Retrieving PRs (#${firstPR} thru #${lastPR})`); + let openPRs = await paginate(octokit.pullRequests.getAll, octokit, firstPR, lastPR, prPropsToGet); + return { firstPR, lastPR, openPRs }; +} - console.log(`Retrieving PRs (starting at #${firstPR}) ...`); - const openPRs = await paginate(octokit.pullRequests.getAll, methodProps, octokit); - if (startingPR) { - openPRs = openPRs.filter(pr => pr.number >= startingPR); - } - - return { firstPR, lastPR, openPRs } -})() -.then(data => { - console.log(`# of PRs Retrieved: ${data.openPRs.length}`); - console.log(`PR Range: ${data.firstPR} - ${data.lastPR}`); - saveToFile('data/open-prs.json', JSON.stringify(data)); -}) -.catch(err => { - console.log(err); -}) +exports.getOpenPrs = getOpenPrs; +exports.getPrRange = getPrRange; diff --git a/labelOpenPrs.js b/labelOpenPrs.js index ec020693b7..a066f18720 100644 --- a/labelOpenPrs.js +++ b/labelOpenPrs.js @@ -1,9 +1,11 @@ require('dotenv').config(); const { owner, repo, fccBaseUrl, prBaseUrl } = require('./constants'); const fs = require('fs'); +const formatDate = require('date-fns/format'); const { saveToFile, openJSONFile } = require('./fileFunctions'); const { octokitConfig, octokitAuth } = require('./octokitConfig'); const octokit = require('@octokit/rest')(octokitConfig); +const { getOpenPrs, getPrRange } = require('./getOpenPrs'); const { validLabels } = require('./validLabels'); const { addLabels } = require('./addLabels'); @@ -22,43 +24,51 @@ const labelsAdder = (number, existingLabels, labelsToAdd, log) => { const { PrProcessingLog } = require('./prProcessingLog'); const log = new PrProcessingLog(); -(async () => { - let [ n, f, fileName ] = process.argv; - const openPRs = openJSONFile(fileName); - log.start(); - const { openPRs: prs } = openPRs; +const prPropsToGet = ['number', 'labels']; - console.log('Starting labeling process ...'); - const maxCount = 50; - let count = 0 - let interval = setInterval(async () => { - const { number, labels } = prs[count]; - if (count < maxCount ) { - const { data: prFiles } = await octokit.pullRequests.getFiles({ owner, repo, number }); - log.add(number) - const existingLabels = labels.map(({ name }) => name); - /* holds potential labels to add based on file path */ - const labelsToAdd = {}; - prFiles.forEach(({ filename }) => { - /* remove '/challenges' from filename so variable second (below) will be the language */ - const filenameReplacement = filename.replace(/^curriculum\/challenges\//, 'curriculum\/'); - const regex = /^(docs|curriculum|guide)(?:\/)(arabic|chinese|portuguese|russian|spanish)?\/?/ - const [ _, first, second ] = filenameReplacement.match(regex) || []; // need an array to pass to labelsAdder - if (first && validLabels[first]) { labelsToAdd[validLabels[first]] = 1 } - if (second && validLabels[second]) { labelsToAdd[validLabels[second]] = 1 } - }) - labelsAdder(number, existingLabels, labelsToAdd, log); - } - else { - clearInterval(interval); - interval = null; - log.export(); - } - if (count % 25 === 0) { - log.export(); - } - count++; - }, 1500); +(async () => { + const { firstPR, lastPR } = await getPrRange(); + const { openPRs } = await getOpenPrs(firstPR, lastPR, prPropsToGet); + + if (openPRs.length) { + console.log(`# of PRs Retrieved: ${openPRs.length}`); + console.log(`PR Range: ${firstPR} - ${lastPR}`); + const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss'); + const fileName = `data/openprs_${firstPR}-${lastPR}_${now}.json`; + saveToFile(fileName, JSON.stringify(openPRs)); + console.log(`Data saved in file: ${fileName}`); + + log.start(); + console.log('Starting labeling process...'); + let count = 0; + const maxCount = openPRs.length; + + let interval = setInterval(async () => { + if (count < maxCount ) { + let { number, labels } = openPRs[count]; + const { data: prFiles } = await octokit.pullRequests.getFiles({ owner, repo, number }); + log.add(number) + const existingLabels = labels.map(({ name }) => name); + const labelsToAdd = {}; // holds potential labels to add based on file path + prFiles.forEach(({ filename }) => { + /* remove '/challenges' from filename so variable second (below) will be the language */ + const filenameReplacement = filename.replace(/^curriculum\/challenges\//, 'curriculum\/'); + const regex = /^(docs|curriculum|guide)(?:\/)(arabic|chinese|portuguese|russian|spanish)?\/?/ + const [ _, first, second ] = filenameReplacement.match(regex) || []; // need an array to pass to labelsAdder + if (first && validLabels[first]) { labelsToAdd[validLabels[first]] = 1 } + if (second && validLabels[second]) { labelsToAdd[validLabels[second]] = 1 } + }) + labelsAdder(number, existingLabels, labelsToAdd, log); + } + else { + clearInterval(interval); + interval = null; + log.export(); + } + if (count % 25 === 0) { log.export() } + count++; + }, 1000); + } })() .then(() => { log.finish(); diff --git a/package-lock.json b/package-lock.json index 48e341a095..805643d5ca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,3 +1,159 @@ { - "lockfileVersion": 1 + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "@octokit/rest": { + "version": "15.16.1", + "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-15.16.1.tgz", + "integrity": "sha512-86RGoibm8AJ3ZBlM0NdMI932wEZ/bdo2eEppHtliEYwJT9hsy5qt+i9HA+T8CC90r4atoFQDrY7mDLBcAjp9ow==", + "requires": { + "before-after-hook": "^1.1.0", + "btoa-lite": "^1.0.0", + "debug": "^3.1.0", + "http-proxy-agent": "^2.1.0", + "https-proxy-agent": "^2.2.0", + "lodash": "^4.17.4", + "node-fetch": "^2.1.1", + "universal-user-agent": "^2.0.0", + "url-template": "^2.0.8" + } + }, + "agent-base": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz", + "integrity": "sha512-JVwXMr9nHYTUXsBFKUqhJwvlcYU/blreOEUkhNR2eXZIvwd+c+o5V4MgDPKWnMS/56awN3TRzIP+KoPn+roQtg==", + "requires": { + "es6-promisify": "^5.0.0" + } + }, + "before-after-hook": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-1.2.0.tgz", + "integrity": "sha512-wI3QtdLppHNkmM1VgRVLCrlWCKk/YexlPicYbXPs4eYdd1InrUCTFsx5bX1iUQzzMsoRXXPpM1r+p7JEJJydag==" + }, + "btoa-lite": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/btoa-lite/-/btoa-lite-1.0.0.tgz", + "integrity": "sha1-M3dm2hWAEhD92VbCLpxokaudAzc=" + }, + "date-fns": { + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-1.29.0.tgz", + "integrity": "sha512-lbTXWZ6M20cWH8N9S6afb0SBm6tMk+uUg6z3MqHPKE9atmsY3kJkTm8vKe93izJ2B2+q5MV990sM2CHgtAZaOw==" + }, + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "dotenv": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-6.1.0.tgz", + "integrity": "sha512-/veDn2ztgRlB7gKmE3i9f6CmDIyXAy6d5nBq+whO9SLX+Zs1sXEgFLPi+aSuWqUuusMfbi84fT8j34fs1HaYUw==" + }, + "es6-promise": { + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.5.tgz", + "integrity": "sha512-n6wvpdE43VFtJq+lUDYDBFUwV8TZbuGXLV4D6wKafg13ldznKsyEvatubnmUe31zcvelSzOHF+XbaT+Bl9ObDg==" + }, + "es6-promisify": { + "version": "5.0.0", + "resolved": "http://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", + "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", + "requires": { + "es6-promise": "^4.0.3" + } + }, + "http-proxy-agent": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz", + "integrity": "sha512-qwHbBLV7WviBl0rQsOzH6o5lwyOIvwp/BdFnvVxXORldu5TmjFfjzBcWUWS5kWAZhmv+JtiDhSuQCp4sBfbIgg==", + "requires": { + "agent-base": "4", + "debug": "3.1.0" + }, + "dependencies": { + "debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } + } + }, + "https-proxy-agent": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", + "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==", + "requires": { + "agent-base": "^4.1.0", + "debug": "^3.1.0" + } + }, + "lodash": { + "version": "4.17.11", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", + "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" + }, + "macos-release": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-1.1.0.tgz", + "integrity": "sha512-mmLbumEYMi5nXReB9js3WGsB8UE6cDBWyIO62Z4DNx6GbRhDxHNjA1MlzSpJ2S2KM1wyiPRA0d19uHWYYvMHjA==" + }, + "ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + }, + "node-fetch": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.2.1.tgz", + "integrity": "sha512-ObXBpNCD3A/vYQiQtEWl7DuqjAXjfptYFuGHLdPl5U19/6kJuZV+8uMHLrkj3wJrJoyfg4nhgyFixZdaZoAiEQ==" + }, + "os-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/os-name/-/os-name-2.0.1.tgz", + "integrity": "sha1-uaOGNhwXrjohc27wWZQFyajF3F4=", + "requires": { + "macos-release": "^1.0.0", + "win-release": "^1.0.0" + } + }, + "semver": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", + "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==" + }, + "universal-user-agent": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-2.0.1.tgz", + "integrity": "sha512-vz+heWVydO0iyYAa65VHD7WZkYzhl7BeNVy4i54p4TF8OMiLSXdbuQe4hm+fmWAsL+rVibaQHXfhvkw3c1Ws2w==", + "requires": { + "os-name": "^2.0.1" + } + }, + "url-template": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", + "integrity": "sha1-/FZaPMy/93MMd19WQflVV5FDnyE=" + }, + "win-release": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/win-release/-/win-release-1.1.1.tgz", + "integrity": "sha1-X6VeAr58qTTt/BJmVjLoSbcuUgk=", + "requires": { + "semver": "^5.0.1" + } + } + } } diff --git a/paginate.js b/paginate.js index 7acc0555f0..2beb5afdc4 100644 --- a/paginate.js +++ b/paginate.js @@ -1,14 +1,39 @@ -exports.paginate = async function paginate (method, methodProps, octokit, maxPages = 10000) { - let response = await method(methodProps); - let { data } = response; - let pageNum = 1; - while (octokit.hasNextPage(response) && pageNum < maxPages) { - response = await octokit.getNextPage(response); - data = data.concat(response.data); - if (pageNum % 5 === 0) { - console.log(`page #${pageNum}`); +const { owner, repo } = require('./constants'); + +const paginate = async function paginate (method, octokit, firstPR, lastPR, prPropsToGet) { + const minMaxFilter = (prs, first, last, prPropsToGet) => { + return prs.reduce((filtered, pr) => { + if (pr.number >= first && pr.number <= last) { + const propsObj = prPropsToGet.reduce((obj, prop) => { + obj[prop] = pr[prop]; + return obj; + } ,{}); + filtered = [propsObj, ...filtered]; } - pageNum++; - } - return data; -} + return filtered; + }, []); + }; + + const includesPRNum = (prs, prNum) => prs.some(({ number }) => number >= prNum); + + const methodProps = { + owner, repo, state: 'open', + base: 'master', sort: 'created', + direction: 'asc', page: 1, per_page: 100 + }; + + let done = false; // will be true when lastPR is seen paginated results + let response = await method(methodProps); + let { data } = response; + data = minMaxFilter(data, firstPR, lastPR, prPropsToGet); + done = includesPRNum(data, lastPR); + while (octokit.hasNextPage(response) && !done) { + response = await octokit.getNextPage(response); + let dataFiltered = minMaxFilter(response.data, firstPR, lastPR, prPropsToGet); + data = data.concat(dataFiltered); + done = includesPRNum(dataFiltered, lastPR); + } + return data; +}; + +exports.paginate = paginate;