diff --git a/findFailures.js b/findFailures.js index 4fe5df6ed7..af7b2a3678 100644 --- a/findFailures.js +++ b/findFailures.js @@ -1,75 +1,88 @@ require('dotenv').config(); const { owner, repo, fccBaseUrl, prBaseUrl } = require('./constants'); const fs = require('fs'); +const formatDate = require('date-fns/format'); +const { saveToFile, openJSONFile } = require('./fileFunctions'); const { octokitConfig, octokitAuth } = require('./octokitConfig'); const octokit = require('@octokit/rest')(octokitConfig); -const { paginate } = require('./paginate'); -const { saveToFile, writeToFile } = './fileFunctions'; +const { getOpenPrs, getPrRange } = require('./getOpenPrs'); const fetch = require('node-fetch'); const { getStatuses } = require('./getStatuses'); octokit.authenticate(octokitAuth); -const findFailures = async (errorsToFind, maxPrs) => { - const methodProps = { - owner, repo, - state: 'open', base: 'master', sort: 'created', - direction: 'asc', page: 1, per_page: 100 - }; - if (maxPRs) { - maxPages = Math.ceil(maxPrs / 100); // limits - } +//const { PrProcessingLog } = require('./prProcessingLog'); +//const log = new PrProcessingLog(); - const allOpenPRs = await paginate(octokit.pullRequests.getAll, methodProps, octokit, maxPages); +const prPropsToGet = ['number', 'head']; - const failedPRs = allOpenPRs.reduce(async (prevPromise, { number: pr, head: { sha: ref }) => { - const failed = await prevPromise; - const statuses = await getStatuses(octokit.repos.getStatuses, {owner, repo, ref}); - if (statuses.length) { - const { state, target_url } = statuses[0]; // first element contain most recent status - const hasProblem = state === 'failure' || state === 'error'; - if (hasProblem) { - let buildNum = Number(target_url.match(/\/builds\/(\d+)\?/i)[1]); - buildNum++; // full build log file is 1 # higher than buildNum (same as job number) - const travisLogUrl = `https://api.travis-ci.org/v3/job/${buildNum}/log.txt`; - errorsToFind.forEach(async ({ errorDesc, errorRegex }) => { - const response = await fetch(travisLogUrl) - const logText = await response.text(); - if (errorRegex.test(logText)) { - failed.push({ - errorDesc, - pr, - buildLog: buildNum - }) +const errorsToFind = require('./failuresToFind.json'); + +(async () => { + const { firstPR, lastPR } = await getPrRange(); + const { openPRs } = await getOpenPrs(firstPR, lastPR, prPropsToGet); + + if (openPRs.length) { + console.log(`# of PRs Retrieved: ${openPRs.length}`); + console.log(`PR Range: ${firstPR} - ${lastPR}`); + const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss'); + const fileName = `data/openprs_statuses_urls_${firstPR}-${lastPR}_${now}.json`; + saveToFile(fileName, JSON.stringify(openPRs)); + console.log(`Data saved in file: ${fileName}`); + + //log.start(); + console.log('Starting error finding process...'); + let count = 0; + const maxCount = openPRs.length; + const failed = []; + let interval = setInterval(async () => { + if (count < maxCount ) { + let { number, head: { sha: ref } } = openPRs[count]; + const statuses = await getStatuses(octokit.repos.getStatuses, {owner, repo, ref}); + //log.add(number) + if (statuses.length) { + const { state, target_url } = statuses[0]; // first element contain most recent status + const hasProblem = state === 'failure' || state === 'error'; + if (hasProblem) { + let buildNum = Number(target_url.match(/\/builds\/(\d+)\?/i)[1]); + buildNum++; // full build log file is 1 # higher than buildNum (same as job number) + const travisLogUrl = `https://api.travis-ci.org/v3/job/${buildNum}/log.txt`; + console.log(number + '\'s errors:'); + errorsToFind.forEach(async ({ error: errorDesc, regex }) => { + const response = await fetch(travisLogUrl) + const logText = await response.text(); + regex = RegExp(regex); + if (regex.test(logText)) { + const error = { + errorDesc, + number, + buildLog: travisLogUrl + } + failed.push(error) + console.log(' ' + errorDesc); + } + console.log() + }); } - }); + } } - } - return failed; - }, Promise.resolve([])); - - return failedPRs; -}; - -/* Main Program */ -const [ n, f, inputFile, outputFile, maxPRs ] = process.argv; -const inputFileMsg = 'Please specify an input file containing errors to find.\n'; -const outputFileMsg = 'Please specify an output file to save the results of the search.\n'; -const maxPRMsg = 'Please specify an integer from 1-4500 (inclusive) for the max # of PR to search.'; -let errors = ''; -if (!inputFile) { errors += inputFileMsg } -if (!outputFile) { errors += outputFileMsg } -if (!maxPRs || parseInt(maxPRs) < 1 || parseInt(maxPRs) > 4500) { - errors += maxPRMsg; -} - -if (errors) { return console.log(errors) } - -fs.readFile(inputFile, 'utf8', (err, errorsToFind) => { - if (err) { throw err } - findFailures(errorsToFind, maxPRs) - .then((failedPRs) => { - saveToFile(outputFile, JSON.stringify(failedPRs)); - console.log(`# PRs matching specified error: ${failedPRs.length}`); - }) -}); + else { + clearInterval(interval); + interval = null; + //log.export(); + } + if (count % 25 === 0) { + //log.export() + } + count++; + }, 1000); + } +})() +.then(() => { + //log.finish(); + console.log('Successfully finding all specified errors.'); +}) +.catch(err => { + //log.finish(); + console.log(err) +}) diff --git a/prOpenClose.js b/prOpenClose.js index a2cb37da7d..74051550cc 100644 --- a/prOpenClose.js +++ b/prOpenClose.js @@ -1,14 +1,23 @@ -const prOpenClose = async () => { - const result = await octokit.pullRequests.update({ owner, repo , number, state: 'closed', base }) +require('dotenv').config(); +const { owner, repo, fccBaseUrl, prBaseUrl } = require('./constants'); +const { octokitConfig, octokitAuth } = require('./octokitConfig'); +const octokit = require('@octokit/rest')(octokitConfig); +octokit.authenticate(octokitAuth); + +const prOpenClose = async (number) => { + const result = await octokit.pullRequests.update({ owner, repo , number, state: 'closed', base: 'master' }) .then(() => { - return octokit.pullRequests.update({ owner, repo , number, state: 'open', base }) + return octokit.pullRequests.update({ owner, repo , number, state: 'open', base: 'master' }) }) .then(() => { - log.update(true) + console.log('success') + //log.update(true) }) - .catch(() => { - log.update(false) + .catch((err) => { + console.log('catch') + console.log(err) + //log.update(false) }) }; -exports.changePrState = changePrState; +exports.prOpenClose = prOpenClose;