diff --git a/.gitignore b/.gitignore index c245872855..aa70595094 100644 --- a/.gitignore +++ b/.gitignore @@ -79,4 +79,7 @@ typings/ # CUSTOM FILES # local-work-logs -work-logs/* \ No newline at end of file +work-logs/* + +# local input files +input-files \ No newline at end of file diff --git a/constants.js b/constants.js index f3399a2bac..8c3cb1e052 100644 --- a/constants.js +++ b/constants.js @@ -1,7 +1,26 @@ require('dotenv').config(); + const owner = process.env.REPOSITORY_OWNER; const repo = process.env.REPOSITORY; const fccBaseUrl = `https://github.com/${owner}/${repo}/`; const prBaseUrl = `${fccBaseUrl}pull/`; -module.exports = { owner, repo, fccBaseUrl, prBaseUrl } +const octokitConfig = { + timeout: 0, // 0 means no request timeout + headers: { + accept: 'application/vnd.github.v3+json', + 'user-agent': 'octokit/rest.js v1.2.3' // v1.2.3 will be current version + }, + // custom GitHub Enterprise URL + baseUrl: 'https://api.github.com', + // Node only: advanced request options can be passed as http(s) agent + agent: undefined +} + +const octokitAuth = { + type: 'basic', + username: process.env.USERNAME, + password: process.env.ACCESS_TOKEN +}; + +module.exports = { owner, repo, prBaseUrl, octokitConfig, octokitAuth } diff --git a/failuresToFind.json b/failuresToFind.json deleted file mode 100644 index f74f41366a..0000000000 --- a/failuresToFind.json +++ /dev/null @@ -1,18 +0,0 @@ -[ - { - "error": "missing script: test-ci", - "regex": "missing script: test-ci" - }, - { - "error": "lerna ERR! npm ci exited 1 in '@freecodecamp/api-server", - "regex": "lerna ERR! npm ci exited 1 in '@freecodecamp\/api-server'" - }, - { - "error": "localStorage is not available for opaque origins", - "regex": "localStorage is not available for opaque origins" - }, - { - "error": "Upper case characters found in 3D, all folder names must be lower case", - "regex": "Upper case characters found in 3D, all folder names must be lower case" - } -] diff --git a/fileFunctions.js b/fileFunctions.js deleted file mode 100644 index 8fd7a98493..0000000000 --- a/fileFunctions.js +++ /dev/null @@ -1,15 +0,0 @@ -const fs = require('fs'); - -const saveToFile = (fileName, data) => { - fs.writeFileSync(fileName, data, err => { - if (err) { return console.log(err) } - }); -} - -const openJSONFile = fileName => { - const data = JSON.parse(fs.readFileSync(fileName, 'utf8')); - console.log('Opened JSON file') - return data; -}; - -module.exports = { saveToFile, openJSONFile }; diff --git a/findFailures.js b/findFailures.js deleted file mode 100644 index 657899730c..0000000000 --- a/findFailures.js +++ /dev/null @@ -1,91 +0,0 @@ -require('dotenv').config(); - -const path = require('path'); -const fs = require('fs'); - -const { owner, repo, fccBaseUrl, prBaseUrl } = require('./constants'); -const formatDate = require('date-fns/format'); -const { saveToFile, openJSONFile } = require('./fileFunctions'); -const { octokitConfig, octokitAuth } = require('./octokitConfig'); -const octokit = require('@octokit/rest')(octokitConfig); -const { getOpenPrs, getPrRange } = require('./getOpenPrs'); -const fetch = require('node-fetch'); -const { getStatuses } = require('./getStatuses'); - -octokit.authenticate(octokitAuth); - -//const { PrProcessingLog } = require('./prProcessingLog'); -//const log = new PrProcessingLog(); - -const prPropsToGet = ['number', 'head']; - -const errorsToFind = require('./failuresToFind.json'); - -(async () => { - const { firstPR, lastPR } = await getPrRange(); - const { openPRs } = await getOpenPrs(firstPR, lastPR, prPropsToGet); - - if (openPRs.length) { - console.log(`# of PRs Retrieved: ${openPRs.length}`); - console.log(`PR Range: ${firstPR} - ${lastPR}`); - const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss'); - const fileName = path.resolve(__dirname, `./work-logs/openprs_statuses_urls_${firstPR}-${lastPR}_${now}.json`); - saveToFile(fileName, JSON.stringify(openPRs)); - console.log(`Data saved in file: ${fileName}`); - - //log.start(); - console.log('Starting error finding process...'); - let count = 0; - const maxCount = openPRs.length; - const failed = []; - let interval = setInterval(async () => { - if (count < maxCount ) { - let { number, head: { sha: ref } } = openPRs[count]; - const statuses = await getStatuses(octokit.repos.getStatuses, {owner, repo, ref}); - //log.add(number) - if (statuses.length) { - const { state, target_url } = statuses[0]; // first element contain most recent status - const hasProblem = state === 'failure' || state === 'error'; - if (hasProblem) { - let buildNum = Number(target_url.match(/\/builds\/(\d+)\?/i)[1]); - buildNum++; // full build log file is 1 # higher than buildNum (same as job number) - const travisLogUrl = `https://api.travis-ci.org/v3/job/${buildNum}/log.txt`; - console.log(number + '\'s errors:'); - errorsToFind.forEach(async ({ error: errorDesc, regex }) => { - const response = await fetch(travisLogUrl) - const logText = await response.text(); - regex = RegExp(regex); - if (regex.test(logText)) { - const error = { - errorDesc, - number, - buildLog: travisLogUrl - } - failed.push(error) - console.log(' ' + errorDesc); - } - console.log() - }); - } - } - } - else { - clearInterval(interval); - interval = null; - //log.export(); - } - if (count % 25 === 0) { - //log.export() - } - count++; - }, 1000); - } -})() -.then(() => { - //log.finish(); - console.log('Successfully finding all specified errors.'); -}) -.catch(err => { - //log.finish(); - console.log(err) -}) diff --git a/frontmatterChecks.js b/frontmatterChecks.js deleted file mode 100644 index feb6734971..0000000000 --- a/frontmatterChecks.js +++ /dev/null @@ -1,13 +0,0 @@ -const matter = require('gray-matter'); - -const checkFrontmatter = (fullPath, isTranslation, content) => { - const { data: frontmatter } = matter(content); - let errors = []; - if (!frontmatter || _.isEmpty(frontmatter) || !frontmatter.title) { - errors.push(`${fullPath} is missing \`title key\` frontmatter.`); - } - if (isTranslation && !frontmatter.localeTitle) { - errors.push(`${fullPath} is missing \`localeTitle\`frontmatter.`); - } - return errors; -} diff --git a/getOpenPrs.js b/getOpenPrs.js deleted file mode 100644 index d056fde4cc..0000000000 --- a/getOpenPrs.js +++ /dev/null @@ -1,47 +0,0 @@ -require('dotenv').config(); -const formatDate = require('date-fns/format'); -const { owner, repo } = require('./constants'); -const fs = require('fs'); -const { saveToFile } = require('./fileFunctions'); -const { octokitConfig, octokitAuth } = require('./octokitConfig'); -const octokit = require('@octokit/rest')(octokitConfig); -const { paginate } = require('./paginate'); -const { getOpenPrRange } = require('./openPrStats'); - -octokit.authenticate(octokitAuth); - -const getPrRange = async () => { - let [ n, f, type, start, end ] = process.argv; - let [ firstPR, lastPR ] = await getOpenPrRange().then(data => data); - - if (type !== 'all' && type !== 'range') { - throw `Please specify either all or range for 1st arg.`; - } - if (type === 'range') { - start = parseInt(start); - end = parseInt(end); - if (!start || !end) { - throw `Please specify both a starting PR # (2nd arg) and ending PR # (3rd arg).`; - } - if (start > end) { - throw `Starting PR # must be less than or equal to end PR #.`; - } - if (start < firstPR) { - throw `Starting PR # can not be less than first open PR # (${firstPR})`; - } - firstPR = start - if (end > lastPR) { - throw `Ending PR # can not be greater than last open PR # (${lastPR})`; - } - lastPR = end; - } - return {firstPR, lastPR}; -}; - -const getOpenPrs = async (firstPR, lastPR, prPropsToGet) => { - console.log(`Retrieving PRs (#${firstPR} thru #${lastPR})`); - let openPRs = await paginate(octokit.pullRequests.getAll, octokit, firstPR, lastPR, prPropsToGet); - return { firstPR, lastPR, openPRs }; -} - -module.exports = { getOpenPrs, getPrRange }; diff --git a/getPRs/index.js b/getPRs/index.js new file mode 100644 index 0000000000..8bf2248566 --- /dev/null +++ b/getPRs/index.js @@ -0,0 +1,83 @@ +require('dotenv').config(); +// const formatDate = require('date-fns/format'); + +const { owner, repo, octokitConfig, octokitAuth } = require('../constants'); + +const octokit = require('@octokit/rest')(octokitConfig); +const { getRange } = require('./prStats'); + +octokit.authenticate(octokitAuth); + +const paginate = async function paginate (method, octokit, firstPR, lastPR, prPropsToGet) { + + const prFilter = (prs, first, last, prPropsToGet) => { + const filtered = []; + for (let pr of prs) { + if (pr.number >= first && pr.number <= last) { + const propsObj = prPropsToGet.reduce((obj, prop) => { + obj[prop] = pr[prop]; + return obj; + } ,{}); + filtered.push(propsObj); + } + if (pr.number >= last) { + done = true; + return filtered; + } + } + return filtered; + }; + + const methodProps = { + owner, repo, state: 'open', + base: 'master', sort: 'created', + direction: 'asc', page: 1, per_page: 100 + }; + + let done = false; // will be true when lastPR is seen in paginated results + let response = await method(methodProps); + let { data } = response; + data = prFilter(data, firstPR, lastPR, prPropsToGet); + while (octokit.hasNextPage(response) && !done ) { + response = await octokit.getNextPage(response); + let dataFiltered = prFilter(response.data, firstPR, lastPR, prPropsToGet); + data = data.concat(dataFiltered); + } + return data; +}; + +const getUserInput = async () => { + let [ n, f, type, start, end ] = process.argv; + let [ firstPR, lastPR ] = await getRange().then(data => data); + + if (type !== 'all' && type !== 'range') { + throw `Please specify either all or range for 1st arg.`; + } + if (type === 'range') { + start = parseInt(start); + end = parseInt(end); + if (!start || !end) { + throw `Please specify both a starting PR # (2nd arg) and ending PR # (3rd arg).`; + } + if (start > end) { + throw `Starting PR # must be less than or equal to end PR #.`; + } + if (start < firstPR) { + throw `Starting PR # can not be less than first open PR # (${firstPR})`; + } + firstPR = start + if (end > lastPR) { + throw `Ending PR # can not be greater than last open PR # (${lastPR})`; + } + lastPR = end; + } + return {firstPR, lastPR}; +}; + +const getPRs = async (firstPR, lastPR, prPropsToGet) => { + console.log(`Retrieving PRs (#${firstPR} thru #${lastPR})`); + let openPRs = await paginate(octokit.pullRequests.list, octokit, firstPR, lastPR, prPropsToGet); + return { firstPR, lastPR, openPRs }; +} + +module.exports = { getPRs, getUserInput }; diff --git a/getPRs/prStats.js b/getPRs/prStats.js new file mode 100644 index 0000000000..07d43f49ab --- /dev/null +++ b/getPRs/prStats.js @@ -0,0 +1,39 @@ +require('dotenv').config(); + +const { owner, repo, octokitConfig, octokitAuth } = require('../constants'); + +const octokit = require('@octokit/rest')(octokitConfig); + +octokit.authenticate(octokitAuth); + +const state = `open`; +const base = 'master'; +const sort = 'created'; +const page = 1; +const per_page = 1; + +const getCount = async () => { + const { data } = await octokit.search.issues({ + q: `repo:${owner}/${repo}+is:open+type:pr+base:master`, + sort: 'created', order: 'asc', page: 1, per_page: 1 + }); + return data.length; +}; + +const getFirst = async () => { + let methodProps = {owner, repo, state, base, sort, direction: 'asc', page, per_page}; + let response = await octokit.pullRequests.list(methodProps); + return response.data[0].number; +}; + +const getRange = async () => { + let methodProps = {owner, repo, state, base, sort, direction: 'asc', page, per_page}; + let response = await octokit.pullRequests.list(methodProps); + const firstPR = response.data[0].number; + methodProps = {owner, repo, state, base, sort, direction: 'desc', page, per_page}; + response = await octokit.pullRequests.list(methodProps); + const lastPR = response.data[0].number; + return [firstPR, lastPR]; +}; + +module.exports = { getCount, getFirst, getRange }; diff --git a/getStatuses.js b/getStatuses.js deleted file mode 100644 index 1d19f5bdd9..0000000000 --- a/getStatuses.js +++ /dev/null @@ -1,13 +0,0 @@ -require('dotenv').config(); -const { owner, repo } = require('./constants'); -const { octokitConfig, octokitAuth } = require('./octokitConfig'); -const octokit = require('@octokit/rest')(octokitConfig); - -octokit.authenticate(octokitAuth); - -const getStatuses = async function getStatuses (method, methodProps) { - const { data } = await method(methodProps); - return data; -}; - -module.exports = { getStatuses } diff --git a/guideFolderChecks.js b/guideFolderChecks.js deleted file mode 100644 index f4b801dde1..0000000000 --- a/guideFolderChecks.js +++ /dev/null @@ -1,90 +0,0 @@ -const dedent = require("dedent"); - -const allowedLangDirNames = [ - "arabic", - "chinese", - "english", - "portuguese", - "russian", - "spanish" -]; - -const createErrorMsg = (errors, user) => { - let errorMsgHeader = dedent` - Hi @${user}, - - Thanks for this pull request (PR). - - Unfortunately, these changes have failed some of our recommended guidelines. Please correct the issue(s) listed below, so we can consider merging your content. - - | Issue | Description | File Path | - | :---: | --- | --- | - `; - - let errorMsgTable = errors.reduce((msgStr, { msg, fullPath }, idx) => { - return (msgStr += "\n" + dedent` - | ${idx + 1} | ${msg} | ${fullPath} | - `); - }, ""); - - let errorMsgFooter = dedent` - P.S: I am just friendly bot. You should reach out to the [Contributors Chat room](https://gitter.im/FreeCodeCamp/Contributors) for more help. - `; - - return errorMsgHeader + errorMsgTable + "\n\n" + errorMsgFooter; -}; - -const checkPath = fullPath => { - let errorMsgs = []; - const remaining = fullPath.split("/"); - - if (!allowedLangDirNames.includes(remaining[1])) { - errorMsgs.push({ - msg: `\`${remaining[1]}\` is not a valid language directory`, - fullPath - }); - } - - if (remaining[remaining.length - 1] !== "index.md") { - errorMsgs.push({ - msg: `\`${remaining[remaining.length - 1]}\` is not a valid file name, please use \`index.md\``, - fullPath - }); - } else if (remaining[2] === "index.md") { - errorMsgs.push({ - msg: `This file is not in its own sub-directory`, - fullPath - }); - } - - const dirName = fullPath.replace("/index.md", ""); - if (dirName.replace(/(\s|\_)/, "") !== dirName) { - errorMsgs.push({ - msg: `Invalid character found in a directory name, please use \`-\` as separators`, - fullPath - }); - } - - if (dirName.toLowerCase() !== dirName) { - errorMsgs.push({ - msg: `Upper case characters found in the file path, all file paths must be lower case`, - fullPath - }); - } - - return errorMsgs; -}; - -const guideFolderChecks = (prFiles, user) => { - const prErrors = prFiles.reduce((errorsFound, { filename: fullPath }) => { - let newErrors; - if (/^guide\//.test(fullPath)) { - newErrors = checkPath(fullPath); - } - return newErrors ? errorsFound.concat(newErrors) : errorsFound; - }, []); - - return prErrors.length ? createErrorMsg(prErrors, user) : null; -}; - -module.exports = { guideFolderChecks }; diff --git a/labelOpenPrs.js b/labelOpenPrs.js deleted file mode 100644 index 3f8d6a845f..0000000000 --- a/labelOpenPrs.js +++ /dev/null @@ -1,97 +0,0 @@ -require('dotenv').config(); - -const path = require('path'); -const fs = require('fs'); -const formatDate = require('date-fns/format'); - -const { owner, repo, fccBaseUrl, prBaseUrl } = require('./constants'); -const { saveToFile, openJSONFile } = require('./fileFunctions'); -const { octokitConfig, octokitAuth } = require('./octokitConfig'); -const octokit = require('@octokit/rest')(octokitConfig); -const { getOpenPrs, getPrRange } = require('./getOpenPrs'); -const { validLabels } = require('./validLabels'); -const { addLabels } = require('./addLabels'); -const { guideFolderChecks } = require('./guideFolderChecks'); -const { addComment } = require('./addComment'); -const { rateLimiter, savePrData } = require('./utils'); - -octokit.authenticate(octokitAuth); - -const { PrProcessingLog } = require('./prProcessingLog'); -const log = new PrProcessingLog(); - -const prPropsToGet = ['number', 'labels', 'user']; - -(async () => { - const { firstPR, lastPR } = await getPrRange(); - const { openPRs } = await getOpenPrs(firstPR, lastPR, prPropsToGet); - - if (openPRs.length) { - savePrData(openPRs, firstPR, lastPR); - - log.start(); - console.log('Starting labeling process...'); - for (let count = 0; count < openPRs.length; count++) { - let { number, labels, user: { login: username } } = openPRs[count]; - const { data: prFiles } = await octokit.pullRequests.getFiles({ owner, repo, number }); - log.add(number, 'labels', 'comment'); - const labelsToAdd = {}; // holds potential labels to add based on file path - - const guideFolderErrorsComment = guideFolderChecks(prFiles, username); - if (guideFolderErrorsComment) { - log.update(number, 'comment', guideFolderErrorsComment); - if (process.env.PRODUCTION_RUN === 'true') { - const result = await addComment(number, guideFolderErrorsComment); - } - await rateLimiter(process.env.RATELIMIT_INTERVAL | 1500); - labelsToAdd['status: needs update'] = 1; - } - else { - log.update(number, 'comment', 'not added'); - } - - const existingLabels = labels.map(({ name }) => name); - - prFiles.forEach(({ filename }) => { - /* remove '/challenges' from filename so language variable hold the language */ - const filenameReplacement = filename.replace(/^curriculum\/challenges\//, 'curriculum\/'); - const regex = /^(docs|curriculum|guide)(?:\/)(arabic|chinese|portuguese|russian|spanish)?\/?/ - const [ _, articleType, language ] = filenameReplacement.match(regex) || []; // need an array to pass to labelsAdder - - if (articleType && validLabels[articleType]) { - labelsToAdd[validLabels[articleType]] = 1 - } - if (language && validLabels[language]) { - labelsToAdd[validLabels[language]] = 1 - } - if (articleType === 'curriculum') { - labelsToAdd['status: need to test locally'] = 1; - } - }) - - /* this next section only adds needed labels which are NOT currently on the PR. */ - const newLabels = Object.keys(labelsToAdd).filter(label => !existingLabels.includes(label)); - if (newLabels.length) { - log.update(number, 'labels', newLabels); - if (process.env.PRODUCTION_RUN === 'true') { - addLabels(number, newLabels, log); - } - await rateLimiter(process.env.RATELIMIT_INTERVAL | 1500); - } - else { - log.update(number, 'labels', 'none added'); - } - if (count % 25 === 0) { - log.export() - } - } - } -})() -.then(() => { - log.finish(); - console.log('Successfully completed labeling'); -}) -.catch(err => { - log.finish(); - console.log(err) -}) diff --git a/octokitConfig.js b/octokitConfig.js deleted file mode 100644 index 883aacee1b..0000000000 --- a/octokitConfig.js +++ /dev/null @@ -1,19 +0,0 @@ -exports.octokitConfig = { - timeout: 0, // 0 means no request timeout - headers: { - accept: 'application/vnd.github.v3+json', - 'user-agent': 'octokit/rest.js v1.2.3' // v1.2.3 will be current version - }, - // custom GitHub Enterprise URL - baseUrl: 'https://api.github.com', - // Node only: advanced request options can be passed as http(s) agent - agent: undefined -} - -const octokitAuth = { - type: 'basic', - username: process.env.USERNAME, - password: process.env.ACCESS_TOKEN -}; - -module.exports = { octokitAuth }; diff --git a/addTestLocallyLabel.js b/one-off-scripts/addTestLocallyLabel.js similarity index 55% rename from addTestLocallyLabel.js rename to one-off-scripts/addTestLocallyLabel.js index 1e093fd57e..781e6d6e1b 100644 --- a/addTestLocallyLabel.js +++ b/one-off-scripts/addTestLocallyLabel.js @@ -1,33 +1,26 @@ -require('dotenv').config(); +require('dotenv').config({ path: '../.env' }); +const { owner, repo, octokitConfig, octokitAuth } = require('../constants'); -const path = require('path'); -const fs = require('fs'); -const formatDate = require('date-fns/format'); - -const { owner, repo, fccBaseUrl, prBaseUrl } = require('./constants'); -const { saveToFile, openJSONFile } = require('./fileFunctions'); -const { octokitConfig, octokitAuth } = require('./octokitConfig'); const octokit = require('@octokit/rest')(octokitConfig); -const { getOpenPrs, getPrRange } = require('./getOpenPrs'); -const { addLabels } = require('./addLabels'); -const { rateLimiter, savePrData } = require('./utils'); + +const { getPRs, getUserInput } = require('../getPRs'); +const { addLabels } = require('../prTasks'); +const { rateLimiter, savePrData, ProcessingLog } = require('../utils'); octokit.authenticate(octokitAuth); -const { PrProcessingLog } = require('./prProcessingLog'); -const log = new PrProcessingLog(); - -const prPropsToGet = ['number', 'labels']; +const log = new ProcessingLog(); (async () => { - const { firstPR, lastPR } = await getPrRange(); - const { openPRs } = await getOpenPrs(firstPR, lastPR, prPropsToGet); + const { firstPR, lastPR } = await getUserInput(); + const prPropsToGet = ['number', 'labels']; + const { openPRs } = await getPRs(firstPR, lastPR, prPropsToGet); if (openPRs.length) { savePrData(openPRs, firstPR, lastPR); log.start(); console.log('Starting labeling process...'); - for (let count = 0; count < openPRs.length; count++) { + for (let count in openPRs) { let { number, labels } = openPRs[count]; log.add(number, 'labels'); const labelsToAdd = {}; // holds potential labels to add based on file path @@ -35,7 +28,6 @@ const prPropsToGet = ['number', 'labels']; if (existingLabels.includes('scope: curriculum')) { labelsToAdd['status: need to test locally'] = 1; } - log.add(number, 'labels'); /* this next section only adds needed labels which are NOT currently on the PR. */ const newLabels = Object.keys(labelsToAdd).filter(label => !existingLabels.includes(label)); @@ -43,15 +35,12 @@ const prPropsToGet = ['number', 'labels']; log.update(number, 'labels', newLabels); if (process.env.PRODUCTION_RUN === 'true') { addLabels(number, newLabels, log); - await rateLimiter(process.env.RATELIMIT_INTERVAL | 1500); + await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500); } } else { log.update(number, 'labels', 'none added'); } - if (count % 25 === 0) { - log.export() - } } } })() diff --git a/one-off-scripts/findFailures.js b/one-off-scripts/findFailures.js new file mode 100644 index 0000000000..e54b8c3969 --- /dev/null +++ b/one-off-scripts/findFailures.js @@ -0,0 +1,66 @@ +require('dotenv').config({ path: '../.env' }); +const Travis = require('travis-ci'); + +const path = require('path'); +const fetch = require('node-fetch'); + +const { owner, repo, octokitConfig, octokitAuth } = require('../constants'); + +const octokit = require('@octokit/rest')(octokitConfig); +const { getPRs, getUserInput } = require('../getPRs'); +const { savePrData, ProcessingLog } = require('../utils'); + +octokit.authenticate(octokitAuth); + +const log = new ProcessingLog(); + +const errorsToFind = require(path.resolve(__dirname, '../input-files/failuresToFind.json')); + +(async () => { + const { firstPR, lastPR } = await getUserInput(); + const prPropsToGet = ['number', 'head']; + const { openPRs } = await getPRs(firstPR, lastPR, prPropsToGet); + + if (openPRs.length) { + savePrData(openPRs, firstPR, lastPR); + log.start(); + console.log('Starting error finding process...'); + for (let count in openPRs) { + let { number, head: { sha: ref } } = openPRs[count]; + log.add(number, 'error'); + const { data: statuses } = await octokit.repos.getStatuses({ owner, repo, ref }); + + if (statuses.length) { + const { state, target_url } = statuses[0]; // first element contain most recent status + const hasProblem = state === 'failure' || state === 'error'; + if (hasProblem) { + let buildNum = Number(target_url.match(/\/builds\/(\d+)\?/i)[1]);' + const logNumber = 'need to use Travis api to access the full log for the buildNum above' + const travisLogUrl = `https://api.travis-ci.org/v3/job/${logNumber}/log.txt`; + const response = await fetch(travisLogUrl) + const logText = await response.text(); + let found = false; + let error; + for (let { error: errorDesc, regex } of errorsToFind) { + regex = RegExp(regex); + if (regex.test(logText)) { + error = errorDesc; + found = true; + break; + } + } + const errorDesc = error ? error : 'unkown error'; + log.update(number, 'error', { errorDesc, buildLog: travisLogUrl }); + } + } + } + } +})() +.then(() => { + log.finish(); + console.log('Successfully finding all specified errors.'); +}) +.catch(err => { + log.finish(); + console.log(err) +}) diff --git a/openPrStats.js b/openPrStats.js deleted file mode 100644 index 7c370060a6..0000000000 --- a/openPrStats.js +++ /dev/null @@ -1,42 +0,0 @@ -require('dotenv').config(); -const { owner, repo } = require('./constants'); -const { octokitConfig, octokitAuth } = require('./octokitConfig'); -const octokit = require('@octokit/rest')(octokitConfig); - -octokit.authenticate(octokitAuth); - -const getOpenPrCount = async () => { - const { data } = await octokit.search.issues({ - q: 'repo:freeCodeCamp/freeCodeCamp+is:open+type:pr+base:master', - sort: 'created', order: 'asc', page: 1, per_page: 1 - }); - return data.length; -}; - -const getFirstPr = async () => { - const state = `open`; - const base = 'master'; - const sort = 'created'; - const page = 1; - const per_page = 1; - let methodProps = {owner, repo, state, base, sort, direction: 'asc', page, per_page}; - let response = await octokit.pullRequests.getAll(methodProps); - return response.data[0].number; -}; - -const getOpenPrRange = async () => { - const state = `open`; - const base = 'master'; - const sort = 'created'; - const page = 1; - const per_page = 1; - let methodProps = {owner, repo, state, base, sort, direction: 'asc', page, per_page}; - let response = await octokit.pullRequests.getAll(methodProps); - const firstPR = response.data[0].number; - methodProps = {owner, repo, state, base, sort, direction: 'desc', page, per_page}; - response = await octokit.pullRequests.getAll(methodProps); - const lastPR = response.data[0].number; - return [firstPR, lastPR]; -}; - -module.exports = { getOpenPrCount, getFirstPr, getOpenPrRange }; diff --git a/paginate.js b/paginate.js deleted file mode 100644 index 3ea07e3bd8..0000000000 --- a/paginate.js +++ /dev/null @@ -1,41 +0,0 @@ -const { owner, repo } = require('./constants'); - -const paginate = async function paginate (method, octokit, firstPR, lastPR, prPropsToGet) { - - const prFilter = (prs, first, last, prPropsToGet) => { - const filtered = []; - for (let pr of prs) { - if (pr.number >= first && pr.number <= last) { - const propsObj = prPropsToGet.reduce((obj, prop) => { - obj[prop] = pr[prop]; - return obj; - } ,{}); - filtered.push(propsObj); - } - if (pr.number >= last) { - done = true; - return filtered; - } - } - return filtered; - }; - - const methodProps = { - owner, repo, state: 'open', - base: 'master', sort: 'created', - direction: 'asc', page: 1, per_page: 100 - }; - - let done = false; // will be true when lastPR is seen paginated results - let response = await method(methodProps); - let { data } = response; - data = prFilter(data, firstPR, lastPR, prPropsToGet); - while (octokit.hasNextPage(response) && !done ) { - response = await octokit.getNextPage(response); - let dataFiltered = prFilter(response.data, firstPR, lastPR, prPropsToGet); - data = data.concat(dataFiltered); - } - return data; -}; - -module.exports = { paginate }; diff --git a/addComment.js b/prTasks/addComment.js similarity index 74% rename from addComment.js rename to prTasks/addComment.js index 0e0bd70354..2738de72cf 100644 --- a/addComment.js +++ b/prTasks/addComment.js @@ -1,11 +1,11 @@ require('dotenv').config(); -const { owner, repo } = require('./constants'); -const { octokitConfig, octokitAuth } = require('./octokitConfig'); + +const { owner, repo, octokitConfig, octokitAuth } = require('../constants'); const octokit = require('@octokit/rest')(octokitConfig); octokit.authenticate(octokitAuth); -const addComment = async (number, comment, log) => { +const addComment = async (number, comment) => { const result = await octokit.issues.createComment({ owner, repo, number, body: comment }) .catch((err) => { console.log(`PR #${number} had an error when trying to add a comment\n`); @@ -15,7 +15,6 @@ const addComment = async (number, comment, log) => { if (result) { console.log(`PR #${number} successfully added a comment\n`); } - return result; }; diff --git a/addLabels.js b/prTasks/addLabels.js similarity index 75% rename from addLabels.js rename to prTasks/addLabels.js index 48ade4a2e4..1433983cea 100644 --- a/addLabels.js +++ b/prTasks/addLabels.js @@ -1,6 +1,6 @@ require('dotenv').config(); -const { owner, repo } = require('./constants'); -const { octokitConfig, octokitAuth } = require('./octokitConfig'); + +const { owner, repo, octokitConfig, octokitAuth } = require('../constants'); const octokit = require('@octokit/rest')(octokitConfig); octokit.authenticate(octokitAuth); @@ -11,7 +11,7 @@ const addLabels = (number, labels, log) => { console.log(`PR #${number} added ${JSON.stringify(labels)}\n`); }) .catch((err) => { - console.log(`PR #${number} had an error when trying to label with ${JSON.stringify(labels)}\n`); + console.log(`PR #${number} had an error when trying to labels: ${JSON.stringify(labels)}\n`); console.log(err) log.finish() }) diff --git a/prOpenClose.js b/prTasks/closeOpen.js similarity index 78% rename from prOpenClose.js rename to prTasks/closeOpen.js index 8f1e20cae5..930bca8cbe 100644 --- a/prOpenClose.js +++ b/prTasks/closeOpen.js @@ -1,12 +1,11 @@ -/* closes and reopens an open PR with applicable comment */ require('dotenv').config(); -const { owner, repo, fccBaseUrl, prBaseUrl } = require('./constants'); -const { octokitConfig, octokitAuth } = require('./octokitConfig'); + +const { owner, repo, octokitConfig, octokitAuth } = require('../constants'); const octokit = require('@octokit/rest')(octokitConfig); -const { addComment } = require('./addComment'); octokit.authenticate(octokitAuth); +/* closes and reopens an open PR with applicable comment */ const prOpenClose = async (number) => { const result = await octokit.pullRequests.update({ owner, repo , number, state: 'closed', base: 'master' }) .then(() => { diff --git a/prTasks/index.js b/prTasks/index.js new file mode 100644 index 0000000000..120dc2fea6 --- /dev/null +++ b/prTasks/index.js @@ -0,0 +1,5 @@ +const { addComment } = require('./addComment'); +const { addLabels } = require('./addLabels'); +const { closeOpen } = require('./closeOpen'); + +module.exports = { addComment, addLabels, closeOpen }; diff --git a/utils/index.js b/utils/index.js new file mode 100644 index 0000000000..c46cc4803e --- /dev/null +++ b/utils/index.js @@ -0,0 +1,7 @@ +const { rateLimiter } = require('./rateLimiter'); +const { savePrData } = require('./savePrData'); +const { saveToFile } = require('./saveToFile'); +const { openJSONFile } = require('./openJSONFile'); +const { ProcessingLog } = require('./processingLog'); + +module.exports = { rateLimiter, savePrData, saveToFile, openJSONFile, ProcessingLog }; diff --git a/utils/openJSONFile.js b/utils/openJSONFile.js new file mode 100644 index 0000000000..6de44445f0 --- /dev/null +++ b/utils/openJSONFile.js @@ -0,0 +1,9 @@ +const fs = require('fs'); + +const openJSONFile = fileName => { + const data = JSON.parse(fs.readFileSync(fileName, 'utf8')); + console.log('Opened JSON file') + return data; +}; + +module.exports = { openJSONFile }; diff --git a/prProcessingLog.js b/utils/processingLog.js similarity index 81% rename from prProcessingLog.js rename to utils/processingLog.js index ea0087c772..30af0a805e 100644 --- a/prProcessingLog.js +++ b/utils/processingLog.js @@ -1,16 +1,16 @@ const path = require('path'); const fs = require('fs'); -const { saveToFile } = require('./fileFunctions'); +const { saveToFile } = require('./saveToFile'); -class PrProcessingLog { +class ProcessingLog { constructor() { this._start = null; this._lastUpdate = null; this._lastPRlogged = null; this._finish = null; this._prs = {}; - this._logfile = path.resolve(__dirname, `./work-logs/${this.getRunType()}_open-prs-processed.json`); + this._logfile = path.resolve(__dirname, `../work-logs/${this.getRunType()}_open-prs-processed.json`); } getRunType() { @@ -52,4 +52,4 @@ class PrProcessingLog { } }; -module.exports = { PrProcessingLog }; +module.exports = { ProcessingLog }; diff --git a/utils/rateLimiter.js b/utils/rateLimiter.js new file mode 100644 index 0000000000..c9efe8ecf2 --- /dev/null +++ b/utils/rateLimiter.js @@ -0,0 +1,5 @@ +const rateLimiter = (delay) => { + return new Promise(resolve => setTimeout(() => resolve(true), delay)); +}; + +module.exports = { rateLimiter }; diff --git a/utils.js b/utils/savePrData.js similarity index 54% rename from utils.js rename to utils/savePrData.js index ddd6b490e0..5c7b4531bc 100644 --- a/utils.js +++ b/utils/savePrData.js @@ -1,20 +1,15 @@ -const path = require('path'); -const fs = require('fs'); const formatDate = require('date-fns/format'); +const path = require('path'); -const { saveToFile } = require('./fileFunctions'); - -const rateLimiter = (delay) => { - return new Promise(resolve => setTimeout(() => resolve(true), delay)); -}; +const { saveToFile } = require('./saveToFile'); const savePrData = (openPRs, firstPR, lastPR) => { const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss'); - const filename = path.resolve(__dirname, `./work-logs/openprs_${firstPR}-${lastPR}_${now}.json`); + const filename = path.resolve(__dirname, `../work-logs/openprs_${firstPR}-${lastPR}_${now}.json`); console.log(`# of PRs Retrieved: ${openPRs.length}`); console.log(`PR Range: ${firstPR} - ${lastPR}`); saveToFile(filename, JSON.stringify(openPRs)); console.log(`Data saved in file: ${filename}`); }; -module.exports = { rateLimiter, savePrData }; +module.exports = { savePrData }; diff --git a/utils/saveToFile.js b/utils/saveToFile.js new file mode 100644 index 0000000000..ddba7879c2 --- /dev/null +++ b/utils/saveToFile.js @@ -0,0 +1,9 @@ +const fs = require('fs'); + +const saveToFile = (fileName, data) => { + fs.writeFileSync(fileName, data, err => { + if (err) { return console.log(err) } + }); +} + +module.exports = { saveToFile }; diff --git a/validation/guideFolderChecks/checkPath.js b/validation/guideFolderChecks/checkPath.js new file mode 100644 index 0000000000..1ed58b5f8b --- /dev/null +++ b/validation/guideFolderChecks/checkPath.js @@ -0,0 +1,56 @@ +const { frontmatterCheck } = require('./frontmatterCheck'); + +const allowedLangDirNames = [ + "arabic", + "chinese", + "english", + "portuguese", + "russian", + "spanish" +]; + +const checkPath = (fullPath, fileContent) => { + let errorMsgs = []; + const remaining = fullPath.split("/"); + + if (!allowedLangDirNames.includes(remaining[1])) { + errorMsgs.push({ + msg: `\`${remaining[1]}\` is not a valid language directory`, + fullPath + }); + } + + if (remaining[remaining.length - 1] !== "index.md") { + errorMsgs.push({ + msg: `\`${remaining[remaining.length - 1]}\` is not a valid file name, please use \`index.md\``, + fullPath + }); + } else if (remaining[2] === "index.md") { + errorMsgs.push({ + msg: `This file is not in its own sub-directory`, + fullPath + }); + } + + const dirName = fullPath.replace("/index.md", ""); + if (dirName.replace(/(\s|\_)/, "") !== dirName) { + errorMsgs.push({ + msg: `Invalid character found in a directory name, please use \`-\` as separators`, + fullPath + }); + } + + if (dirName.toLowerCase() !== dirName) { + errorMsgs.push({ + msg: `Upper case characters found in the file path, all file paths must be lower case`, + fullPath + }); + } + + const isTranslation = allowedLangDirNames.includes(remaining[1]) && remaining[1] !== 'english'; + const frontMatterErrMsgs = frontmatterCheck(fullPath, isTranslation, fileContent); + + return errorMsgs.concat(frontMatterErrMsgs); +}; + +module.exports = { checkPath }; diff --git a/validation/guideFolderChecks/createErrorMsg.js b/validation/guideFolderChecks/createErrorMsg.js new file mode 100644 index 0000000000..1f54d279f3 --- /dev/null +++ b/validation/guideFolderChecks/createErrorMsg.js @@ -0,0 +1,28 @@ +const dedent = require("dedent"); + +const createErrorMsg = (errors, user) => { + let errorMsgHeader = dedent` + Hi @${user}, + + Thanks for this pull request (PR). + + Unfortunately, these changes have failed some of our recommended guidelines. Please correct the issue(s) listed below, so we can consider merging your content. + + | Issue | Description | File Path | + | :---: | --- | --- | + `; + + let errorMsgTable = errors.reduce((msgStr, { msg, fullPath }, idx) => { + return (msgStr += "\n" + dedent` + | ${idx + 1} | ${msg} | ${fullPath} | + `); + }, ""); + + let errorMsgFooter = dedent` + P.S: I am just friendly bot. You should reach out to the [Contributors Chat room](https://gitter.im/FreeCodeCamp/Contributors) for more help. + `; + + return errorMsgHeader + errorMsgTable + "\n\n" + errorMsgFooter; +}; + +module.exports = { createErrorMsg }; diff --git a/validation/guideFolderChecks/frontmatterCheck.js b/validation/guideFolderChecks/frontmatterCheck.js new file mode 100644 index 0000000000..8080625aff --- /dev/null +++ b/validation/guideFolderChecks/frontmatterCheck.js @@ -0,0 +1,22 @@ +const matter = require('gray-matter'); +const _ = require('lodash'); + +const frontmatterCheck = (fullPath, isTranslation, fileContent) => { + const { data: frontmatter } = matter(fileContent); + let errors = []; + if (!frontmatter || _.isEmpty(frontmatter) || !frontmatter.title) { + errors.push({ + msg: `Missing \`title key\` frontmatter.`, + fullPath + }); + } + if (isTranslation && !frontmatter.localeTitle) { + errors.push({ + msg: `Missing \`localeTitle key\`frontmatter.`, + fullPath + }); + } + return errors; +}; + +module.exports = { frontmatterCheck }; diff --git a/validation/guideFolderChecks/index.js b/validation/guideFolderChecks/index.js new file mode 100644 index 0000000000..ad8b69f237 --- /dev/null +++ b/validation/guideFolderChecks/index.js @@ -0,0 +1,36 @@ +const fetch = require('node-fetch'); + +const { addComment } = require('../../prTasks'); +const { rateLimiter } = require('../../utils'); +const { createErrorMsg } = require('./createErrorMsg'); +const { checkPath } = require('./checkPath'); + +/* check for guide folder issues and add applicable comment */ +const guideFolderChecks = async (number, prFiles, user) => { + let prErrors = []; + for (let { filename: fullPath, raw_url: fileUrl } of prFiles) { + let newErrors; + if (/^guide\//.test(fullPath)) { + const response = await fetch(fileUrl); + const fileContent = await response.text(); + newErrors = checkPath(fullPath, fileContent); + } + if (newErrors) { + prErrors = prErrors.concat(newErrors); + } + } + + if (prErrors.length) { + const comment = createErrorMsg(prErrors, user) + if (process.env.PRODUCTION_RUN === 'true') { + const result = await addComment(number, comment); + } + await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500); + return comment; + } + else { + return null; + } +}; + +module.exports = { guideFolderChecks }; diff --git a/validation/index.js b/validation/index.js new file mode 100644 index 0000000000..26b0864523 --- /dev/null +++ b/validation/index.js @@ -0,0 +1,4 @@ +const { validLabels } = require('./validLabels'); +const { guideFolderChecks } = require('./guideFolderChecks'); + +module.exports = { validLabels, guideFolderChecks }; diff --git a/validLabels.js b/validation/validLabels.js similarity index 100% rename from validLabels.js rename to validation/validLabels.js