gaspersic/freeCodeCamp
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat: 1st attempt repo restructure

Browse Source
This commit is contained in:
Randell Dawson
2018-11-20 15:31:28 -08:00
committed by mrugesh mohapatra
parent 51d7b02dae
commit ae7b1eba37
33 changed files with 422 additions and 535 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -79,4 +79,7 @@ typings/
# CUSTOM FILES
# local-work-logs
work-logs/*
work-logs/*
# local input files
input-files

21
constants.js
View File

@ -1,7 +1,26 @@
require('dotenv').config();
const owner = process.env.REPOSITORY_OWNER;
const repo = process.env.REPOSITORY;
const fccBaseUrl = `https://github.com/${owner}/${repo}/`;
const prBaseUrl = `${fccBaseUrl}pull/`;
module.exports = { owner, repo, fccBaseUrl, prBaseUrl }
const octokitConfig = {
timeout: 0, // 0 means no request timeout
headers: {
accept: 'application/vnd.github.v3+json',
'user-agent': 'octokit/rest.js v1.2.3' // v1.2.3 will be current version
},
// custom GitHub Enterprise URL
baseUrl: 'https://api.github.com',
// Node only: advanced request options can be passed as http(s) agent
agent: undefined
}
const octokitAuth = {
type: 'basic',
username: process.env.USERNAME,
password: process.env.ACCESS_TOKEN
};
module.exports = { owner, repo, prBaseUrl, octokitConfig, octokitAuth }

18
failuresToFind.json
View File

@ -1,18 +0,0 @@
[
{
"error": "missing script: test-ci",
"regex": "missing script: test-ci"
},
{
"error": "lerna ERR! npm ci exited 1 in '@freecodecamp/api-server",
"regex": "lerna ERR! npm ci exited 1 in '@freecodecamp\/api-server'"
},
{
"error": "localStorage is not available for opaque origins",
"regex": "localStorage is not available for opaque origins"
},
{
"error": "Upper case characters found in 3D, all folder names must be lower case",
"regex": "Upper case characters found in 3D, all folder names must be lower case"
}
]

15
fileFunctions.js
View File

@ -1,15 +0,0 @@
const fs = require('fs');
const saveToFile = (fileName, data) => {
fs.writeFileSync(fileName, data, err => {
if (err) { return console.log(err) }
});
}
const openJSONFile = fileName => {
const data = JSON.parse(fs.readFileSync(fileName, 'utf8'));
console.log('Opened JSON file')
return data;
};
module.exports = { saveToFile, openJSONFile };

91
findFailures.js
View File

@ -1,91 +0,0 @@
require('dotenv').config();
const path = require('path');
const fs = require('fs');
const { owner, repo, fccBaseUrl, prBaseUrl } = require('./constants');
const formatDate = require('date-fns/format');
const { saveToFile, openJSONFile } = require('./fileFunctions');
const { octokitConfig, octokitAuth } = require('./octokitConfig');
const octokit = require('@octokit/rest')(octokitConfig);
const { getOpenPrs, getPrRange } = require('./getOpenPrs');
const fetch = require('node-fetch');
const { getStatuses } = require('./getStatuses');
octokit.authenticate(octokitAuth);
//const { PrProcessingLog } = require('./prProcessingLog');
//const log = new PrProcessingLog();
const prPropsToGet = ['number', 'head'];
const errorsToFind = require('./failuresToFind.json');
(async () => {
const { firstPR, lastPR } = await getPrRange();
const { openPRs } = await getOpenPrs(firstPR, lastPR, prPropsToGet);
if (openPRs.length) {
console.log(`# of PRs Retrieved: ${openPRs.length}`);
console.log(`PR Range: ${firstPR} - ${lastPR}`);
const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss');
const fileName = path.resolve(__dirname, `./work-logs/openprs_statuses_urls_${firstPR}-${lastPR}_${now}.json`);
saveToFile(fileName, JSON.stringify(openPRs));
console.log(`Data saved in file: ${fileName}`);
//log.start();
console.log('Starting error finding process...');
let count = 0;
const maxCount = openPRs.length;
const failed = [];
let interval = setInterval(async () => {
if (count < maxCount ) {
let { number, head: { sha: ref } } = openPRs[count];
const statuses = await getStatuses(octokit.repos.getStatuses, {owner, repo, ref});
//log.add(number)
if (statuses.length) {
const { state, target_url } = statuses[0]; // first element contain most recent status
const hasProblem = state === 'failure' || state === 'error';
if (hasProblem) {
let buildNum = Number(target_url.match(/\/builds\/(\d+)\?/i)[1]);
buildNum++; // full build log file is 1 # higher than buildNum (same as job number)
const travisLogUrl = `https://api.travis-ci.org/v3/job/${buildNum}/log.txt`;
console.log(number + '\'s errors:');
errorsToFind.forEach(async ({ error: errorDesc, regex }) => {
const response = await fetch(travisLogUrl)
const logText = await response.text();
regex = RegExp(regex);
if (regex.test(logText)) {
const error = {
errorDesc,
number,
buildLog: travisLogUrl
}
failed.push(error)
console.log(' ' + errorDesc);
}
console.log()
});
}
}
}
else {
clearInterval(interval);
interval = null;
//log.export();
}
if (count % 25 === 0) {
//log.export()
}
count++;
}, 1000);
}
})()
.then(() => {
//log.finish();
console.log('Successfully finding all specified errors.');
})
.catch(err => {
//log.finish();
console.log(err)
})

13
frontmatterChecks.js
View File

@ -1,13 +0,0 @@
const matter = require('gray-matter');
const checkFrontmatter = (fullPath, isTranslation, content) => {
const { data: frontmatter } = matter(content);
let errors = [];
if (!frontmatter || _.isEmpty(frontmatter) || !frontmatter.title) {
errors.push(`${fullPath} is missing \`title key\` frontmatter.`);
}
if (isTranslation && !frontmatter.localeTitle) {
errors.push(`${fullPath} is missing \`localeTitle\`frontmatter.`);
}
return errors;
}

47
getOpenPrs.js
View File

@ -1,47 +0,0 @@
require('dotenv').config();
const formatDate = require('date-fns/format');
const { owner, repo } = require('./constants');
const fs = require('fs');
const { saveToFile } = require('./fileFunctions');
const { octokitConfig, octokitAuth } = require('./octokitConfig');
const octokit = require('@octokit/rest')(octokitConfig);
const { paginate } = require('./paginate');
const { getOpenPrRange } = require('./openPrStats');
octokit.authenticate(octokitAuth);
const getPrRange = async () => {
let [ n, f, type, start, end ] = process.argv;
let [ firstPR, lastPR ] = await getOpenPrRange().then(data => data);
if (type !== 'all' && type !== 'range') {
throw `Please specify either all or range for 1st arg.`;
}
if (type === 'range') {
start = parseInt(start);
end = parseInt(end);
if (!start || !end) {
throw `Please specify both a starting PR # (2nd arg) and ending PR # (3rd arg).`;
}
if (start > end) {
throw `Starting PR # must be less than or equal to end PR #.`;
}
if (start < firstPR) {
throw `Starting PR # can not be less than first open PR # (${firstPR})`;
}
firstPR = start
if (end > lastPR) {
throw `Ending PR # can not be greater than last open PR # (${lastPR})`;
}
lastPR = end;
}
return {firstPR, lastPR};
};
const getOpenPrs = async (firstPR, lastPR, prPropsToGet) => {
console.log(`Retrieving PRs (#${firstPR} thru #${lastPR})`);
let openPRs = await paginate(octokit.pullRequests.getAll, octokit, firstPR, lastPR, prPropsToGet);
return { firstPR, lastPR, openPRs };
}
module.exports = { getOpenPrs, getPrRange };

83
getPRs/index.js Normal file
View File

@ -0,0 +1,83 @@
require('dotenv').config();
// const formatDate = require('date-fns/format');
const { owner, repo, octokitConfig, octokitAuth } = require('../constants');
const octokit = require('@octokit/rest')(octokitConfig);
const { getRange } = require('./prStats');
octokit.authenticate(octokitAuth);
const paginate = async function paginate (method, octokit, firstPR, lastPR, prPropsToGet) {
const prFilter = (prs, first, last, prPropsToGet) => {
const filtered = [];
for (let pr of prs) {
if (pr.number >= first && pr.number <= last) {
const propsObj = prPropsToGet.reduce((obj, prop) => {
obj[prop] = pr[prop];
return obj;
} ,{});
filtered.push(propsObj);
}
if (pr.number >= last) {
done = true;
return filtered;
}
}
return filtered;
};
const methodProps = {
owner, repo, state: 'open',
base: 'master', sort: 'created',
direction: 'asc', page: 1, per_page: 100
};
let done = false; // will be true when lastPR is seen in paginated results
let response = await method(methodProps);
let { data } = response;
data = prFilter(data, firstPR, lastPR, prPropsToGet);
while (octokit.hasNextPage(response) && !done ) {
response = await octokit.getNextPage(response);
let dataFiltered = prFilter(response.data, firstPR, lastPR, prPropsToGet);
data = data.concat(dataFiltered);
}
return data;
};
const getUserInput = async () => {
let [ n, f, type, start, end ] = process.argv;
let [ firstPR, lastPR ] = await getRange().then(data => data);
if (type !== 'all' && type !== 'range') {
throw `Please specify either all or range for 1st arg.`;
}
if (type === 'range') {
start = parseInt(start);
end = parseInt(end);
if (!start || !end) {
throw `Please specify both a starting PR # (2nd arg) and ending PR # (3rd arg).`;
}
if (start > end) {
throw `Starting PR # must be less than or equal to end PR #.`;
}
if (start < firstPR) {
throw `Starting PR # can not be less than first open PR # (${firstPR})`;
}
firstPR = start
if (end > lastPR) {
throw `Ending PR # can not be greater than last open PR # (${lastPR})`;
}
lastPR = end;
}
return {firstPR, lastPR};
};
const getPRs = async (firstPR, lastPR, prPropsToGet) => {
console.log(`Retrieving PRs (#${firstPR} thru #${lastPR})`);
let openPRs = await paginate(octokit.pullRequests.list, octokit, firstPR, lastPR, prPropsToGet);
return { firstPR, lastPR, openPRs };
}
module.exports = { getPRs, getUserInput };

39
getPRs/prStats.js Normal file
View File

@ -0,0 +1,39 @@
require('dotenv').config();
const { owner, repo, octokitConfig, octokitAuth } = require('../constants');
const octokit = require('@octokit/rest')(octokitConfig);
octokit.authenticate(octokitAuth);
const state = `open`;
const base = 'master';
const sort = 'created';
const page = 1;
const per_page = 1;
const getCount = async () => {
const { data } = await octokit.search.issues({
q: `repo:${owner}/${repo}+is:open+type:pr+base:master`,
sort: 'created', order: 'asc', page: 1, per_page: 1
});
return data.length;
};
const getFirst = async () => {
let methodProps = {owner, repo, state, base, sort, direction: 'asc', page, per_page};
let response = await octokit.pullRequests.list(methodProps);
return response.data[0].number;
};
const getRange = async () => {
let methodProps = {owner, repo, state, base, sort, direction: 'asc', page, per_page};
let response = await octokit.pullRequests.list(methodProps);
const firstPR = response.data[0].number;
methodProps = {owner, repo, state, base, sort, direction: 'desc', page, per_page};
response = await octokit.pullRequests.list(methodProps);
const lastPR = response.data[0].number;
return [firstPR, lastPR];
};
module.exports = { getCount, getFirst, getRange };

13
getStatuses.js
View File

@ -1,13 +0,0 @@
require('dotenv').config();
const { owner, repo } = require('./constants');
const { octokitConfig, octokitAuth } = require('./octokitConfig');
const octokit = require('@octokit/rest')(octokitConfig);
octokit.authenticate(octokitAuth);
const getStatuses = async function getStatuses (method, methodProps) {
const { data } = await method(methodProps);
return data;
};
module.exports = { getStatuses }

90
guideFolderChecks.js
View File

@ -1,90 +0,0 @@
const dedent = require("dedent");
const allowedLangDirNames = [
"arabic",
"chinese",
"english",
"portuguese",
"russian",
"spanish"
];
const createErrorMsg = (errors, user) => {
let errorMsgHeader = dedent`
Hi @${user},
Thanks for this pull request (PR).
Unfortunately, these changes have failed some of our recommended guidelines. Please correct the issue(s) listed below, so we can consider merging your content.
| Issue | Description | File Path |
| :---: | --- | --- |
`;
let errorMsgTable = errors.reduce((msgStr, { msg, fullPath }, idx) => {
return (msgStr += "\n" + dedent`
| ${idx + 1} | ${msg} | ${fullPath} |
`);
}, "");
let errorMsgFooter = dedent`
P.S: I am just friendly bot. You should reach out to the [Contributors Chat room](https://gitter.im/FreeCodeCamp/Contributors) for more help.
`;
return errorMsgHeader + errorMsgTable + "\n\n" + errorMsgFooter;
};
const checkPath = fullPath => {
let errorMsgs = [];
const remaining = fullPath.split("/");
if (!allowedLangDirNames.includes(remaining[1])) {
errorMsgs.push({
msg: `\`${remaining[1]}\` is not a valid language directory`,
fullPath
});
}
if (remaining[remaining.length - 1] !== "index.md") {
errorMsgs.push({
msg: `\`${remaining[remaining.length - 1]}\` is not a valid file name, please use \`index.md\``,
fullPath
});
} else if (remaining[2] === "index.md") {
errorMsgs.push({
msg: `This file is not in its own sub-directory`,
fullPath
});
}
const dirName = fullPath.replace("/index.md", "");
if (dirName.replace(/(\s|\_)/, "") !== dirName) {
errorMsgs.push({
msg: `Invalid character found in a directory name, please use \`-\` as separators`,
fullPath
});
}
if (dirName.toLowerCase() !== dirName) {
errorMsgs.push({
msg: `Upper case characters found in the file path, all file paths must be lower case`,
fullPath
});
}
return errorMsgs;
};
const guideFolderChecks = (prFiles, user) => {
const prErrors = prFiles.reduce((errorsFound, { filename: fullPath }) => {
let newErrors;
if (/^guide\//.test(fullPath)) {
newErrors = checkPath(fullPath);
}
return newErrors ? errorsFound.concat(newErrors) : errorsFound;
}, []);
return prErrors.length ? createErrorMsg(prErrors, user) : null;
};
module.exports = { guideFolderChecks };

97
labelOpenPrs.js
View File

@ -1,97 +0,0 @@
require('dotenv').config();
const path = require('path');
const fs = require('fs');
const formatDate = require('date-fns/format');
const { owner, repo, fccBaseUrl, prBaseUrl } = require('./constants');
const { saveToFile, openJSONFile } = require('./fileFunctions');
const { octokitConfig, octokitAuth } = require('./octokitConfig');
const octokit = require('@octokit/rest')(octokitConfig);
const { getOpenPrs, getPrRange } = require('./getOpenPrs');
const { validLabels } = require('./validLabels');
const { addLabels } = require('./addLabels');
const { guideFolderChecks } = require('./guideFolderChecks');
const { addComment } = require('./addComment');
const { rateLimiter, savePrData } = require('./utils');
octokit.authenticate(octokitAuth);
const { PrProcessingLog } = require('./prProcessingLog');
const log = new PrProcessingLog();
const prPropsToGet = ['number', 'labels', 'user'];
(async () => {
const { firstPR, lastPR } = await getPrRange();
const { openPRs } = await getOpenPrs(firstPR, lastPR, prPropsToGet);
if (openPRs.length) {
savePrData(openPRs, firstPR, lastPR);
log.start();
console.log('Starting labeling process...');
for (let count = 0; count < openPRs.length; count++) {
let { number, labels, user: { login: username } } = openPRs[count];
const { data: prFiles } = await octokit.pullRequests.getFiles({ owner, repo, number });
log.add(number, 'labels', 'comment');
const labelsToAdd = {}; // holds potential labels to add based on file path
const guideFolderErrorsComment = guideFolderChecks(prFiles, username);
if (guideFolderErrorsComment) {
log.update(number, 'comment', guideFolderErrorsComment);
if (process.env.PRODUCTION_RUN === 'true') {
const result = await addComment(number, guideFolderErrorsComment);
}
await rateLimiter(process.env.RATELIMIT_INTERVAL | 1500);
labelsToAdd['status: needs update'] = 1;
}
else {
log.update(number, 'comment', 'not added');
}
const existingLabels = labels.map(({ name }) => name);
prFiles.forEach(({ filename }) => {
/* remove '/challenges' from filename so language variable hold the language */
const filenameReplacement = filename.replace(/^curriculum\/challenges\//, 'curriculum\/');
const regex = /^(docs|curriculum|guide)(?:\/)(arabic|chinese|portuguese|russian|spanish)?\/?/
const [ _, articleType, language ] = filenameReplacement.match(regex) || []; // need an array to pass to labelsAdder
if (articleType && validLabels[articleType]) {
labelsToAdd[validLabels[articleType]] = 1
}
if (language && validLabels[language]) {
labelsToAdd[validLabels[language]] = 1
}
if (articleType === 'curriculum') {
labelsToAdd['status: need to test locally'] = 1;
}
})
/* this next section only adds needed labels which are NOT currently on the PR. */
const newLabels = Object.keys(labelsToAdd).filter(label => !existingLabels.includes(label));
if (newLabels.length) {
log.update(number, 'labels', newLabels);
if (process.env.PRODUCTION_RUN === 'true') {
addLabels(number, newLabels, log);
}
await rateLimiter(process.env.RATELIMIT_INTERVAL | 1500);
}
else {
log.update(number, 'labels', 'none added');
}
if (count % 25 === 0) {
log.export()
}
}
}
})()
.then(() => {
log.finish();
console.log('Successfully completed labeling');
})
.catch(err => {
log.finish();
console.log(err)
})

19
octokitConfig.js
View File

@ -1,19 +0,0 @@
exports.octokitConfig = {
timeout: 0, // 0 means no request timeout
headers: {
accept: 'application/vnd.github.v3+json',
'user-agent': 'octokit/rest.js v1.2.3' // v1.2.3 will be current version
},
// custom GitHub Enterprise URL
baseUrl: 'https://api.github.com',
// Node only: advanced request options can be passed as http(s) agent
agent: undefined
}
const octokitAuth = {
type: 'basic',
username: process.env.USERNAME,
password: process.env.ACCESS_TOKEN
};
module.exports = { octokitAuth };

35
addTestLocallyLabel.js → one-off-scripts/addTestLocallyLabel.js
View File

@ -1,33 +1,26 @@
require('dotenv').config();
require('dotenv').config({ path: '../.env' });
const { owner, repo, octokitConfig, octokitAuth } = require('../constants');
const path = require('path');
const fs = require('fs');
const formatDate = require('date-fns/format');
const { owner, repo, fccBaseUrl, prBaseUrl } = require('./constants');
const { saveToFile, openJSONFile } = require('./fileFunctions');
const { octokitConfig, octokitAuth } = require('./octokitConfig');
const octokit = require('@octokit/rest')(octokitConfig);
const { getOpenPrs, getPrRange } = require('./getOpenPrs');
const { addLabels } = require('./addLabels');
const { rateLimiter, savePrData } = require('./utils');
const { getPRs, getUserInput } = require('../getPRs');
const { addLabels } = require('../prTasks');
const { rateLimiter, savePrData, ProcessingLog } = require('../utils');
octokit.authenticate(octokitAuth);
const { PrProcessingLog } = require('./prProcessingLog');
const log = new PrProcessingLog();
const prPropsToGet = ['number', 'labels'];
const log = new ProcessingLog();
(async () => {
const { firstPR, lastPR } = await getPrRange();
const { openPRs } = await getOpenPrs(firstPR, lastPR, prPropsToGet);
const { firstPR, lastPR } = await getUserInput();
const prPropsToGet = ['number', 'labels'];
const { openPRs } = await getPRs(firstPR, lastPR, prPropsToGet);
if (openPRs.length) {
savePrData(openPRs, firstPR, lastPR);
log.start();
console.log('Starting labeling process...');
for (let count = 0; count < openPRs.length; count++) {
for (let count in openPRs) {
let { number, labels } = openPRs[count];
log.add(number, 'labels');
const labelsToAdd = {}; // holds potential labels to add based on file path
@ -35,7 +28,6 @@ const prPropsToGet = ['number', 'labels'];
if (existingLabels.includes('scope: curriculum')) {
labelsToAdd['status: need to test locally'] = 1;
}
log.add(number, 'labels');
/* this next section only adds needed labels which are NOT currently on the PR. */
const newLabels = Object.keys(labelsToAdd).filter(label => !existingLabels.includes(label));
@ -43,15 +35,12 @@ const prPropsToGet = ['number', 'labels'];
log.update(number, 'labels', newLabels);
if (process.env.PRODUCTION_RUN === 'true') {
addLabels(number, newLabels, log);
await rateLimiter(process.env.RATELIMIT_INTERVAL | 1500);
await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500);
}
}
else {
log.update(number, 'labels', 'none added');
}
if (count % 25 === 0) {
log.export()
}
}
}
})()

66
one-off-scripts/findFailures.js Normal file
View File

@ -0,0 +1,66 @@
require('dotenv').config({ path: '../.env' });
const Travis = require('travis-ci');
const path = require('path');
const fetch = require('node-fetch');
const { owner, repo, octokitConfig, octokitAuth } = require('../constants');
const octokit = require('@octokit/rest')(octokitConfig);
const { getPRs, getUserInput } = require('../getPRs');
const { savePrData, ProcessingLog } = require('../utils');
octokit.authenticate(octokitAuth);
const log = new ProcessingLog();
const errorsToFind = require(path.resolve(__dirname, '../input-files/failuresToFind.json'));
(async () => {
const { firstPR, lastPR } = await getUserInput();
const prPropsToGet = ['number', 'head'];
const { openPRs } = await getPRs(firstPR, lastPR, prPropsToGet);
if (openPRs.length) {
savePrData(openPRs, firstPR, lastPR);
log.start();
console.log('Starting error finding process...');
for (let count in openPRs) {
let { number, head: { sha: ref } } = openPRs[count];
log.add(number, 'error');
const { data: statuses } = await octokit.repos.getStatuses({ owner, repo, ref });
if (statuses.length) {
const { state, target_url } = statuses[0]; // first element contain most recent status
const hasProblem = state === 'failure' || state === 'error';
if (hasProblem) {
let buildNum = Number(target_url.match(/\/builds\/(\d+)\?/i)[1]);'
const logNumber = 'need to use Travis api to access the full log for the buildNum above'
const travisLogUrl = `https://api.travis-ci.org/v3/job/${logNumber}/log.txt`;
const response = await fetch(travisLogUrl)
const logText = await response.text();
let found = false;
let error;
for (let { error: errorDesc, regex } of errorsToFind) {
regex = RegExp(regex);
if (regex.test(logText)) {
error = errorDesc;
found = true;
break;
}
}
const errorDesc = error ? error : 'unkown error';
log.update(number, 'error', { errorDesc, buildLog: travisLogUrl });
}
}
}
}
})()
.then(() => {
log.finish();
console.log('Successfully finding all specified errors.');
})
.catch(err => {
log.finish();
console.log(err)
})

42
openPrStats.js
View File

@ -1,42 +0,0 @@
require('dotenv').config();
const { owner, repo } = require('./constants');
const { octokitConfig, octokitAuth } = require('./octokitConfig');
const octokit = require('@octokit/rest')(octokitConfig);
octokit.authenticate(octokitAuth);
const getOpenPrCount = async () => {
const { data } = await octokit.search.issues({
q: 'repo:freeCodeCamp/freeCodeCamp+is:open+type:pr+base:master',
sort: 'created', order: 'asc', page: 1, per_page: 1
});
return data.length;
};
const getFirstPr = async () => {
const state = `open`;
const base = 'master';
const sort = 'created';
const page = 1;
const per_page = 1;
let methodProps = {owner, repo, state, base, sort, direction: 'asc', page, per_page};
let response = await octokit.pullRequests.getAll(methodProps);
return response.data[0].number;
};
const getOpenPrRange = async () => {
const state = `open`;
const base = 'master';
const sort = 'created';
const page = 1;
const per_page = 1;
let methodProps = {owner, repo, state, base, sort, direction: 'asc', page, per_page};
let response = await octokit.pullRequests.getAll(methodProps);
const firstPR = response.data[0].number;
methodProps = {owner, repo, state, base, sort, direction: 'desc', page, per_page};
response = await octokit.pullRequests.getAll(methodProps);
const lastPR = response.data[0].number;
return [firstPR, lastPR];
};
module.exports = { getOpenPrCount, getFirstPr, getOpenPrRange };

41
paginate.js
View File

@ -1,41 +0,0 @@
const { owner, repo } = require('./constants');
const paginate = async function paginate (method, octokit, firstPR, lastPR, prPropsToGet) {
const prFilter = (prs, first, last, prPropsToGet) => {
const filtered = [];
for (let pr of prs) {
if (pr.number >= first && pr.number <= last) {
const propsObj = prPropsToGet.reduce((obj, prop) => {
obj[prop] = pr[prop];
return obj;
} ,{});
filtered.push(propsObj);
}
if (pr.number >= last) {
done = true;
return filtered;
}
}
return filtered;
};
const methodProps = {
owner, repo, state: 'open',
base: 'master', sort: 'created',
direction: 'asc', page: 1, per_page: 100
};
let done = false; // will be true when lastPR is seen paginated results
let response = await method(methodProps);
let { data } = response;
data = prFilter(data, firstPR, lastPR, prPropsToGet);
while (octokit.hasNextPage(response) && !done ) {
response = await octokit.getNextPage(response);
let dataFiltered = prFilter(response.data, firstPR, lastPR, prPropsToGet);
data = data.concat(dataFiltered);
}
return data;
};
module.exports = { paginate };

7
addComment.js → prTasks/addComment.js
View File

@ -1,11 +1,11 @@
require('dotenv').config();
const { owner, repo } = require('./constants');
const { octokitConfig, octokitAuth } = require('./octokitConfig');
const { owner, repo, octokitConfig, octokitAuth } = require('../constants');
const octokit = require('@octokit/rest')(octokitConfig);
octokit.authenticate(octokitAuth);
const addComment = async (number, comment, log) => {
const addComment = async (number, comment) => {
const result = await octokit.issues.createComment({ owner, repo, number, body: comment })
.catch((err) => {
console.log(`PR #${number} had an error when trying to add a comment\n`);
@ -15,7 +15,6 @@ const addComment = async (number, comment, log) => {
if (result) {
console.log(`PR #${number} successfully added a comment\n`);
}
return result;
};

6
addLabels.js → prTasks/addLabels.js
View File

@ -1,6 +1,6 @@
require('dotenv').config();
const { owner, repo } = require('./constants');
const { octokitConfig, octokitAuth } = require('./octokitConfig');
const { owner, repo, octokitConfig, octokitAuth } = require('../constants');
const octokit = require('@octokit/rest')(octokitConfig);
octokit.authenticate(octokitAuth);
@ -11,7 +11,7 @@ const addLabels = (number, labels, log) => {
console.log(`PR #${number} added ${JSON.stringify(labels)}\n`);
})
.catch((err) => {
console.log(`PR #${number} had an error when trying to label with ${JSON.stringify(labels)}\n`);
console.log(`PR #${number} had an error when trying to labels: ${JSON.stringify(labels)}\n`);
console.log(err)
log.finish()
})

7
prOpenClose.js → prTasks/closeOpen.js
View File

@ -1,12 +1,11 @@
/* closes and reopens an open PR with applicable comment */
require('dotenv').config();
const { owner, repo, fccBaseUrl, prBaseUrl } = require('./constants');
const { octokitConfig, octokitAuth } = require('./octokitConfig');
const { owner, repo, octokitConfig, octokitAuth } = require('../constants');
const octokit = require('@octokit/rest')(octokitConfig);
const { addComment } = require('./addComment');
octokit.authenticate(octokitAuth);
/* closes and reopens an open PR with applicable comment */
const prOpenClose = async (number) => {
const result = await octokit.pullRequests.update({ owner, repo , number, state: 'closed', base: 'master' })
.then(() => {

5
prTasks/index.js Normal file
View File

@ -0,0 +1,5 @@
const { addComment } = require('./addComment');
const { addLabels } = require('./addLabels');
const { closeOpen } = require('./closeOpen');
module.exports = { addComment, addLabels, closeOpen };

7
utils/index.js Normal file
View File

@ -0,0 +1,7 @@
const { rateLimiter } = require('./rateLimiter');
const { savePrData } = require('./savePrData');
const { saveToFile } = require('./saveToFile');
const { openJSONFile } = require('./openJSONFile');
const { ProcessingLog } = require('./processingLog');
module.exports = { rateLimiter, savePrData, saveToFile, openJSONFile, ProcessingLog };

9
utils/openJSONFile.js Normal file
View File

@ -0,0 +1,9 @@
const fs = require('fs');
const openJSONFile = fileName => {
const data = JSON.parse(fs.readFileSync(fileName, 'utf8'));
console.log('Opened JSON file')
return data;
};
module.exports = { openJSONFile };

8
prProcessingLog.js → utils/processingLog.js
View File

@ -1,16 +1,16 @@
const path = require('path');
const fs = require('fs');
const { saveToFile } = require('./fileFunctions');
const { saveToFile } = require('./saveToFile');
class PrProcessingLog {
class ProcessingLog {
constructor() {
this._start = null;
this._lastUpdate = null;
this._lastPRlogged = null;
this._finish = null;
this._prs = {};
this._logfile = path.resolve(__dirname, `./work-logs/${this.getRunType()}_open-prs-processed.json`);
this._logfile = path.resolve(__dirname, `../work-logs/${this.getRunType()}_open-prs-processed.json`);
}
getRunType() {
@ -52,4 +52,4 @@ class PrProcessingLog {
}
};
module.exports = { PrProcessingLog };
module.exports = { ProcessingLog };

5
utils/rateLimiter.js Normal file
View File

@ -0,0 +1,5 @@
const rateLimiter = (delay) => {
return new Promise(resolve => setTimeout(() => resolve(true), delay));
};
module.exports = { rateLimiter };

13
utils.js → utils/savePrData.js
View File

@ -1,20 +1,15 @@
const path = require('path');
const fs = require('fs');
const formatDate = require('date-fns/format');
const path = require('path');
const { saveToFile } = require('./fileFunctions');
const rateLimiter = (delay) => {
return new Promise(resolve => setTimeout(() => resolve(true), delay));
};
const { saveToFile } = require('./saveToFile');
const savePrData = (openPRs, firstPR, lastPR) => {
const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss');
const filename = path.resolve(__dirname, `./work-logs/openprs_${firstPR}-${lastPR}_${now}.json`);
const filename = path.resolve(__dirname, `../work-logs/openprs_${firstPR}-${lastPR}_${now}.json`);
console.log(`# of PRs Retrieved: ${openPRs.length}`);
console.log(`PR Range: ${firstPR} - ${lastPR}`);
saveToFile(filename, JSON.stringify(openPRs));
console.log(`Data saved in file: ${filename}`);
};
module.exports = { rateLimiter, savePrData };
module.exports = { savePrData };

9
utils/saveToFile.js Normal file
View File

@ -0,0 +1,9 @@
const fs = require('fs');
const saveToFile = (fileName, data) => {
fs.writeFileSync(fileName, data, err => {
if (err) { return console.log(err) }
});
}
module.exports = { saveToFile };

56
validation/guideFolderChecks/checkPath.js Normal file
View File

@ -0,0 +1,56 @@
const { frontmatterCheck } = require('./frontmatterCheck');
const allowedLangDirNames = [
"arabic",
"chinese",
"english",
"portuguese",
"russian",
"spanish"
];
const checkPath = (fullPath, fileContent) => {
let errorMsgs = [];
const remaining = fullPath.split("/");
if (!allowedLangDirNames.includes(remaining[1])) {
errorMsgs.push({
msg: `\`${remaining[1]}\` is not a valid language directory`,
fullPath
});
}
if (remaining[remaining.length - 1] !== "index.md") {
errorMsgs.push({
msg: `\`${remaining[remaining.length - 1]}\` is not a valid file name, please use \`index.md\``,
fullPath
});
} else if (remaining[2] === "index.md") {
errorMsgs.push({
msg: `This file is not in its own sub-directory`,
fullPath
});
}
const dirName = fullPath.replace("/index.md", "");
if (dirName.replace(/(\s|\_)/, "") !== dirName) {
errorMsgs.push({
msg: `Invalid character found in a directory name, please use \`-\` as separators`,
fullPath
});
}
if (dirName.toLowerCase() !== dirName) {
errorMsgs.push({
msg: `Upper case characters found in the file path, all file paths must be lower case`,
fullPath
});
}
const isTranslation = allowedLangDirNames.includes(remaining[1]) && remaining[1] !== 'english';
const frontMatterErrMsgs = frontmatterCheck(fullPath, isTranslation, fileContent);
return errorMsgs.concat(frontMatterErrMsgs);
};
module.exports = { checkPath };

28
validation/guideFolderChecks/createErrorMsg.js Normal file
View File

@ -0,0 +1,28 @@
const dedent = require("dedent");
const createErrorMsg = (errors, user) => {
let errorMsgHeader = dedent`
Hi @${user},
Thanks for this pull request (PR).
Unfortunately, these changes have failed some of our recommended guidelines. Please correct the issue(s) listed below, so we can consider merging your content.
| Issue | Description | File Path |
| :---: | --- | --- |
`;
let errorMsgTable = errors.reduce((msgStr, { msg, fullPath }, idx) => {
return (msgStr += "\n" + dedent`
| ${idx + 1} | ${msg} | ${fullPath} |
`);
}, "");
let errorMsgFooter = dedent`
P.S: I am just friendly bot. You should reach out to the [Contributors Chat room](https://gitter.im/FreeCodeCamp/Contributors) for more help.
`;
return errorMsgHeader + errorMsgTable + "\n\n" + errorMsgFooter;
};
module.exports = { createErrorMsg };

22
validation/guideFolderChecks/frontmatterCheck.js Normal file
View File

@ -0,0 +1,22 @@
const matter = require('gray-matter');
const _ = require('lodash');
const frontmatterCheck = (fullPath, isTranslation, fileContent) => {
const { data: frontmatter } = matter(fileContent);
let errors = [];
if (!frontmatter || _.isEmpty(frontmatter) || !frontmatter.title) {
errors.push({
msg: `Missing \`title key\` frontmatter.`,
fullPath
});
}
if (isTranslation && !frontmatter.localeTitle) {
errors.push({
msg: `Missing \`localeTitle key\`frontmatter.`,
fullPath
});
}
return errors;
};
module.exports = { frontmatterCheck };

36
validation/guideFolderChecks/index.js Normal file
View File

@ -0,0 +1,36 @@
const fetch = require('node-fetch');
const { addComment } = require('../../prTasks');
const { rateLimiter } = require('../../utils');
const { createErrorMsg } = require('./createErrorMsg');
const { checkPath } = require('./checkPath');
/* check for guide folder issues and add applicable comment */
const guideFolderChecks = async (number, prFiles, user) => {
let prErrors = [];
for (let { filename: fullPath, raw_url: fileUrl } of prFiles) {
let newErrors;
if (/^guide\//.test(fullPath)) {
const response = await fetch(fileUrl);
const fileContent = await response.text();
newErrors = checkPath(fullPath, fileContent);
}
if (newErrors) {
prErrors = prErrors.concat(newErrors);
}
}
if (prErrors.length) {
const comment = createErrorMsg(prErrors, user)
if (process.env.PRODUCTION_RUN === 'true') {
const result = await addComment(number, comment);
}
await rateLimiter(+process.env.RATELIMIT_INTERVAL | 1500);
return comment;
}
else {
return null;
}
};
module.exports = { guideFolderChecks };

4
validation/index.js Normal file
View File

@ -0,0 +1,4 @@
const { validLabels } = require('./validLabels');
const { guideFolderChecks } = require('./guideFolderChecks');
module.exports = { validLabels, guideFolderChecks };

0
validLabels.js → validation/validLabels.js
View File