fix: labler and getOpenPrs working

2018-11-07 20:12:04 -08:00
parent a70cf738c5
commit 74989b28bc
5 changed files with 283 additions and 88 deletions
--- a/addLabels.js
+++ b/addLabels.js
@ -6,16 +6,17 @@ const octokit = require('@octokit/rest')(octokitConfig);
 octokit.authenticate(octokitAuth);

 const addLabels = (number, labels, log) => {
-  octokit.issues.addLabels({ owner, repo, number, labels })
-  .then(() => {
-    log.update(number, true);
  console.log(`PR #${number} added ${JSON.stringify(labels)}\n`);
-  })
-  .catch((err) => {
-    console.log(`PR #${number} had an error when trying to label with ${JSON.stringify(labels)}\n`);
-    console.log(err)
-    log.finish()
-  })
+  // octokit.issues.addLabels({ owner, repo, number, labels })
+  // .then(() => {
+  //   log.update(number, true);
+  //   console.log(`PR #${number} added ${JSON.stringify(labels)}\n`);
+  // })
+  // .catch((err) => {
+  //   console.log(`PR #${number} had an error when trying to label with ${JSON.stringify(labels)}\n`);
+  //   console.log(err)
+  //   log.finish()
+  // })
 };

 exports.addLabels = addLabels;
--- a/getOpenPrs.js
+++ b/getOpenPrs.js
@ -1,4 +1,5 @@
 require('dotenv').config();
+const formatDate = require('date-fns/format');
 const { owner, repo } = require('./constants');
 const fs = require('fs');
 const { saveToFile } = require('./fileFunctions');
@ -9,37 +10,39 @@ const { getOpenPrRange } = require('./openPrStats');

 octokit.authenticate(octokitAuth);

-(async () => {
-  let [ n, f, startingPR ] = process.argv;
-
+const getPrRange = async () => {
+  let [ n, f, type, start, end ] = process.argv;
  let [ firstPR, lastPR ] = await getOpenPrRange().then(data => data);
-  if (startingPR) {
-    startingPR = parseInt(startingPR);
-    if (startingPR < firstPR)  {
+
+  if (type !== 'all' && type !== 'range') {
+    throw `Please specify either all or range for 1st arg.`;
+  }
+  if (type === 'range') {
+    start = parseInt(start);
+    end = parseInt(end);
+    if (!start || !end) {
+      throw `Please specify both a starting PR # (2nd arg) and ending PR # (3rd arg).`;
+    }
+    if (start > end) {
+      throw `Starting PR # must be less than or equal to end PR #.`;
+    }
+    if (start < firstPR)  {
      throw `Starting PR # can not be less than first open PR # (${firstPR})`;
    }
-    firstPR = startingPR
+    firstPR = start
+    if (end > lastPR)  {
+      throw `Ending PR # can not be greater than last open PR # (${lastPR})`;
    }
-
-  const methodProps = {
-    owner,  repo, state: 'open',
-    base: 'master',  sort: 'created',
-    direction: 'asc', page: 1, per_page: 100
-  };
-
-  console.log(`Retrieving PRs (starting at #${firstPR}) ...`);
-  const openPRs = await paginate(octokit.pullRequests.getAll, methodProps, octokit);
-  if (startingPR) {
-    openPRs = openPRs.filter(pr => pr.number >= startingPR);
+    lastPR = end;
  }
+  return {firstPR, lastPR};
+};

-  return { firstPR, lastPR, openPRs }
-})()
-.then(data => {
-  console.log(`# of PRs Retrieved: ${data.openPRs.length}`);
-  console.log(`PR Range: ${data.firstPR} - ${data.lastPR}`);
-  saveToFile('data/open-prs.json', JSON.stringify(data));
-})
-.catch(err => {
-  console.log(err);
-})
+const getOpenPrs = async (firstPR, lastPR, prPropsToGet) => {
+  console.log(`Retrieving PRs (#${firstPR} thru #${lastPR})`);
+  let openPRs = await paginate(octokit.pullRequests.getAll, octokit, firstPR, lastPR, prPropsToGet);
+  return { firstPR, lastPR, openPRs };
+}
+
+exports.getOpenPrs = getOpenPrs;
+exports.getPrRange = getPrRange;
--- a/labelOpenPrs.js
+++ b/labelOpenPrs.js
@ -1,9 +1,11 @@
 require('dotenv').config();
 const { owner, repo, fccBaseUrl, prBaseUrl } = require('./constants');
 const fs = require('fs');
+const formatDate = require('date-fns/format');
 const { saveToFile, openJSONFile } = require('./fileFunctions');
 const { octokitConfig, octokitAuth } = require('./octokitConfig');
 const octokit = require('@octokit/rest')(octokitConfig);
+const { getOpenPrs, getPrRange } = require('./getOpenPrs');
 const { validLabels } = require('./validLabels');
 const { addLabels } = require('./addLabels');

@ -22,23 +24,32 @@ const labelsAdder = (number, existingLabels, labelsToAdd, log) => {
 const { PrProcessingLog } = require('./prProcessingLog');
 const log = new PrProcessingLog();

-(async () => {
-  let [ n, f, fileName ] = process.argv;
-  const openPRs = openJSONFile(fileName);
-  log.start();
-  const { openPRs: prs } = openPRs;
+const prPropsToGet = ['number', 'labels'];
+
+(async () => {
+  const { firstPR, lastPR } = await getPrRange();
+  const { openPRs } = await getOpenPrs(firstPR, lastPR, prPropsToGet);
+
+  if (openPRs.length) {
+    console.log(`# of PRs Retrieved: ${openPRs.length}`);
+    console.log(`PR Range: ${firstPR} - ${lastPR}`);
+    const now = formatDate(new Date(), 'YYYY-MM-DDTHHmmss');
+    const fileName = `data/openprs_${firstPR}-${lastPR}_${now}.json`;
+    saveToFile(fileName, JSON.stringify(openPRs));
+    console.log(`Data saved in file: ${fileName}`);
+
+    log.start();
+    console.log('Starting labeling process...');
+    let count = 0;
+    const maxCount = openPRs.length;

-  console.log('Starting labeling process ...');
-  const maxCount = 50;
-  let count = 0
    let interval = setInterval(async () => {
-    const { number, labels } = prs[count];
      if (count < maxCount ) {
+        let { number, labels } = openPRs[count];
        const { data: prFiles } = await octokit.pullRequests.getFiles({ owner, repo, number });
        log.add(number)
        const existingLabels = labels.map(({ name }) => name);
-      /* holds potential labels to add based on file path */
-      const labelsToAdd = {};
+        const labelsToAdd = {}; // holds potential labels to add based on file path
        prFiles.forEach(({ filename }) => {
          /* remove '/challenges' from filename so variable second (below) will be the language */
          const filenameReplacement = filename.replace(/^curriculum\/challenges\//, 'curriculum\/');
@ -54,11 +65,10 @@ const log = new PrProcessingLog();
        interval = null;
        log.export();
      }
-    if (count % 25 === 0) {
-      log.export();
-    }
+      if (count % 25 === 0) { log.export() }
      count++;
-  }, 1500);
+    }, 1000);
+  }
 })()
 .then(() => {
  log.finish();
--- a/package-lock.json
+++ b/package-lock.json
@ -1,3 +1,159 @@
 {
-  "lockfileVersion": 1
+  "requires": true,
+  "lockfileVersion": 1,
+  "dependencies": {
+    "@octokit/rest": {
+      "version": "15.16.1",
+      "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-15.16.1.tgz",
+      "integrity": "sha512-86RGoibm8AJ3ZBlM0NdMI932wEZ/bdo2eEppHtliEYwJT9hsy5qt+i9HA+T8CC90r4atoFQDrY7mDLBcAjp9ow==",
+      "requires": {
+        "before-after-hook": "^1.1.0",
+        "btoa-lite": "^1.0.0",
+        "debug": "^3.1.0",
+        "http-proxy-agent": "^2.1.0",
+        "https-proxy-agent": "^2.2.0",
+        "lodash": "^4.17.4",
+        "node-fetch": "^2.1.1",
+        "universal-user-agent": "^2.0.0",
+        "url-template": "^2.0.8"
+      }
+    },
+    "agent-base": {
+      "version": "4.2.1",
+      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz",
+      "integrity": "sha512-JVwXMr9nHYTUXsBFKUqhJwvlcYU/blreOEUkhNR2eXZIvwd+c+o5V4MgDPKWnMS/56awN3TRzIP+KoPn+roQtg==",
+      "requires": {
+        "es6-promisify": "^5.0.0"
+      }
+    },
+    "before-after-hook": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-1.2.0.tgz",
+      "integrity": "sha512-wI3QtdLppHNkmM1VgRVLCrlWCKk/YexlPicYbXPs4eYdd1InrUCTFsx5bX1iUQzzMsoRXXPpM1r+p7JEJJydag=="
+    },
+    "btoa-lite": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/btoa-lite/-/btoa-lite-1.0.0.tgz",
+      "integrity": "sha1-M3dm2hWAEhD92VbCLpxokaudAzc="
+    },
+    "date-fns": {
+      "version": "1.29.0",
+      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-1.29.0.tgz",
+      "integrity": "sha512-lbTXWZ6M20cWH8N9S6afb0SBm6tMk+uUg6z3MqHPKE9atmsY3kJkTm8vKe93izJ2B2+q5MV990sM2CHgtAZaOw=="
+    },
+    "debug": {
+      "version": "3.2.6",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
+      "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
+      "requires": {
+        "ms": "^2.1.1"
+      }
+    },
+    "dotenv": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-6.1.0.tgz",
+      "integrity": "sha512-/veDn2ztgRlB7gKmE3i9f6CmDIyXAy6d5nBq+whO9SLX+Zs1sXEgFLPi+aSuWqUuusMfbi84fT8j34fs1HaYUw=="
+    },
+    "es6-promise": {
+      "version": "4.2.5",
+      "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.5.tgz",
+      "integrity": "sha512-n6wvpdE43VFtJq+lUDYDBFUwV8TZbuGXLV4D6wKafg13ldznKsyEvatubnmUe31zcvelSzOHF+XbaT+Bl9ObDg=="
+    },
+    "es6-promisify": {
+      "version": "5.0.0",
+      "resolved": "http://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz",
+      "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=",
+      "requires": {
+        "es6-promise": "^4.0.3"
+      }
+    },
+    "http-proxy-agent": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz",
+      "integrity": "sha512-qwHbBLV7WviBl0rQsOzH6o5lwyOIvwp/BdFnvVxXORldu5TmjFfjzBcWUWS5kWAZhmv+JtiDhSuQCp4sBfbIgg==",
+      "requires": {
+        "agent-base": "4",
+        "debug": "3.1.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "ms": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+          "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+        }
+      }
+    },
+    "https-proxy-agent": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz",
+      "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==",
+      "requires": {
+        "agent-base": "^4.1.0",
+        "debug": "^3.1.0"
+      }
+    },
+    "lodash": {
+      "version": "4.17.11",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
+      "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
+    },
+    "macos-release": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-1.1.0.tgz",
+      "integrity": "sha512-mmLbumEYMi5nXReB9js3WGsB8UE6cDBWyIO62Z4DNx6GbRhDxHNjA1MlzSpJ2S2KM1wyiPRA0d19uHWYYvMHjA=="
+    },
+    "ms": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
+      "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
+    },
+    "node-fetch": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.2.1.tgz",
+      "integrity": "sha512-ObXBpNCD3A/vYQiQtEWl7DuqjAXjfptYFuGHLdPl5U19/6kJuZV+8uMHLrkj3wJrJoyfg4nhgyFixZdaZoAiEQ=="
+    },
+    "os-name": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/os-name/-/os-name-2.0.1.tgz",
+      "integrity": "sha1-uaOGNhwXrjohc27wWZQFyajF3F4=",
+      "requires": {
+        "macos-release": "^1.0.0",
+        "win-release": "^1.0.0"
+      }
+    },
+    "semver": {
+      "version": "5.6.0",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz",
+      "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg=="
+    },
+    "universal-user-agent": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-2.0.1.tgz",
+      "integrity": "sha512-vz+heWVydO0iyYAa65VHD7WZkYzhl7BeNVy4i54p4TF8OMiLSXdbuQe4hm+fmWAsL+rVibaQHXfhvkw3c1Ws2w==",
+      "requires": {
+        "os-name": "^2.0.1"
+      }
+    },
+    "url-template": {
+      "version": "2.0.8",
+      "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz",
+      "integrity": "sha1-/FZaPMy/93MMd19WQflVV5FDnyE="
+    },
+    "win-release": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/win-release/-/win-release-1.1.1.tgz",
+      "integrity": "sha1-X6VeAr58qTTt/BJmVjLoSbcuUgk=",
+      "requires": {
+        "semver": "^5.0.1"
+      }
+    }
+  }
 }
--- a/paginate.js
+++ b/paginate.js
@ -1,14 +1,39 @@
-exports.paginate = async function paginate (method, methodProps, octokit, maxPages = 10000) {
+const { owner, repo } = require('./constants');
+
+const paginate = async function paginate (method, octokit, firstPR, lastPR, prPropsToGet) {
+  const minMaxFilter = (prs, first, last, prPropsToGet) => {
+    return prs.reduce((filtered, pr) => {
+      if (pr.number >= first && pr.number <= last) {
+        const propsObj = prPropsToGet.reduce((obj, prop) => {
+          obj[prop] = pr[prop];
+          return obj;
+        } ,{});
+        filtered =  [propsObj, ...filtered];
+      }
+      return filtered;
+    }, []);
+  };
+
+  const includesPRNum = (prs, prNum) => prs.some(({ number }) => number >= prNum);
+
+  const methodProps = {
+    owner,  repo, state: 'open',
+    base: 'master',  sort: 'created',
+    direction: 'asc', page: 1, per_page: 100
+  };
+
+  let done = false; // will be true when lastPR is seen paginated results
  let response = await method(methodProps);
  let { data } = response;
-    let pageNum = 1;
-    while (octokit.hasNextPage(response) && pageNum < maxPages) {
+  data = minMaxFilter(data, firstPR, lastPR, prPropsToGet);
+  done = includesPRNum(data, lastPR);
+  while (octokit.hasNextPage(response) && !done) {
    response = await octokit.getNextPage(response);
-      data = data.concat(response.data);
-      if (pageNum % 5 === 0) {
-        console.log(`page #${pageNum}`);
-      }
-      pageNum++;
+    let dataFiltered = minMaxFilter(response.data, firstPR, lastPR, prPropsToGet);
+    data = data.concat(dataFiltered);
+    done = includesPRNum(dataFiltered, lastPR);
  }
  return data;
-}
+};
+
+exports.paginate = paginate;