From 9f372d014d4d334a4bf7e829eda064716d0e6260 Mon Sep 17 00:00:00 2001
From: Kristofer Koishigawa <scissorsneedfoodtoo@gmail.com>
Date: Mon, 29 Jul 2019 23:11:29 +0900
Subject: [PATCH] feat: news indexing script (#36512)

* feat: Add script to poll Ghost for all published posts and construct index for Algolia.

* feat: Add script to poll Ghost for all published posts and construct index for Algolia.
---
 sample.env                                    |  2 +
 .../data-sources/news/getAllPosts.js          | 62 +++++++++++++++++++
 search-indexing/package-lock.json             | 39 ++++++++++++
 search-indexing/package.json                  |  1 +
 4 files changed, 104 insertions(+)
 create mode 100644 search-indexing/data-sources/news/getAllPosts.js

diff --git a/sample.env b/sample.env
index 17ddccaf25..428dc9d206 100644
--- a/sample.env
+++ b/sample.env
@@ -34,3 +34,5 @@ LOCALE=english
 
 TEST_CHALLENGES_FOR_LANGS=english
 DOCKER_HOST_LOCATION=localhost
+
+GHOST_CLIENT_KEY=123abc
\ No newline at end of file
diff --git a/search-indexing/data-sources/news/getAllPosts.js b/search-indexing/data-sources/news/getAllPosts.js
new file mode 100644
index 0000000000..19272c00cb
--- /dev/null
+++ b/search-indexing/data-sources/news/getAllPosts.js
@@ -0,0 +1,62 @@
+const path = require('path');
+
+const envPath = path.resolve(__dirname, '../../../.env');
+require('dotenv').config({ path: envPath });
+
+const { GHOST_CLIENT_KEY } = process.env;
+
+const axios = require('axios');
+const fs = require('fs');
+
+const getJson = async url => {
+  return axios
+    .get(url)
+    .then(res => res.data)
+    .catch(err => console.log(err));
+};
+
+const constructIndex = async () => {
+  let currPage = 1;
+  let lastPage = 5;
+  const delay = m => new Promise(resolve => setTimeout(resolve, m));
+  const posts = [];
+
+  while (currPage && currPage <= lastPage) {
+    const data = await getJson(
+      `https://www.freecodecamp.org/news/ghost/api/v2/content/posts/?key=${GHOST_CLIENT_KEY}&include=tags,authors&page=${currPage}`
+    );
+
+    data.posts.forEach(post => {
+      const thisPost = {
+        title: post.title,
+        author: {
+          name: post.primary_author.name,
+          url: post.primary_author.url,
+          profileImage: post.primary_author.profile_image
+        },
+        tags: post.tags.map(obj => {
+          return {
+            name: obj.name,
+            url: obj.url
+          };
+        }),
+        url: post.url,
+        featureImage: post.feature_image,
+        ghostId: post.id,
+        publishedAt: post.published_at
+      };
+
+      posts.push(thisPost);
+    });
+
+    currPage = data.meta.pagination.next;
+    lastPage = data.meta.pagination.pages;
+
+    console.log(posts);
+
+    fs.writeFileSync('posts.json', JSON.stringify(posts, null, 2));
+    await delay(1000);
+  }
+};
+
+constructIndex();
diff --git a/search-indexing/package-lock.json b/search-indexing/package-lock.json
index 5a274cabf1..07aea3fb32 100644
--- a/search-indexing/package-lock.json
+++ b/search-indexing/package-lock.json
@@ -662,6 +662,22 @@
       "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==",
       "dev": true
     },
+    "axios": {
+      "version": "0.19.0",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.0.tgz",
+      "integrity": "sha512-1uvKqKQta3KBxIz14F2v06AEHZ/dIoeKfbTRkK1E5oqjDnuEerLmYTgJB5AiQZHJcljpg1TuRzdjDR06qNk0DQ==",
+      "requires": {
+        "follow-redirects": "1.5.10",
+        "is-buffer": "^2.0.2"
+      },
+      "dependencies": {
+        "is-buffer": {
+          "version": "2.0.3",
+          "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz",
+          "integrity": "sha512-U15Q7MXTuZlrbymiz95PJpZxu8IlipAp4dtS3wOdgPXx3mqBnslrWU14kxfHB+Py/+2PVKSr37dMAgM2A4uArw=="
+        }
+      }
+    },
     "babel-jest": {
       "version": "24.8.0",
       "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-24.8.0.tgz",
@@ -1609,6 +1625,29 @@
         "locate-path": "^3.0.0"
       }
     },
+    "follow-redirects": {
+      "version": "1.5.10",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz",
+      "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==",
+      "requires": {
+        "debug": "=3.1.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "ms": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+          "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+        }
+      }
+    },
     "for-in": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
diff --git a/search-indexing/package.json b/search-indexing/package.json
index 5584a8f902..f802dae1b5 100644
--- a/search-indexing/package.json
+++ b/search-indexing/package.json
@@ -28,6 +28,7 @@
   "homepage": "https://github.com/freeCodeCamp/freeCodeCamp#readme",
   "dependencies": {
     "algoliasearch": "^3.25.1",
+    "axios": "^0.19.0",
     "chalk": "^2.3.2",
     "debug": "^4.1.1",
     "dotenv": "^6.2.0",