Add sitemap generation and remove extra files

lib/guide.js

@@ -1,6 +1,6 @@
-import guides from "storage/guides";
+import guides from 'storage/guides';
-import authors from "storage/authors";
+import authors from 'storage/authors';
-import siteConfig from "storage/site";
+import siteConfig from 'storage/site';
 
 export const getAllGuides = () => {
   return guides.filter(guide => !guide.draft)

next.config.js

@@ -6,7 +6,7 @@ const {
   getPageRoutes,
   getGuideRoutes,
   getRoadmapRoutes,
-} = require("./path-map");
+} = require("./scripts/path-map");
 
 const withMDX = require('@next/mdx')({
   extension: /\.(md|mdx)?$/,

package.json

@@ -5,10 +5,9 @@
   "license": "BSD-4-Clause",
   "scripts": {
     "dev": "next",
-    "build": "next build",
+    "html": "next build && next export && echo 'roadmap.sh' > out/CNAME && '' > out/.nojekyll",
-    "start": "next start",
-    "html": "next build && next export && echo 'roadmap.sh' > out/CNAME && touch out/.nojekyll",
     "serve:out": "serve out",
+    "sitemap": "node scripts/sitemap.js",
     "deploy": "NODE_DEBUG=gh-pages gh-pages -d out",
     "test": "jest",
     "test:watch": "jest --watch"

scripts/path-map.js

@@ -1,10 +1,10 @@
 const path = require('path');
 const glob = require('glob');
 
-const guides = require('./storage/guides.json');
+const guides = require('../storage/guides.json');
-const roadmaps = require('./storage/roadmaps');
+const roadmaps = require('../storage/roadmaps');
 
-const PAGES_PATH = path.join(__dirname, 'pages');
+const PAGES_PATH = path.join(__dirname, '../pages');
 
 /**
  * Generate the page routes from the page files inside `/pages`

scripts/sitemap.js

@@ -0,0 +1,126 @@
+// This is a development script executed in the build step of pages
+const fs = require('fs');
+const path = require('path');
+const guides = require('../storage/guides');
+const { getPageRoutes, getGuideRoutes, getRoadmapRoutes } = require('./path-map');
+
+const DOMAIN = 'https://roadmap.sh';
+const PAGES_DIR = path.join(__dirname, '../pages');
+const STORAGE_PATH = path.join(__dirname, '../storage');
+const SITEMAP_PATH = 'static/sitemap.xml';
+const STATIC_PATH = path.join(__dirname, '../static');
+
+// Set the header
+const xmlHeader = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">`;
+
+// Wrap all pages in <urlset> tags
+const xmlUrlWrapper = nodes => `${xmlHeader}
+${nodes}
+</urlset>`;
+
+const getSlugPriority = (pageSlug) => {
+  if (pageSlug === '/') {
+    return '1.0';
+  }
+
+  const slugPriorities = [
+    ['/roadmaps', '/guides'],      // 1.0
+    ['/signup'],                   // 0.9
+    ['/about'],                    // 0.8
+  ];
+
+  const foundIndex = slugPriorities.findIndex(
+    routes => routes.some(route => pageSlug.startsWith(route))
+  );
+
+  if (foundIndex !== -1) {
+    return parseFloat((10 - foundIndex) / 10)
+      .toFixed(1);
+  }
+
+  return 0.5;
+};
+
+function generateNode({
+  slug,
+  basePath,
+  fileName,
+  priority = null,
+  date = null,
+  frequency = 'monthly'
+}) {
+  const pagePath = path.join(basePath, fileName);
+  let pageStats = {};
+  try {
+    pageStats = fs.lstatSync(pagePath);
+  } catch(e) {
+    console.log(`File not found: ${pagePath}`);
+    pageStats = { mtime: (new Date()) }
+  }
+
+  return `<url>
+              <loc>${DOMAIN}${slug}</loc>
+              <changefreq>${ frequency }</changefreq>
+              <lastmod>${date || pageStats.mtime.toISOString()}</lastmod>
+              <priority>${ priority || getSlugPriority(slug) }</priority>
+          </url>`;
+}
+
+function generateSiteMap() {
+  const pageRoutes = getPageRoutes();
+  const pageSlugs = Object.keys(pageRoutes)
+    .filter(route => ![
+      '/privacy',
+      '/terms'
+    ].includes(route));
+
+  const pagesChunk = pageSlugs.map(pageSlug => {
+    return generateNode({
+      basePath: PAGES_DIR,
+      fileName: `${pageRoutes[pageSlug].page}.js`,
+      slug: pageSlug,
+    });
+  });
+
+  // Chunks for each of the guides
+  const guideRoutes = getGuideRoutes();
+  const guideSlugs  = Object.keys(guideRoutes);
+  const guidesChunk = guideSlugs.map(guideSlug => {
+    const foundGuide = guides.find(guide => guide.url === guideSlug) || {};
+    return generateNode({
+      basePath: STORAGE_PATH,
+      fileName: `${guideSlug}.md`,
+      slug: guideSlug,
+      date: foundGuide.updatedAt,
+      priority: '1.0',
+    });
+  });
+
+  // Chunks for each of the roadmaps
+  const roadmapRoutes = getRoadmapRoutes();
+  const roadmapSlugs = Object.keys(roadmapRoutes);
+  const roadmapsChunk = roadmapSlugs.map(roadmapSlug => {
+    const [, role, year = 'latest'] = roadmapSlug.split('/');
+    return generateNode({
+      basePath: STATIC_PATH,
+      fileName: `/roadmaps/${year}/${role}.png`,
+      slug: roadmapSlug,
+      priority: '1.0',
+    });
+  });
+
+  const nodes = [
+    ...roadmapsChunk,
+    ...guidesChunk,
+    ...pagesChunk,
+  ];
+
+  const sitemap = `${xmlUrlWrapper(nodes.join('\n'))}`;
+
+  fs.writeFileSync(SITEMAP_PATH, sitemap);
+
+  console.log(`sitemap.xml with ${nodes.length} entries was written to ${SITEMAP_PATH}`);
+}
+
+generateSiteMap();

static/sitemap.xml

@@ -0,0 +1,141 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">
+<url>
+              <loc>https://roadmap.sh/frontend</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T08:22:01.000Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/frontend/latest</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T08:22:01.000Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/frontend/2018</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T08:22:01.000Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/frontend/2017</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T08:22:01.000Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/backend</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T08:22:01.000Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/backend/latest</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T08:22:01.000Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/backend/2018</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T08:22:01.000Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/backend/2017</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T08:22:01.000Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/devops</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T08:22:01.000Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/devops/latest</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T08:22:01.000Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/devops/2018</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T08:22:01.000Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/devops/2017</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T08:22:01.000Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/fullstack</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T17:21:23.153Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/qa-engineer</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T17:21:23.153Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/guides/design-patterns-for-humans</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-10-09T12:00:00.860Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/guides/journey-to-http2</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2018-12-04T12:00:00.860Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/guides/dns-in-one-picture</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2018-12-04T12:00:00.860Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/guides/http-caching</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2018-11-29T17:00:00.860Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/about</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T08:22:01.000Z</lastmod>
+              <priority>0.8</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/guides</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T08:22:01.000Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T08:22:01.000Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/roadmaps</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T08:22:01.000Z</lastmod>
+              <priority>1.0</priority>
+          </url>
+<url>
+              <loc>https://roadmap.sh/signup</loc>
+              <changefreq>monthly</changefreq>
+              <lastmod>2019-11-14T08:22:01.000Z</lastmod>
+              <priority>0.9</priority>
+          </url>
+</urlset>

test/path-map.spec.js

@@ -7,7 +7,7 @@ const {
   getPageRoutes,
   getGuideRoutes,
   getRoadmapRoutes
-} = require("../path-map");
+} = require("../scripts/path-map");
 
 describe("Build scripts tests", () => {
   test('it should generate valid pathmap for pages', () => {