From 2f207c3827d9a5a7aaf3f0431ee2f838da49ed9d Mon Sep 17 00:00:00 2001
From: Randell Dawson <5313213+RandellDawson@users.noreply.github.com>
Date: Tue, 2 Feb 2021 20:50:04 -0700
Subject: [PATCH] fix(tools): Integrate certificate and comment dictionaries
 into the Curriculum project on Crowdin (#40872)

* fix: integrate certificate and dictionary files to Crowdin

* fix: correct linting issue

* fix: improved the isReserveredHeading function
---
 curriculum/crowdin.yml                        | 13 +++-
 .../hide-non-translated-strings/index.js      | 20 ++++-
 tools/crowdin/utils/files.js                  |  2 -
 tools/crowdin/utils/strings.js                | 73 +++++++++++++++----
 4 files changed, 83 insertions(+), 25 deletions(-)

diff --git a/curriculum/crowdin.yml b/curriculum/crowdin.yml
index 9c0fe23d35..c1357f0471 100644
--- a/curriculum/crowdin.yml
+++ b/curriculum/crowdin.yml
@@ -10,8 +10,15 @@ files: [
   "source" : "/curriculum/challenges/english/**/*.md",
   "translation" : "/curriculum/challenges/%language%/**/%original_file_name%",
   "ignore": [
-    "/**/part-[0-9][0-9][0-9].md",
-    "/curriculum/challenges/english/[0-9][0-9]-certificates/**/*.*"
+    "/**/part-[0-9][0-9][0-9].md"
   ]
- }
+ },
+ {
+  "source" : "/curriculum/challenges/english/12-certificates/**/*.yml",
+  "translation" : "/curriculum/challenges/%language%/12-certificates/**/%original_file_name%",
+ },
+ {
+  "source" : "/curriculum/dictionaries/english/comments.json",
+  "translation" : "/curriculum/dictionaries/%language%/%original_file_name%"
+ },
 ]
diff --git a/tools/crowdin/actions/hide-non-translated-strings/index.js b/tools/crowdin/actions/hide-non-translated-strings/index.js
index 04bce25bd6..11a32d889e 100644
--- a/tools/crowdin/actions/hide-non-translated-strings/index.js
+++ b/tools/crowdin/actions/hide-non-translated-strings/index.js
@@ -1,5 +1,4 @@
 require('dotenv').config({ path: `${__dirname}/../../.env` });
-// const core = require('@actions/core');
 const fs = require('fs');
 const path = require('path');
 const matter = require('gray-matter');
@@ -20,7 +19,13 @@ const createChallengeTitleLookup = (
     const {
       data: { title: challengeTitle }
     } = matter(challengeContent);
-    return { ...lookup, [fileId]: challengeTitle };
+    return {
+      ...lookup,
+      [fileId]: {
+        crowdinFilePath,
+        challengeTitle
+      }
+    };
   } catch (err) {
     console.log(err.name);
     console.log(err.message);
@@ -39,8 +44,15 @@ const hideNonTranslatedStrings = async projectId => {
     const crowdinStrings = await getStrings({ projectId });
     if (crowdinStrings && crowdinStrings.length) {
       for (let string of crowdinStrings) {
-        const challengeTitle = challengeTitleLookup[string.data.fileId];
-        await updateFileString({ projectId, string, challengeTitle });
+        const { crowdinFilePath, challengeTitle } = challengeTitleLookup[
+          string.data.fileId
+        ];
+        await updateFileString({
+          projectId,
+          string,
+          challengeTitle,
+          crowdinFilePath
+        });
       }
     }
   }
diff --git a/tools/crowdin/utils/files.js b/tools/crowdin/utils/files.js
index b2337c8cdc..a380079028 100644
--- a/tools/crowdin/utils/files.js
+++ b/tools/crowdin/utils/files.js
@@ -1,5 +1,4 @@
 const makeRequest = require('./make-request');
-const delay = require('./delay');
 const authHeader = require('./auth-header');
 
 const addFile = async (projectId, filename, fileContent, directoryId) => {
@@ -88,7 +87,6 @@ const getFiles = async projectId => {
   let files = [];
   while (!done) {
     const endPoint = `projects/${projectId}/files?limit=500&offset=${offset}`;
-    await delay(1000);
     const response = await makeRequest({
       method: 'get',
       endPoint,
diff --git a/tools/crowdin/utils/strings.js b/tools/crowdin/utils/strings.js
index 4786d5ee4b..6fa8025d44 100644
--- a/tools/crowdin/utils/strings.js
+++ b/tools/crowdin/utils/strings.js
@@ -1,11 +1,36 @@
 const authHeader = require('./auth-header');
 const makeRequest = require('./make-request');
 
-const isHeading = str => /\h\d/.test(str);
+const isReservedHeading = (context, str) => {
+  const reservedHeadings = [
+    'after-user-code',
+    'answers',
+    'before-user-code',
+    'description',
+    'fcc-editable-region',
+    'hints',
+    'instructions',
+    'question',
+    'seed',
+    'seed-contents',
+    'solutions',
+    'text',
+    'video-solution'
+  ];
+  const captureGroupStr = `(${reservedHeadings.join('|')})`;
+  const regex = new RegExp(`--${captureGroupStr}--`);
+  return !!(context.match(/^Headline/) && str.match(regex));
+};
+
 const isCode = str => /^\/pre\/code|\/code$/.test(str);
 
-const shouldHide = (text, context, challengeTitle) => {
-  if (isHeading(context) || isCode(context)) {
+const isTitle = str => /^(tests\s*->\s*\d+\s*)?->\s*title/.test(str);
+
+const shouldHide = (text, context, challengeTitle, crowdinFilePath) => {
+  if (crowdinFilePath.endsWith('.yml')) {
+    return !isTitle(context);
+  }
+  if (isReservedHeading(context, text) || isCode(context)) {
     return true;
   }
   return text !== challengeTitle && context.includes('id=front-matter');
@@ -13,18 +38,29 @@ const shouldHide = (text, context, challengeTitle) => {
 
 const getStrings = async ({ projectId, fileId }) => {
   let headers = { ...authHeader };
-  let endPoint = `projects/${projectId}/strings?limit=500`;
-  if (fileId) {
-    endPoint += `&fileId=${fileId}`;
-  }
-  const strings = await makeRequest({ method: 'get', endPoint, headers });
-  if (strings.data) {
-    return strings.data;
-  } else {
-    const { error, errors } = strings;
-    console.error(error ? error : errors);
-    return null;
+  let done = false;
+  let offset = 0;
+  let strings = [];
+  while (!done) {
+    let endPoint = `projects/${projectId}/strings?limit=500&offset=${offset}`;
+    if (fileId) {
+      endPoint += `&fileId=${fileId}`;
+    }
+    const response = await makeRequest({ method: 'get', endPoint, headers });
+    if (response.data) {
+      if (response.data.length) {
+        strings = [...strings, ...response.data];
+        offset += 500;
+      } else {
+        done = true;
+        return strings;
+      }
+    } else {
+      const { error, errors } = response;
+      console.error(error ? error : errors);
+    }
   }
+  return null;
 };
 
 const updateString = async ({ projectId, stringId, propsToUpdate }) => {
@@ -69,11 +105,16 @@ const updateFileStrings = async ({ projectId, fileId, challengeTitle }) => {
   }
 };
 
-const updateFileString = async ({ projectId, string, challengeTitle }) => {
+const updateFileString = async ({
+  projectId,
+  string,
+  challengeTitle,
+  crowdinFilePath
+}) => {
   const {
     data: { id: stringId, text, isHidden, context }
   } = string;
-  const hideString = shouldHide(text, context, challengeTitle);
+  const hideString = shouldHide(text, context, challengeTitle, crowdinFilePath);
   if (!isHidden && hideString) {
     await changeHiddenStatus(projectId, stringId, true);
     console.log(