chore: move search-indexing, supress webpack-cli progress (#39367)

2020-08-09 04:17:10 +05:30
parent 6e16a50329
commit adef746299
17 changed files with 4 additions and 5 deletions
--- a/tools/search-indexing/data-sources/youtube/index.js
+++ b/tools/search-indexing/data-sources/youtube/index.js
@@ -0,0 +1,124 @@
+const { timer, from, zip, iif, of } = require('rxjs');
+const { switchMap, concatMap } = require('rxjs/operators');
+const { google } = require('googleapis');
+const { chunkDocument, stripHTML, stripURLs } = require('../../utils');
+
+const { YOUTUBE_SECRET } = process.env;
+const youtube = google.youtube({ version: 'v3', auth: YOUTUBE_SECRET });
+
+function getPlaylistItems(
+  { playlistId, playlistTitle },
+  nextPage,
+  currentItems = []
+) {
+  return zip(
+    timer(2000),
+    from(
+      new Promise((resolve, reject) => {
+        youtube.playlistItems.list(
+          {
+            part: 'snippet',
+            playlistId,
+            pageToken: nextPage ? nextPage : ''
+          },
+          (err, data) => {
+            if (err) {
+              return reject(err);
+            }
+            return resolve({ ...data.data, playlistTitle });
+          }
+        );
+      })
+    ),
+    (a, b) => b
+  ).pipe(
+    switchMap(({ nextPageToken, items, playlistTitle }) => {
+      const allItems = currentItems.concat(items);
+      return iif(
+        () => !!nextPageToken,
+        getPlaylistItems(
+          { playlistId, playlistTitle },
+          nextPageToken,
+          allItems
+        ),
+        of({ videos: allItems, playlistTitle })
+      );
+    })
+  );
+}
+
+function getPlayLists(nextPage, currentItems = []) {
+  return from(
+    new Promise((resolve, reject) => {
+      youtube.playlists.list(
+        {
+          auth: YOUTUBE_SECRET,
+          part: 'snippet',
+          channelId: 'UC8butISFwT-Wl7EV0hUK0BQ',
+          pageToken: nextPage ? nextPage : ''
+        },
+        (err, data) => {
+          if (err) {
+            return reject(err);
+          }
+          return resolve(data.data);
+        }
+      );
+    })
+  ).pipe(
+    switchMap(({ nextPageToken, items }) => {
+      const allItems = items.concat(currentItems);
+      return iif(
+        () => !!nextPageToken,
+        getPlayLists(nextPageToken, allItems),
+        of(allItems)
+      );
+    })
+  );
+}
+
+exports.getYoutubeData = function getYoutubeData() {
+  return getPlayLists().pipe(
+    switchMap(playlists => {
+      return from(playlists).pipe(
+        concatMap(({ id, snippet: { title } }) =>
+          getPlaylistItems({ playlistId: id, playlistTitle: title })
+        )
+      );
+    }),
+    switchMap(({ videos, playlistTitle }) => {
+      const formattedVideos = videos
+        .map(video => {
+          const {
+            id,
+            snippet: {
+              title,
+              description,
+              resourceId: { videoId },
+              thumbnails
+            }
+          } = video;
+          return {
+            id,
+            videoId,
+            title: stripHTML(title),
+            description: stripURLs(stripHTML(description)),
+            thumbnails,
+            playlistTitle
+          };
+        })
+        .reduce(
+          (chunked, current) => [
+            ...chunked,
+            ...chunkDocument(
+              current,
+              ['id', 'videoId', 'title', 'thumbnail', 'playlistTitle'],
+              'description'
+            )
+          ],
+          []
+        );
+      return of(formattedVideos);
+    })
+  );
+};