From ba020c874bf4f04c0bd06fbf72a1cfa6e9a8af47 Mon Sep 17 00:00:00 2001
From: Ashhhleyyy <ash@ashhhleyyy.dev>
Date: Mon, 1 Aug 2022 21:03:11 +0100
Subject: [PATCH] fix(search): submit documents in batches

---
 src/export-data.ts | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/export-data.ts b/src/export-data.ts
index 7a79182..ad0f96f 100644
--- a/src/export-data.ts
+++ b/src/export-data.ts
@@ -27,14 +27,20 @@ import { prisma } from './db';
 
     const index = meilisearch.index('comic_pages');
 
-    await index.addDocuments(
-        pages.map((page) => {
-            return {
-                ...page,
-                id: `${page.comic.id}-${page.id}`,
-            };
-        })
-    );
+    const batchSize = 100;
+    for (let i = 0; i < Math.ceil(pages.length / batchSize); i++) {
+        const startIndex = i * 100;
+        const endIndex = Math.min(pages.length, (i + 1) * 100);
+        await index.addDocuments(
+            pages.slice(startIndex, endIndex).map((page) => {
+                return {
+                    ...page,
+                    id: `${page.comic.id}-${page.id}`,
+                };
+            })
+        );
+        console.log('submitted', endIndex - startIndex, 'pages in batch no.', i);
+    }
 
     console.log('submitted', pages.length, 'pages to be indexed');