feat: script to export bubble data to meilisearch

.env.template

@@ -17,5 +17,16 @@ GITHUB_CLIENT_SECRET=
 # Discord webhook to log audit events to
 AUDIT_WEBHOOK=
 
+# MeiliSearch
+MEILISEARCH_URL=
+NEXT_PUBLIC_MEILISEARCH_URL=
+
+# optional - you should set this in production
+# this key should have write permissions for the search database
+# MEILISEARCH_KEY=
+# optional - you should set this in production
+# this key should only have read permissions for the search database
+# NEXT_PUBLIC_MEILISEARCH_SEARCH_KEY=
+
 # Path to the database file (relative to the prisma directory)
 DATABASE_URL="file:./database.db"

package.json

@@ -8,7 +8,8 @@
         "start": "next start",
         "lint": "next lint",
         "format": "prettier --write .",
-        "format:check": "prettier --check ."
+        "format:check": "prettier --check .",
+        "update-indexes": "ts-node --compiler-options '{\"module\":\"CommonJS\"}' src/export-data.ts"
     },
     "dependencies": {
         "@bmunozg/react-image-area": "^1.0.2",
@@ -19,6 +20,7 @@
         "@prisma/client": "^4.1.0",
         "diff": "^5.1.0",
         "discord.js": "^14.0.3",
+        "meilisearch": "^0.27.0",
         "next": "12.2.2",
         "next-auth": "^4.10.1",
         "react": "18.2.0",

pnpm-lock.yaml

@@ -17,6 +17,7 @@ specifiers:
   discord.js: ^14.0.3
   eslint: 8.20.0
   eslint-config-next: 12.2.2
+  meilisearch: ^0.27.0
   next: 12.2.2
   next-auth: ^4.10.1
   postcss: ^8.4.14
@@ -40,6 +41,7 @@ dependencies:
   '@prisma/client': 4.1.0_prisma@4.1.0
   diff: 5.1.0
   discord.js: 14.0.3
+  meilisearch: 0.27.0
   next: 12.2.2_biqbaboplfbrettd7655fr4n2y
   next-auth: 4.10.1_biqbaboplfbrettd7655fr4n2y
   react: 18.2.0
@@ -1110,6 +1112,14 @@ packages:
     resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==}
     dev: true
 
+  /cross-fetch/3.1.5:
+    resolution: {integrity: sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==}
+    dependencies:
+      node-fetch: 2.6.7
+    transitivePeerDependencies:
+      - encoding
+    dev: false
+
   /cross-spawn/7.0.3:
     resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==}
     engines: {node: '>= 8'}
@@ -2125,6 +2135,14 @@ packages:
     resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==}
     dev: true
 
+  /meilisearch/0.27.0:
+    resolution: {integrity: sha512-kZOZFIuSO7c6xRf+Y2/9/h6A9pl0sCl/G44X4KuaSwxGbruOZPhmxbeVEgLHBv4pUFvQ56rNVTA/2d/5GCU1YA==}
+    dependencies:
+      cross-fetch: 3.1.5
+    transitivePeerDependencies:
+      - encoding
+    dev: false
+
   /memoize-one/5.2.1:
     resolution: {integrity: sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==}
     dev: false

src/export-data.ts

@@ -0,0 +1,48 @@
+/**
+ * This script exports bubble data to external databases.
+ * Most notably, meilisearch, which is used to power the
+ * search feature, although this script will later also
+ * be used to generate the database dumps that will be
+ * published to GitHub.
+ */
+
+import MeiliSearch from 'meilisearch';
+import { prisma } from './db';
+
+(async function main() {
+    if (!process.env.MEILISEARCH_URL) throw new Error('meilisearch url must be set');
+
+    const meilisearch = new MeiliSearch({
+        host: process.env.MEILISEARCH_URL,
+    });
+
+    const pages = await prisma.comicPage.findMany({
+        include: {
+            bubbles: true,
+            comic: true,
+        },
+    });
+
+    const index = meilisearch.index('comic_pages');
+
+    await index.addDocuments(pages.map(page => {
+        return {
+            ...page,
+            id: `${page.comic.id}-${page.id}`,
+        }
+    }));
+
+    console.log('submitted', pages.length, 'pages to be indexed');
+
+    // Limit what can be searched
+    await index.updateSearchableAttributes([
+        'bubbles.content',
+        'title',
+        'comic.title',
+    ]);
+    
+    // Allow filtering by comic
+    await index.updateFilterableAttributes([
+        'comic.id',
+    ]);
+})();