Delete kmails after processing them in the etl, and try to load up to…

… 1000 messages (#83)
loathers · Jun 18, 2024 · 4c52f46 · 4c52f46
1 parent 945ac29
commit 4c52f46
Show file tree

Hide file tree

Showing 2 changed files with 70 additions and 38 deletions.
diff --git a/package.json b/package.json
@@ -11,7 +11,8 @@
   "scripts": {
     "build": "yarn workspace excavator-script run build",
     "format": "yarn workspaces run format && yarn prettier --write .github",
-    "lint": "yarn workspaces run lint && yarn prettier --check .github"
+    "lint": "yarn workspaces run lint && yarn prettier --check .github",
+    "etl": "yarn workspace excavator-web run etl"
   },
   "engines": {
     "node": ">= 18.20"

diff --git a/packages/excavator-web/etl.ts b/packages/excavator-web/etl.ts
@@ -49,6 +49,27 @@ async function loadKmails() {
   return (await request.json()) as Kmail[];
 }
 
+async function getPwd() {
+  const request = await fetch(
+    "https://www.kingdomofloathing.com/api.php?what=status&for=excavator",
+  );
+  const json = await request.json();
+  return json["pwd"];
+}
+
+async function deleteKmails(ids: number[]) {
+  const pwd = await getPwd();
+  const response = await fetch(
+    "https://www.kingdomofloathing.com/messages.php",
+    {
+      body: `the_action=delete&pwd=${pwd}&box=Inbox&${ids.map((id) => `sel${id}=on`).join("&")}`,
+      method: "POST",
+    },
+  );
+  const text = await response.text();
+  return text.includes(`${ids.length} messages deleted.`);
+}
+
 function hashData(data: Record<string, string | number | boolean>) {
   return crypto
     .createHash("md5")
@@ -98,45 +119,55 @@ async function main() {
     return;
   }
 
-  for (const kmail of await loadKmails()) {
-    try {
-      const message = decodeURIComponent(
-        kmail.message.replace(/ /g, "").replace(/\+/g, " "),
-      );
-      const fixed = applyFixes(JSON.parse(message) as SpadingData);
-
-      if (fixed === null) continue;
-
-      const { _PROJECT, _VERSION, ...data } = fixed;
-
-      const id = Number(kmail.id);
-
-      await prisma.spadingData.upsert({
-        create: {
-          id,
-          createdAt: new Date(Number(kmail.azunixtime) * 1000),
-          playerId: Number(kmail.fromid),
-          project: _PROJECT,
-          version: _VERSION,
-          dataHash: hashData(data),
-          data,
-        },
-        update: {},
-        where: { id },
-      });
-    } catch (error) {
-      const intro = `Kmail ${kmail.id} from ${kmail.fromname} (#${kmail.fromid})`;
-      if (error instanceof URIError) {
-        console.error(intro, "is poorly encoded");
-        continue;
-      }
-      if (error instanceof SyntaxError) {
-        console.error(intro, "contains bad JSON");
-        continue;
+  // Try reloading kmails 10 times
+  for (let i = 0; i < 10; i++) {
+    const kmails = await loadKmails();
+
+    // If there are no kmails, we're done
+    if (kmails.length === 0) break;
+
+    for (const kmail of kmails) {
+      try {
+        const message = decodeURIComponent(
+          kmail.message.replace(/ /g, "").replace(/\+/g, " "),
+        );
+        const fixed = applyFixes(JSON.parse(message) as SpadingData);
+
+        if (fixed === null) continue;
+
+        const { _PROJECT, _VERSION, ...data } = fixed;
+
+        const id = Number(kmail.id);
+
+        await prisma.spadingData.upsert({
+          create: {
+            id,
+            createdAt: new Date(Number(kmail.azunixtime) * 1000),
+            playerId: Number(kmail.fromid),
+            project: _PROJECT,
+            version: _VERSION,
+            dataHash: hashData(data),
+            data,
+          },
+          update: {},
+          where: { id },
+        });
+      } catch (error) {
+        const intro = `Kmail ${kmail.id} from ${kmail.fromname} (#${kmail.fromid})`;
+        if (error instanceof URIError) {
+          console.error(intro, "is poorly encoded");
+          continue;
+        }
+        if (error instanceof SyntaxError) {
+          console.error(intro, "contains bad JSON");
+          continue;
+        }
+
+        console.error(intro, "causes some other error", error);
       }
-
-      console.error(intro, "causes some other error", error);
     }
+
+    await deleteKmails(kmails.map((k) => Number(k.id)));
   }
 }