feat(core): split upload in chunks

To avoid overloading the network.

packages/core/src/debug.ts

@@ -1,3 +1,19 @@
 import createDebug from "debug";
 
-export const debug = createDebug("@argos-ci/core");
+const KEY = "@argos-ci/core";
+
+export const debug = createDebug(KEY);
+
+export const debugTime = (arg: string) => {
+  const enabled = createDebug.enabled(KEY);
+  if (enabled) {
+    console.time(arg);
+  }
+};
+
+export const debugTimeEnd = (arg: string) => {
+  const enabled = createDebug.enabled(KEY);
+  if (enabled) {
+    console.timeEnd(arg);
+  }
+};

packages/core/src/upload.ts

@@ -5,7 +5,13 @@ import { optimizeScreenshot } from "./optimize";
 import { hashFile } from "./hashing";
 import { createArgosApiClient, getBearerToken } from "./api-client";
 import { upload as uploadToS3 } from "./s3";
-import { debug } from "./debug";
+import { debug, debugTime, debugTimeEnd } from "./debug";
+import { chunk } from "./util/chunk";
+
+/**
+ * Size of the chunks used to upload screenshots to Argos.
+ */
+const CHUNK_SIZE = 10;
 
 export interface UploadParameters {
   /** Globs matching image file paths to upload */
@@ -119,20 +125,29 @@ export const upload = async (params: UploadParameters) => {
 
   debug("Got screenshots", result);
 
-  // Upload screenshots
-  debug("Uploading screenshots");
-  await Promise.all(
-    result.screenshots.map(async ({ key, putUrl }) => {
-      const screenshot = screenshots.find((s) => s.hash === key);
-      if (!screenshot) {
-        throw new Error(`Invariant: screenshot with hash ${key} not found`);
-      }
-      await uploadToS3({
-        url: putUrl,
-        path: screenshot.optimizedPath,
-      });
-    })
-  );
+  debug(`Split screenshots in chunks of ${CHUNK_SIZE}`);
+  const chunks = chunk(result.screenshots, CHUNK_SIZE);
+
+  debug(`Starting upload of ${chunks.length} chunks`);
+
+  for (let i = 0; i < chunks.length; i++) {
+    debug(`Uploading chunk ${i + 1}/${chunks.length}`);
+    const timeLabel = `Chunk ${i + 1}/${chunks.length}`;
+    debugTime(timeLabel);
+    await Promise.all(
+      chunks[i].map(async ({ key, putUrl }) => {
+        const screenshot = screenshots.find((s) => s.hash === key);
+        if (!screenshot) {
+          throw new Error(`Invariant: screenshot with hash ${key} not found`);
+        }
+        await uploadToS3({
+          url: putUrl,
+          path: screenshot.optimizedPath,
+        });
+      })
+    );
+    debugTimeEnd(timeLabel);
+  }
 
   // Update build
   debug("Updating build");

packages/core/src/util/chunk.ts

@@ -0,0 +1,16 @@
+/**
+ * Split an array into chunks of a given size.
+ */
+export const chunk = <T>(collection: T[], size: number) => {
+  const result = [];
+
+  // add each chunk to the result
+  for (let x = 0; x < Math.ceil(collection.length / size); x++) {
+    let start = x * size;
+    let end = start + size;
+
+    result.push(collection.slice(start, end));
+  }
+
+  return result;
+};