add concurrency and backoff

config.ts

@@ -29,5 +29,15 @@ export const config = program({
         "Fail on the first download error instead of collecting all failures and continuing",
       ...field(z.boolean(), field.default(false)),
     },
+    concurrency: {
+      description:
+        "Maximum number of concurrent downloads. Lower values avoid upstream rate limiting.",
+      ...field(z.number(), field.default(75)),
+    },
+    retries: {
+      description:
+        "Number of times to retry a failed download before giving up. Defaults to 0 in strict mode.",
+      ...field(z.number()),
+    },
   }),
 });

downloader.ts

@@ -1,7 +1,7 @@
 import {
-  call,
   type Operation,
   resource,
+  sleep,
   until,
   useAbortSignal,
 } from "effection";
@@ -22,6 +22,8 @@ export interface DownloaderOptions {
   base: URL;
   outdir: string;
   strict?: boolean;
+  concurrency?: number;
+  retries?: number;
 }
 
 export type DownloadResult =
@@ -35,7 +37,7 @@ export const DownloadApi = createApi("@staticalize/download", {
     source: URL,
     referrer: URL,
   ): Operation<DownloadResult> {
-    let { host, base, outdir, strict } = opts;
+    let { host, base, outdir, strict, retries = 3 } = opts;
     let signal = yield* useAbortSignal();
     let path = normalize(join(outdir, source.pathname));
 
@@ -52,100 +54,118 @@ export const DownloadApi = createApi("@staticalize/download", {
     };
 
     try {
-      let response = yield* until(fetch(source.toString(), { signal }));
-      if (response.ok) {
-        if (response.headers.get("Content-Type")?.includes("html")) {
-          let destpath = join(path, "index.html");
-          let content = yield* until(response.text());
-          let html = fromHtml(content);
-
-          let links = selectAll("link[href]", html);
-
-          for (let link of links) {
-            let href = link.properties.href as string;
-            yield* downloader.download(href, source);
-
-            // replace self-referencing absolute urls with the destination site
-            if (href.startsWith(host.origin)) {
-              let url = new URL(href);
-              url.host = base.host;
-              url.port = base.port;
-              url.protocol = base.protocol;
-              link.properties.href = url.href;
-            }
+      let response = yield* fetchWithRetry(source.toString(), signal, retries);
+
+      if (response.headers.get("Content-Type")?.includes("html")) {
+        let destpath = join(path, "index.html");
+        let content = yield* until(response.text());
+        let html = fromHtml(content);
+
+        let links = selectAll("link[href]", html);
+
+        for (let link of links) {
+          let href = link.properties.href as string;
+          yield* downloader.download(href, source);
+
+          // replace self-referencing absolute urls with the destination site
+          if (href.startsWith(host.origin)) {
+            let url = new URL(href);
+            url.host = base.host;
+            url.port = base.port;
+            url.protocol = base.protocol;
+            link.properties.href = url.href;
           }
+        }
 
-          let assets = selectAll("[src]", html);
+        let assets = selectAll("[src]", html);
 
-          for (let element of assets) {
-            let src = element.properties.src as string;
-            yield* downloader.download(src, source);
+        for (let element of assets) {
+          let src = element.properties.src as string;
+          yield* downloader.download(src, source);
 
-            // replace self-referencing absolute urls with the destination site
-            if (src.startsWith(host.origin)) {
-              let url = new URL(src);
-              url.host = base.host;
-              url.port = base.port;
-              url.protocol = base.protocol;
-              element.properties.src = url.href;
-            }
+          // replace self-referencing absolute urls with the destination site
+          if (src.startsWith(host.origin)) {
+            let url = new URL(src);
+            url.host = base.host;
+            url.port = base.port;
+            url.protocol = base.protocol;
+            element.properties.src = url.href;
           }
+        }
 
-          let withContents = selectAll("[content]", html);
-          for (let element of withContents) {
-            let attr = String(element.properties.content);
-            if (attr.startsWith(host.origin)) {
-              yield* downloader.download(attr, source);
-              let url = new URL(attr);
-              url.host = base.host;
-              url.port = base.port;
-              url.protocol = base.protocol;
-              element.properties.content = url.href;
-            }
+        let withContents = selectAll("[content]", html);
+        for (let element of withContents) {
+          let attr = String(element.properties.content);
+          if (attr.startsWith(host.origin)) {
+            yield* downloader.download(attr, source);
+            let url = new URL(attr);
+            url.host = base.host;
+            url.port = base.port;
+            url.protocol = base.protocol;
+            element.properties.content = url.href;
           }
-
-          let output = toHtml(html);
-          yield* call(async () => {
-            let destdir = dirname(destpath);
-            await ensureDir(destdir);
-            await Deno.writeTextFile(destpath, output);
-          });
-          return {
-            ok: true,
-            bytes: new TextEncoder().encode(output).byteLength,
-          };
-        } else {
-          let size = Number(response.headers.get("Content-Length") ?? 0);
-          yield* call(async () => {
-            let destdir = dirname(path);
-            await ensureDir(destdir);
-            await Deno.writeFile(path, response.body!);
-          });
-          return { ok: true, bytes: size };
         }
+
+        let output = toHtml(html);
+        let destdir = dirname(destpath);
+        yield* until(ensureDir(destdir));
+        yield* until(Deno.writeTextFile(destpath, output));
+        return {
+          ok: true,
+          bytes: new TextEncoder().encode(output).byteLength,
+        };
       } else {
-        return fail(
-          new Error(
-            `GET ${source} responded ${response.status} ${response.statusText}`,
-          ),
-        );
+        let size = Number(response.headers.get("Content-Length") ?? 0);
+        let destdir = dirname(path);
+        yield* until(ensureDir(destdir));
+        yield* until(Deno.writeFile(path, response.body!));
+        return { ok: true, bytes: size };
       }
-    } catch (cause) {
-      // e.g. a gzip/transport decode error, an HTML parse error, or a
-      // filesystem write error — none of which name the url on their own.
-      return fail(new Error(`could not download ${source}`, { cause }));
+    } catch (cause: unknown) {
+      return fail(
+        cause instanceof Error
+          ? new Error(`could not download ${source}`, { cause })
+          : new Error(`could not download ${source}`),
+      );
     }
   },
 });
 
+function* fetchWithRetry(
+  url: string,
+  signal: AbortSignal,
+  retries: number,
+): Operation<Response> {
+  let lastError: Error | undefined;
+  for (let attempt = 0; attempt <= retries; attempt++) {
+    if (attempt > 0) {
+      // exponential backoff: 1s, 2s, 4s, ...
+      yield* sleep(1000 * 2 ** (attempt - 1));
+    }
+
+    try {
+      let response = yield* until(fetch(url, { signal }));
+      if (response.ok) {
+        return response;
+      }
+      lastError = new Error(
+        `GET ${url} responded ${response.status} ${response.statusText}`,
+      );
+    } catch (cause) {
+      lastError = new Error(`could not download ${url}`, { cause });
+    }
+  }
+  throw lastError!;
+}
+
 const { download } = DownloadApi.operations;
 
 export function useDownloader(opts: DownloaderOptions): Operation<Downloader> {
   let seen = new Map<string, boolean>();
   return resource(function* (provide) {
-    let { host } = opts;
+    let { host, concurrency = 75 } = opts;
 
-    let buffer = yield* useTaskBuffer(75);
+    let buffer = yield* useTaskBuffer(concurrency);
 
     let downloader: Downloader = {
       *download(loc, context = host) {

main.ts

@@ -18,7 +18,10 @@ await main(function* (args) {
     case "main": {
       let result = parser.parse();
       if (result.ok) {
-        let { base, site, output, strict } = result.value;
+        let { base, site, output, strict, concurrency, retries: retriesRaw } =
+          result.value;
+        // don't have a great way to default dynamically based on strict mode
+        let retries = retriesRaw ?? (strict ? 0 : 3);
 
         let stdin = yield* initStdin;
 
@@ -33,6 +36,8 @@ await main(function* (args) {
           host: new URL(site),
           dir: output,
           strict,
+          concurrency,
+          retries,
         });
 
         let { errors } = yield* withProgress({

staticalize.ts

@@ -16,6 +16,8 @@ export interface StaticalizeOptions {
   base: URL;
   dir: string;
   strict?: boolean;
+  concurrency?: number;
+  retries?: number;
 }
 
 export interface Staticalizer {
@@ -26,7 +28,7 @@ export interface Staticalizer {
 export function useStaticalizer(
   options: StaticalizeOptions,
 ): Operation<Staticalizer> {
-  let { host, base, dir, strict } = options;
+  let { host, base, dir, strict, concurrency, retries } = options;
 
   return resource(function* (provide) {
     let signal = yield* useAbortSignal();
@@ -69,7 +71,14 @@ export function useStaticalizer(
       );
     });
 
-    let downloader = yield* useDownloader({ host, base, outdir: dir, strict });
+    let downloader = yield* useDownloader({
+      host,
+      base,
+      outdir: dir,
+      strict,
+      concurrency,
+      retries,
+    });
 
     yield* provide({
       urls,