From 8c1b1b96c8029ff55da030467e8613c93d2a1e9d Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Mon, 13 Jul 2026 17:36:10 -0500 Subject: [PATCH 1/4] add concurrency and backoff --- config.ts | 10 ++++++++++ downloader.ts | 42 +++++++++++++++++++++++++++--------------- main.ts | 4 +++- staticalize.ts | 6 ++++-- 4 files changed, 44 insertions(+), 18 deletions(-) diff --git a/config.ts b/config.ts index 5156cd9..038d025 100644 --- a/config.ts +++ b/config.ts @@ -29,5 +29,15 @@ export const config = program({ "Fail on the first download error instead of collecting all failures and continuing", ...field(z.boolean(), field.default(false)), }, + concurrency: { + description: + "Maximum number of concurrent downloads. Lower values avoid upstream rate limiting.", + ...field(z.number(), field.default(75)), + }, + retries: { + description: + "Number of times to retry a failed download before giving up.", + ...field(z.number(), field.default(3)), + }, }), }); diff --git a/downloader.ts b/downloader.ts index 9b517cd..7a2686e 100644 --- a/downloader.ts +++ b/downloader.ts @@ -2,6 +2,7 @@ import { call, type Operation, resource, + sleep, until, useAbortSignal, } from "effection"; @@ -22,6 +23,8 @@ export interface DownloaderOptions { base: URL; outdir: string; strict?: boolean; + concurrency?: number; + retries?: number; } export type DownloadResult = @@ -35,7 +38,7 @@ export const DownloadApi = createApi("@staticalize/download", { source: URL, referrer: URL, ): Operation { - let { host, base, outdir, strict } = opts; + let { host, base, outdir, strict, retries = 3 } = opts; let signal = yield* useAbortSignal(); let path = normalize(join(outdir, source.pathname)); @@ -51,9 +54,16 @@ export const DownloadApi = createApi("@staticalize/download", { }; }; - try { - let response = yield* until(fetch(source.toString(), { signal })); - if (response.ok) { + let lastError: Error | undefined; + for (let attempt = 0; attempt <= retries; attempt++) { + if (attempt > 0) { + // exponential backoff: 1s, 2s, 4s, ... + yield* sleep(1000 * 2 ** (attempt - 1)); + } + + try { + let response = yield* until(fetch(source.toString(), { signal })); + if (response.ok) { if (response.headers.get("Content-Type")?.includes("html")) { let destpath = join(path, "index.html"); let content = yield* until(response.text()); @@ -123,18 +133,20 @@ export const DownloadApi = createApi("@staticalize/download", { }); return { ok: true, bytes: size }; } - } else { - return fail( - new Error( + } else { + lastError = new Error( `GET ${source} responded ${response.status} ${response.statusText}`, - ), - ); + ); + if (attempt < retries) continue; + return fail(lastError); + } + } catch (cause) { + lastError = new Error(`could not download ${source}`, { cause }); + if (attempt < retries) continue; + return fail(lastError); } - } catch (cause) { - // e.g. a gzip/transport decode error, an HTML parse error, or a - // filesystem write error — none of which name the url on their own. - return fail(new Error(`could not download ${source}`, { cause })); } + return fail(lastError!); }, }); @@ -143,9 +155,9 @@ const { download } = DownloadApi.operations; export function useDownloader(opts: DownloaderOptions): Operation { let seen = new Map(); return resource(function* (provide) { - let { host } = opts; + let { host, concurrency = 75 } = opts; - let buffer = yield* useTaskBuffer(75); + let buffer = yield* useTaskBuffer(concurrency); let downloader: Downloader = { *download(loc, context = host) { diff --git a/main.ts b/main.ts index 4a23ccd..905c6cc 100644 --- a/main.ts +++ b/main.ts @@ -18,7 +18,7 @@ await main(function* (args) { case "main": { let result = parser.parse(); if (result.ok) { - let { base, site, output, strict } = result.value; + let { base, site, output, strict, concurrency, retries } = result.value; let stdin = yield* initStdin; @@ -33,6 +33,8 @@ await main(function* (args) { host: new URL(site), dir: output, strict, + concurrency, + retries, }); let { errors } = yield* withProgress({ diff --git a/staticalize.ts b/staticalize.ts index 04f234d..0036288 100644 --- a/staticalize.ts +++ b/staticalize.ts @@ -16,6 +16,8 @@ export interface StaticalizeOptions { base: URL; dir: string; strict?: boolean; + concurrency?: number; + retries?: number; } export interface Staticalizer { @@ -26,7 +28,7 @@ export interface Staticalizer { export function useStaticalizer( options: StaticalizeOptions, ): Operation { - let { host, base, dir, strict } = options; + let { host, base, dir, strict, concurrency, retries } = options; return resource(function* (provide) { let signal = yield* useAbortSignal(); @@ -69,7 +71,7 @@ export function useStaticalizer( ); }); - let downloader = yield* useDownloader({ host, base, outdir: dir, strict }); + let downloader = yield* useDownloader({ host, base, outdir: dir, strict, concurrency, retries }); yield* provide({ urls, From bb86d4b7d49f57c75ab1735728be338d2aca8a9b Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Mon, 13 Jul 2026 23:44:02 -0500 Subject: [PATCH 2/4] pull retry into separate function --- downloader.ts | 173 +++++++++++++++++++++++++------------------------ staticalize.ts | 9 ++- 2 files changed, 98 insertions(+), 84 deletions(-) diff --git a/downloader.ts b/downloader.ts index 7a2686e..871f49b 100644 --- a/downloader.ts +++ b/downloader.ts @@ -1,5 +1,4 @@ import { - call, type Operation, resource, sleep, @@ -54,102 +53,110 @@ export const DownloadApi = createApi("@staticalize/download", { }; }; - let lastError: Error | undefined; - for (let attempt = 0; attempt <= retries; attempt++) { - if (attempt > 0) { - // exponential backoff: 1s, 2s, 4s, ... - yield* sleep(1000 * 2 ** (attempt - 1)); - } + try { + let response = yield* fetchWithRetry(source.toString(), signal, retries); - try { - let response = yield* until(fetch(source.toString(), { signal })); - if (response.ok) { - if (response.headers.get("Content-Type")?.includes("html")) { - let destpath = join(path, "index.html"); - let content = yield* until(response.text()); - let html = fromHtml(content); - - let links = selectAll("link[href]", html); - - for (let link of links) { - let href = link.properties.href as string; - yield* downloader.download(href, source); - - // replace self-referencing absolute urls with the destination site - if (href.startsWith(host.origin)) { - let url = new URL(href); - url.host = base.host; - url.port = base.port; - url.protocol = base.protocol; - link.properties.href = url.href; - } - } + if (response.headers.get("Content-Type")?.includes("html")) { + let destpath = join(path, "index.html"); + let content = yield* until(response.text()); + let html = fromHtml(content); - let assets = selectAll("[src]", html); + let links = selectAll("link[href]", html); - for (let element of assets) { - let src = element.properties.src as string; - yield* downloader.download(src, source); + for (let link of links) { + let href = link.properties.href as string; + yield* downloader.download(href, source); - // replace self-referencing absolute urls with the destination site - if (src.startsWith(host.origin)) { - let url = new URL(src); - url.host = base.host; - url.port = base.port; - url.protocol = base.protocol; - element.properties.src = url.href; - } + // replace self-referencing absolute urls with the destination site + if (href.startsWith(host.origin)) { + let url = new URL(href); + url.host = base.host; + url.port = base.port; + url.protocol = base.protocol; + link.properties.href = url.href; } + } - let withContents = selectAll("[content]", html); - for (let element of withContents) { - let attr = String(element.properties.content); - if (attr.startsWith(host.origin)) { - yield* downloader.download(attr, source); - let url = new URL(attr); - url.host = base.host; - url.port = base.port; - url.protocol = base.protocol; - element.properties.content = url.href; - } - } + let assets = selectAll("[src]", html); - let output = toHtml(html); - yield* call(async () => { - let destdir = dirname(destpath); - await ensureDir(destdir); - await Deno.writeTextFile(destpath, output); - }); - return { - ok: true, - bytes: new TextEncoder().encode(output).byteLength, - }; - } else { - let size = Number(response.headers.get("Content-Length") ?? 0); - yield* call(async () => { - let destdir = dirname(path); - await ensureDir(destdir); - await Deno.writeFile(path, response.body!); - }); - return { ok: true, bytes: size }; + for (let element of assets) { + let src = element.properties.src as string; + yield* downloader.download(src, source); + + // replace self-referencing absolute urls with the destination site + if (src.startsWith(host.origin)) { + let url = new URL(src); + url.host = base.host; + url.port = base.port; + url.protocol = base.protocol; + element.properties.src = url.href; + } } - } else { - lastError = new Error( - `GET ${source} responded ${response.status} ${response.statusText}`, - ); - if (attempt < retries) continue; - return fail(lastError); + + let withContents = selectAll("[content]", html); + for (let element of withContents) { + let attr = String(element.properties.content); + if (attr.startsWith(host.origin)) { + yield* downloader.download(attr, source); + let url = new URL(attr); + url.host = base.host; + url.port = base.port; + url.protocol = base.protocol; + element.properties.content = url.href; + } } - } catch (cause) { - lastError = new Error(`could not download ${source}`, { cause }); - if (attempt < retries) continue; - return fail(lastError); + + let output = toHtml(html); + let destdir = dirname(destpath); + yield* until(ensureDir(destdir)); + yield* until(Deno.writeTextFile(destpath, output)); + return { + ok: true, + bytes: new TextEncoder().encode(output).byteLength, + }; + } else { + let size = Number(response.headers.get("Content-Length") ?? 0); + let destdir = dirname(path); + yield* until(ensureDir(destdir)); + yield* until(Deno.writeFile(path, response.body!)); + return { ok: true, bytes: size }; + } + } catch (cause: unknown) { + if (cause instanceof Error) { + return fail(cause); } + throw cause; } - return fail(lastError!); }, }); +function* fetchWithRetry( + url: string, + signal: AbortSignal, + retries: number, +): Operation { + let lastError: Error | undefined; + for (let attempt = 0; attempt <= retries; attempt++) { + if (attempt > 0) { + // exponential backoff: 1s, 2s, 4s, ... + yield* sleep(1000 * 2 ** (attempt - 1)); + } + + try { + let response = yield* until(fetch(url, { signal })); + if (response.ok) { + return response; + } + lastError = new Error( + `GET ${url} responded ${response.status} ${response.statusText}`, + ); + } catch (cause) { + lastError = new Error(`could not download ${url}`, { cause }); + } + } + throw lastError!; +} + const { download } = DownloadApi.operations; export function useDownloader(opts: DownloaderOptions): Operation { diff --git a/staticalize.ts b/staticalize.ts index 0036288..3a4ebbe 100644 --- a/staticalize.ts +++ b/staticalize.ts @@ -71,7 +71,14 @@ export function useStaticalizer( ); }); - let downloader = yield* useDownloader({ host, base, outdir: dir, strict, concurrency, retries }); + let downloader = yield* useDownloader({ + host, + base, + outdir: dir, + strict, + concurrency, + retries, + }); yield* provide({ urls, From 2045b938ed6caa96fff5d8c1a471389cc9be277c Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Mon, 13 Jul 2026 23:51:34 -0500 Subject: [PATCH 3/4] change retries in strict mode --- config.ts | 4 ++-- main.ts | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/config.ts b/config.ts index 038d025..7e357b9 100644 --- a/config.ts +++ b/config.ts @@ -36,8 +36,8 @@ export const config = program({ }, retries: { description: - "Number of times to retry a failed download before giving up.", - ...field(z.number(), field.default(3)), + "Number of times to retry a failed download before giving up. Defaults to 0 in strict mode.", + ...field(z.number()), }, }), }); diff --git a/main.ts b/main.ts index 905c6cc..6bacde3 100644 --- a/main.ts +++ b/main.ts @@ -18,7 +18,10 @@ await main(function* (args) { case "main": { let result = parser.parse(); if (result.ok) { - let { base, site, output, strict, concurrency, retries } = result.value; + let { base, site, output, strict, concurrency, retries: retriesRaw } = + result.value; + // don't have a great way to default dynamically based on strict mode + let retries = retriesRaw ?? (strict ? 0 : 3); let stdin = yield* initStdin; From c695a5f9673eafaf6f12c665a6fe1c2d2c77bfa3 Mon Sep 17 00:00:00 2001 From: Jacob Bolda Date: Mon, 13 Jul 2026 23:56:26 -0500 Subject: [PATCH 4/4] try/catch fail throws with context --- downloader.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/downloader.ts b/downloader.ts index 871f49b..550e375 100644 --- a/downloader.ts +++ b/downloader.ts @@ -122,10 +122,11 @@ export const DownloadApi = createApi("@staticalize/download", { return { ok: true, bytes: size }; } } catch (cause: unknown) { - if (cause instanceof Error) { - return fail(cause); - } - throw cause; + return fail( + cause instanceof Error + ? new Error(`could not download ${source}`, { cause }) + : new Error(`could not download ${source}`), + ); } }, });