Refresh only cache objects near expiry

.github/workflows/ci.yml

@@ -18,6 +18,8 @@ jobs:
         with:
           node-version: 20
       - run: npm ci
+      - name: Unit tests
+        run: node --test __tests__/
       - run: npm run build
       - name: dist/ must match src/
         run: git diff --exit-code dist/ || { echo "dist/ is stale; run npm run build and commit"; exit 1; }
@@ -32,6 +34,8 @@ jobs:
         with:
           node-version: 20
       - run: npm ci
+      - name: Unit tests
+        run: node --test __tests__/
       - run: npm run build
       - name: dist/ must match src/
         run: git diff --exit-code dist/ || { echo "dist/ is stale; run npm run build and commit"; exit 1; }

__tests__/refresh-policy.test.js

@@ -0,0 +1,107 @@
+'use strict';
+
+const test = require('node:test');
+const assert = require('node:assert');
+const {
+    parseS3ListAges,
+    nearExpiry,
+    worstCaseAgeDays
+} = require('../src/refresh-policy');
+
+const NOW = Date.parse('2026-08-21T00:00:00Z');
+const ls = (lines) => lines.join('\n');
+
+test('parses aws s3 ls output into key -> age in days', () => {
+    const ages = parseS3ListAges(
+        ls([
+            '2026-08-20 12:00:00       2023 org-x/snap-ca/aaa.tar.zst',
+            '2026-08-14 00:00:00      50000 org-x/snap-ca/bbb.tar.zst'
+        ]),
+        NOW
+    );
+    assert.ok(ages);
+    assert.equal(ages.size, 2);
+    assert.ok(Math.abs(ages.get('aaa.tar.zst') - 0.5) < 0.01);
+    assert.ok(Math.abs(ages.get('bbb.tar.zst') - 7) < 0.01);
+});
+
+test('keys are relative to <ns>/<prefix>/ so they match --include patterns', () => {
+    // blobs live one level deeper; the whole remainder must survive.
+    const ages = parseS3ListAges(
+        ls(['2026-08-20 00:00:00        10 org-x/blobs/sha256/deadbeef']),
+        NOW
+    );
+    assert.deepEqual([...ages.keys()], ['sha256/deadbeef']);
+});
+
+test('timestamps are read as UTC, not local time', () => {
+    // A naive Date.parse of "2026-08-20 12:00:00" is local, which would shift the
+    // age by the TZ offset and could flip a boundary decision.
+    const ages = parseS3ListAges(
+        ls(['2026-08-20 00:00:00        10 org-x/snap-ca/k.tar.zst']),
+        NOW
+    );
+    assert.ok(Math.abs(ages.get('k.tar.zst') - 1) < 1e-6);
+});
+
+test('unparseable and empty lines are skipped, not thrown on', () => {
+    const ages = parseS3ListAges(
+        ls([
+            '',
+            'Bucket: something',
+            'garbage',
+            '2026-08-19 00:00:00        10 org-x/snap-ca/good.tar.zst',
+            '2026-13-45 99:99:99        10 org-x/snap-ca/baddate.tar.zst'
+        ]),
+        NOW
+    );
+    assert.deepEqual([...ages.keys()], ['good.tar.zst']);
+});
+
+test('an empty listing yields null so the caller fails open', () => {
+    assert.equal(parseS3ListAges('', NOW), null);
+    assert.equal(parseS3ListAges('no objects here', NOW), null);
+});
+
+test('nearExpiry: young objects are skipped, old ones refreshed', () => {
+    const ages = new Map([['young', 0.5], ['old', 3.2]]);
+    assert.equal(nearExpiry(ages, 'young', 1), false);
+    assert.equal(nearExpiry(ages, 'old', 1), true);
+});
+
+test('nearExpiry is inclusive at the threshold', () => {
+    const ages = new Map([['exact', 1]]);
+    assert.equal(nearExpiry(ages, 'exact', 1), true);
+});
+
+test('nearExpiry fails OPEN when the listing failed', () => {
+    // Skipping wrongly costs the layer; refreshing wrongly costs one request.
+    assert.equal(nearExpiry(null, 'anything', 1), true);
+});
+
+test('nearExpiry fails OPEN for a key absent from the listing', () => {
+    // Raced with an upload, or listed under an unexpected shape.
+    assert.equal(nearExpiry(new Map([['other', 0.1]]), 'missing', 1), true);
+});
+
+test('worst-case age stays under the 7-day lifecycle at the shipped default', () => {
+    // An active repo builds many times an hour on weekdays, so the binding case
+    // is the weekend: observed inter-build gaps reach ~2.4 days. Shipped
+    // threshold is 1 day.
+    const LIFECYCLE_DAYS = 7;
+    const LONGEST_OBSERVED_GAP = 2.37;
+    const worst = worstCaseAgeDays(1, LONGEST_OBSERVED_GAP);
+    assert.ok(
+        worst < LIFECYCLE_DAYS,
+        `worst-case age ${worst}d must stay under the ${LIFECYCLE_DAYS}d lifecycle`
+    );
+    // And keep real headroom, not just squeak under.
+    assert.ok(LIFECYCLE_DAYS - worst > 3, 'want >3 days of quiet-stretch tolerance');
+});
+
+test('a threshold that would expire referenced layers is detectable', () => {
+    // Guards the reasoning itself: at threshold 5 with the same gap the policy
+    // would let a referenced layer age out. This is the check to re-run before
+    // ever raising BP_CACHE_REFRESH_AGE_DAYS.
+    assert.ok(worstCaseAgeDays(5, 2.37) > 7);
+});

package.json

@@ -4,7 +4,8 @@
   "private": true,
   "description": "BuildPulse setup-docker-builder GitHub Action",
   "scripts": {
-    "build": "ncc build src/index.js -o dist --minify && ncc build src/post.js -o dist-post --minify && mv dist-post/index.js dist/post.js && rm -rf dist-post"
+    "build": "ncc build src/index.js -o dist --minify && ncc build src/post.js -o dist-post --minify && mv dist-post/index.js dist/post.js && rm -rf dist-post",
+    "test": "node --test __tests__/*.test.js"
   },
   "dependencies": {
     "@actions/core": "^1.11.1"

src/post.js

@@ -10,6 +10,7 @@
 // interpolated through a shell. Tenant/bucket are additionally charset-validated
 // before they reach the S3 URI, so a hostile value can neither inject nor traverse.
 const core = require('@actions/core');
+const { parseS3ListAges, nearExpiry } = require('./refresh-policy');
 const fs = require('fs');
 const { execFileSync } = require('child_process');
 
@@ -285,8 +286,60 @@ function commitToS3(bucket, ns, region) {
   //    layers referenced by NOBODY age out. Abandoned tenants never run this, so their whole
   //    prefix (v1 leftovers included) ages out. One `aws s3 cp` per prefix (the CLI
   //    parallelizes), filtered with --exclude '*' + per-object --include.
-  const snapInc = [...new Set(Object.values(manifest))].flatMap((sha) => ['--include', `${sha}.tar.zst`]);
-  const blobInc = blobList.flatMap((n) => ['--include', `sha256/${n}`]);
+  //    Only objects actually CLOSE TO EXPIRY are copied. Touching one with six days left
+  //    on a seven-day rule buys nothing, and every copy is billed as a Tier1 PUT — these
+  //    refreshes measured ~3.8M Tier1 requests over 20 days, 83% of the entire S3 bill for
+  //    runner caching, dwarfing storage itself. A single LIST (Tier2, ~12x cheaper per
+  //    call, paginated 1000 keys at a time) gives every object's age, collapsing the copies
+  //    to just the ones about to age out.
+  //
+  //    The GC property is UNCHANGED. An object is refreshed only when this build references
+  //    it AND it is near expiry; unreferenced orphans are still never touched and still age
+  //    out. Skipping a YOUNG referenced object cannot expire it — by definition it has days
+  //    of life left, and any build inside that window refreshes it then. The only case that
+  //    changes is a tenant whose build interval exceeds the remaining headroom, and that
+  //    tenant's objects expire under the old code too.
+  //
+  //    Fails OPEN: if the LIST errors or yields nothing usable, ages is null and every
+  //    referenced object is refreshed exactly as before. A cost optimisation must never make
+  //    the cache less durable.
+  //    Threshold sizing. An object's worst-case age is `threshold + longest gap between
+  //    builds that reference it`, and that must stay under the bucket's lifecycle (7 days)
+  //    or a referenced layer expires. An active repo builds many times an hour on weekdays,
+  //    so the binding case is the weekend: observed inter-build gaps reach ~2.4 days. A
+  //    1-day threshold therefore caps age near 3.4d, leaving ~3.6 days of tolerance for an
+  //    unusually quiet stretch.
+  //
+  //    1 rather than 3 on purpose: a repo building hourly already skips the overwhelming
+  //    majority of refreshes at 1 day, and raising it to 3 buys little on what remains
+  //    while cutting the quiet-stretch tolerance from ~3.6 days to ~1.6. Nearly all of the
+  //    saving with most of the margin intact.
+  //
+  //    Raise it only alongside evidence about build cadence, and never above
+  //    (lifecycle_days - longest_observed_gap).
+  const REFRESH_WHEN_AGE_DAYS = Number(process.env.BP_CACHE_REFRESH_AGE_DAYS || 1);
+  //    Parsing and the fail-open rule live in src/refresh-policy.js so they can be tested;
+  //    only the AWS call and error handling stay here.
+  const objectAgesDays = (pfx) => {
+    try {
+      const out = awsOut(['s3', 'ls', `${base}/${pfx}/`, '--recursive', '--region', region]).toString();
+      return parseS3ListAges(out, Date.now());
+    } catch (e) {
+      core.warning(`cache age listing (${pfx}) failed, refreshing every referenced object: ${e.message}`);
+      return null;
+    }
+  };
+
+  const snapAges = objectAgesDays('snap-ca');
+  const blobAges = objectAgesDays('blobs');
+  const snapKeys = [...new Set(Object.values(manifest))].map((sha) => `${sha}.tar.zst`);
+  const blobKeys = blobList.map((n) => `sha256/${n}`);
+  const snapInc = snapKeys.filter((k) => nearExpiry(snapAges, k, REFRESH_WHEN_AGE_DAYS)).flatMap((k) => ['--include', k]);
+  const blobInc = blobKeys.filter((k) => nearExpiry(blobAges, k, REFRESH_WHEN_AGE_DAYS)).flatMap((k) => ['--include', k]);
+  core.info(
+    `cache refresh: ${snapInc.length / 2}/${snapKeys.length} snapshot(s) + ${blobInc.length / 2}/${blobKeys.length} blob(s) ` +
+      `refreshed (>=${REFRESH_WHEN_AGE_DAYS}d old); the rest still have lifecycle headroom`
+  );
   // Batch the --include flags so a tenant with thousands of objects can't blow past OS
   // ARG_MAX in a single exec (a failed refresh would silently skip that build's touch, and
   // its objects could then age out). Each object contributes 2 argv entries; 1000/call keeps

src/refresh-policy.js

@@ -0,0 +1,75 @@
+'use strict';
+
+// Refresh policy for the S3 layer cache — extracted so it can be tested.
+//
+// The cache bucket expires objects on LastModified. An ACTIVE tenant keeps its
+// REFERENCED layers alive by copying them in place (REPLACE metadata, no data
+// transfer), which resets that clock. Unreferenced orphans are deliberately
+// never touched, so they age out — that is the entire garbage collection story.
+//
+// Copying every referenced object on every commit is what made this expensive:
+// each copy is billed as a Tier1 PUT, and across the fleet those refreshes were
+// ~3.8M Tier1 requests in 20 days — 83% of the whole S3 bill for runner caching,
+// far more than storage. Touching an object with six days left on a seven-day
+// rule buys nothing.
+//
+// So: one LIST per prefix (Tier2, ~12x cheaper per call) gives every object's
+// age, and only those near expiry are copied. The GC property is unchanged —
+// an object is refreshed only when a build references it AND it is near expiry.
+
+/**
+ * Parse `aws s3 ls <uri> --recursive` output into key -> age in days.
+ *
+ * Lines look like:
+ *   2026-08-20 11:40:12       2023 <ns>/<prefix>/<key>
+ *
+ * The timestamp is UTC. Keys are returned relative to `<ns>/<prefix>/` so they
+ * match the `--include` patterns the caller builds. Unparseable lines are
+ * skipped rather than throwing: a listing that is partly unreadable should
+ * degrade to "refresh more than strictly necessary", never to a crash in a
+ * post-step that runs after a successful build.
+ *
+ * @param {string} text  raw stdout
+ * @param {number} nowMs Date.now() equivalent, injectable for tests
+ * @returns {Map<string, number>|null} null when nothing parsed (caller fails open)
+ */
+function parseS3ListAges(text, nowMs) {
+    const ages = new Map();
+    for (const line of String(text).split('\n')) {
+        const m = line.match(/^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\s+\d+\s+(.+)$/);
+        if (!m) continue;
+        const key = m[2].split('/').slice(2).join('/'); // strip "<ns>/<prefix>/"
+        if (!key) continue;
+        const t = Date.parse(m[1].replace(' ', 'T') + 'Z');
+        if (Number.isNaN(t)) continue;
+        ages.set(key, (nowMs - t) / 86400000);
+    }
+    return ages.size ? ages : null;
+}
+
+/**
+ * Should this referenced object be refreshed?
+ *
+ * Fails OPEN in both unknown cases — a null map (the listing failed) and a key
+ * absent from the map (raced with an upload, or listed under an unexpected
+ * shape). Refreshing unnecessarily costs a request; skipping wrongly costs the
+ * layer. A cost optimisation must never make the cache less durable.
+ */
+function nearExpiry(ages, key, thresholdDays) {
+    if (!ages) return true;
+    const age = ages.get(key);
+    if (age === undefined) return true;
+    return age >= thresholdDays;
+}
+
+/**
+ * Largest age an object can reach under this policy: it can sit just under the
+ * threshold when a build runs, then wait a full inter-build gap before the next
+ * one refreshes it. Must stay below the bucket's lifecycle or a referenced
+ * layer expires.
+ */
+function worstCaseAgeDays(thresholdDays, longestBuildGapDays) {
+    return thresholdDays + longestBuildGapDays;
+}
+
+module.exports = { parseS3ListAges, nearExpiry, worstCaseAgeDays };