feat(storage): browse, download, and delete a bucket's objects

src/commands/services.ts

@@ -47,17 +47,22 @@ export function resolveServiceId(services: Array<{ id: string; type: string; nam
   return svc.id
 }
 
-// Resolve a compute service id: by name, or the sole compute service when name is omitted.
-export function resolveComputeServiceId(services: Array<{ id: string; type: string; name: string }>, name?: string): string {
-  const compute = services.filter((s) => s.type === 'compute')
+// Resolve one service of a type: by name, or the sole one of that type when name is omitted.
+export function resolveSoleService<T extends { id: string; type: string; name: string }>(services: T[], type: string, name?: string): T {
+  const of = services.filter((s) => s.type === type)
   if (name) {
-    const svc = compute.find((s) => s.name === name)
-    if (!svc) throw new Error(`compute service not found: ${name}`)
-    return svc.id
+    const svc = of.find((s) => s.name === name)
+    if (!svc) throw new Error(`${type} service not found: ${name}`)
+    return svc
   }
-  if (compute.length === 0) throw new Error('no compute service in this project (add one with `insta services add compute <name>`)')
-  if (compute.length > 1) throw new Error(`multiple compute services — specify one: ${compute.map((s) => s.name).join(', ')}`)
-  return compute[0]!.id
+  if (of.length === 0) throw new Error(`no ${type} service in this project (add one with \`insta services add ${type} <name>\`)`)
+  if (of.length > 1) throw new Error(`multiple ${type} services — specify one: ${of.map((s) => s.name).join(', ')}`)
+  return of[0]!
+}
+
+// Resolve a compute service id: by name, or the sole compute service when name is omitted.
+export function resolveComputeServiceId(services: Array<{ id: string; type: string; name: string }>, name?: string): string {
+  return resolveSoleService(services, 'compute', name).id
 }
 
 // ---- commands ----

src/commands/storage.ts

@@ -0,0 +1,176 @@
+// `insta storage` — browse, download, and delete the objects in a storage service's bucket.
+import { chmod, rename, rm, stat } from 'node:fs/promises'
+import { createWriteStream, rmSync } from 'node:fs'
+import { randomBytes } from 'node:crypto'
+import { Readable } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
+import { ApiClient, requireProject } from '../api.js'
+import { info, printJson, handleApproval } from '../util.js'
+import { q, resolveSoleService } from './services.js'
+import { fmtBytes } from './db.js' // the repo's tested bytes formatter — don't grow a third copy
+
+type Common = { branch?: string; service?: string; json?: boolean }
+
+function qs(params: Record<string, string | undefined>): string {
+  const u = new URLSearchParams()
+  for (const [k, v] of Object.entries(params)) if (v !== undefined && v !== '') u.set(k, String(v))
+  const s = u.toString()
+  return s ? `?${s}` : ''
+}
+
+// Page size the listing route accepts. Junk must fail here, not travel as `limit=NaN`.
+export function parseObjectLimit(raw: string): number {
+  const n = Number(raw)
+  if (!Number.isInteger(n) || n < 1 || n > 1000) throw new Error(`--limit must be an integer 1..1000, got: ${raw}`)
+  return n
+}
+
+type ObjectParams = { branch?: string; prefix?: string; cursor?: string; limit?: number; key?: string }
+
+// pure: platform path for the objects collection — GET lists it, DELETE removes one `key`.
+export function objectsPath(projectId: string, serviceId: string, params: ObjectParams): string {
+  const { limit, ...rest } = params
+  return `/projects/${projectId}/services/${serviceId}/objects${qs({ ...rest, limit: limit === undefined ? undefined : String(limit) })}`
+}
+
+// pure: the presign route — a static subpath, so keys containing `/` stay in the query.
+export function objectDownloadPath(projectId: string, serviceId: string, params: { branch?: string; key: string }): string {
+  return `/projects/${projectId}/services/${serviceId}/objects/download${qs(params)}`
+}
+
+// pure: one `storage list` row, size-first so the columns line up over variable-length keys.
+export function objectListLine(o: { key: string; size?: number; lastModified?: string }): string {
+  const size = typeof o.size === 'number' ? fmtBytes(o.size) : '—'
+  return `${size.padStart(10)}  ${(o.lastModified ?? '—').padEnd(24)}  ${o.key}`
+}
+
+// Resolve the branch's storage service (named, or the sole one) — its bucket is what we browse.
+async function storageTarget(api: ApiClient, projectId: string, branch: string | undefined, name?: string): Promise<{ id: string; name: string }> {
+  const { services } = await api.request('GET', `/projects/${projectId}/services${q(branch)}`)
+  return resolveSoleService(services as Array<{ id: string; type: string; name: string }>, 'storage', name)
+}
+
+type ListOpts = Common & { prefix?: string; cursor?: string; limit?: string }
+
+// S3 filters by prefix only — there is no substring search, so `--prefix` is the query surface.
+export async function storageList(opts: ListOpts): Promise<void> {
+  const limit = opts.limit === undefined ? undefined : parseObjectLimit(opts.limit)
+  const api = await ApiClient.load()
+  const p = await requireProject()
+  const branch = opts.branch ?? p.branch
+  const svc = await storageTarget(api, p.projectId, branch, opts.service)
+  const res = await api.rawRequest('GET', objectsPath(p.projectId, svc.id, { branch, prefix: opts.prefix, cursor: opts.cursor, limit }))
+  if (handleApproval(res)) return
+  if (opts.json) return printJson(res.body)
+  const objects: Array<{ key: string; size?: number; lastModified?: string }> = res.body?.objects ?? []
+  if (!objects.length) {
+    return info(opts.prefix ? `(no objects under prefix ${opts.prefix} in storage/${svc.name})` : `(storage/${svc.name} is empty)`)
+  }
+  for (const o of objects) info(objectListLine(o))
+  const next = res.body?.nextCursor
+  if (next) info(`  (more — next page: ${nextPageCommand({ ...opts, limit }, next)})`)
+}
+
+// Safe unquoted in every shell we care about — no spaces, quotes, or expansion characters.
+const SHELL_SAFE = /^[\w./:@=+-]+$/
+
+// The continuation must repeat the filters, or following it lists a different set. Quoting rules
+// differ between sh and PowerShell, so rather than guess the shell, a value that would need quotes
+// gets a sentence instead of syntax that is broken somewhere.
+export function nextPageCommand(opts: { branch?: string; service?: string; prefix?: string; limit?: number }, cursor: string): string {
+  const values = [opts.service, opts.branch, opts.prefix, cursor].filter((v): v is string => !!v)
+  if (!values.every((v) => SHELL_SAFE.test(v))) {
+    return `re-run this command with --cursor set to ${cursor}`
+  }
+  const flags = [
+    opts.service ? `--service ${opts.service}` : '',
+    opts.branch ? `--branch ${opts.branch}` : '',
+    opts.prefix ? `--prefix ${opts.prefix}` : '',
+    opts.limit === undefined ? '' : `--limit ${opts.limit}`,
+    `--cursor ${cursor}`,
+  ].filter(Boolean)
+  return `insta storage list ${flags.join(' ')}`
+}
+
+export type GetDeps = { streamTo?: (url: string, out: string) => Promise<number> }
+
+// pure: where the bytes land. Only the last segment is used, so no key can escape cwd.
+export function outputPath(key: string, output?: string): string {
+  if (output) return output
+  // Split on `\` too: a key may contain one, and on Windows that is also a separator.
+  const base = key.split(/[\\/]/).pop() ?? ''
+  if (!base) throw new Error(`cannot infer a filename from key "${key}" — pass -o <file>`)
+  return base
+}
+
+// Stream from the provider (never through the platform, which only signs) straight to disk, so a
+// multi-gigabyte object never has to fit in memory. Returns the byte count written.
+export async function streamPresignedTo(url: string, out: string, fetchImpl: typeof fetch = fetch): Promise<number> {
+  const res = await fetchImpl(url)
+  if (!res.ok || !res.body) throw new Error(`download failed: HTTP ${res.status} (a presigned URL lives ~60s — re-run to mint a fresh one)`)
+  let written = 0
+  const counting = new TransformStream<Uint8Array, Uint8Array>({
+    transform(chunk, controller) { written += chunk.byteLength; controller.enqueue(chunk) },
+  })
+  // Write beside the target, then rename: opening `out` directly would truncate an existing file
+  // that a failed download then deletes. Same directory keeps the rename atomic.
+  const part = `${out}.insta-part-${randomBytes(4).toString('hex')}`
+  // A signal kills the process without unwinding, so the part file needs a synchronous sweep.
+  // Exit 128+signo, so a supervisor still reads interrupted (130) apart from terminated (143).
+  const sweep = (signo: number) => () => { rmSync(part, { force: true }); process.exit(128 + signo) }
+  const onInt = sweep(2)
+  const onTerm = sweep(15)
+  process.once('SIGINT', onInt)
+  process.once('SIGTERM', onTerm)
+  try {
+    await pipeline(Readable.fromWeb(res.body.pipeThrough(counting) as ReadableStream<Uint8Array>), createWriteStream(part))
+    // Replacing a 0600 file must not widen it to the umask default the part was created with.
+    const mode = await stat(out).then((s) => s.mode, () => undefined)
+    if (mode !== undefined) await chmod(part, mode)
+    await rename(part, out)
+  } catch (e) {
+    await rm(part, { force: true })
+    throw e
+  } finally {
+    process.off('SIGINT', onInt)
+    process.off('SIGTERM', onTerm)
+  }
+  return written
+}
+
+// Core, dependency-injected for tests (mirrors runWithSecrets): stream → disk, return byte count.
+export async function saveObject(url: string, out: string, deps: GetDeps = {}): Promise<number> {
+  return (deps.streamTo ?? streamPresignedTo)(url, out)
+}
+
+type GetOpts = Common & { output?: string }
+
+export async function storageGet(key: string, opts: GetOpts, deps: GetDeps = {}): Promise<void> {
+  if (!key) throw new Error('key is required')
+  const api = await ApiClient.load()
+  const p = await requireProject()
+  const branch = opts.branch ?? p.branch
+  const svc = await storageTarget(api, p.projectId, branch, opts.service)
+  const res = await api.rawRequest('GET', objectDownloadPath(p.projectId, svc.id, { branch, key }))
+  if (handleApproval(res)) return
+  // --json hands over the presigned URL instead of downloading, as `insta secrets --json` does.
+  // Before outputPath, so a key with no filename still works when nothing is written to disk.
+  if (opts.json) return printJson(res.body)
+  const out = outputPath(key, opts.output)
+  if (!res.body?.url) throw new Error('the platform returned no download URL')
+  const bytes = await saveObject(res.body.url, out, deps)
+  info(`wrote ${fmtBytes(bytes)} to ${out} (${key} from storage/${svc.name}, branch ${branch})`)
+}
+
+// No prompt, matching every other destructive command here — the governance gate is the guard.
+export async function storageDelete(key: string, opts: Common): Promise<void> {
+  if (!key) throw new Error('key is required')
+  const api = await ApiClient.load()
+  const p = await requireProject()
+  const branch = opts.branch ?? p.branch
+  const svc = await storageTarget(api, p.projectId, branch, opts.service)
+  const res = await api.rawRequest('DELETE', objectsPath(p.projectId, svc.id, { branch, key }))
+  if (handleApproval(res)) return
+  if (opts.json) return printJson(res.body)
+  info(`deleted ${key} from storage/${svc.name} (branch ${branch})`)
+}

src/index.ts

@@ -18,6 +18,7 @@ import * as secretsCmd from './commands/secrets.js'
 import { deploy } from './commands/deploy.js'
 import * as computeCmd from './commands/compute.js'
 import * as dbCmd from './commands/db.js'
+import * as storageCmd from './commands/storage.js'
 import { manifest } from './commands/manifest.js'
 import * as govern from './commands/govern.js'
 import * as observe from './commands/observe.js'
@@ -204,6 +205,26 @@ db.command('volume').description("Show or grow a postgres service's provisioned
   .option('--json').option('--branch <branch>', 'branch (default: current)').option('--group <g>', 'postgres service name (default: the sole/default one)')
   .action(guard((o) => dbCmd.dbVolume(o)))
 
+// ---- storage (bucket objects) ----
+const storage = program.command('storage').description("Browse, download, and delete a storage service's bucket objects")
+storage.command('list').description("List the bucket's objects. S3 filters by prefix only — there is no substring search")
+  .option('--prefix <p>', 'only keys starting with this prefix (applied server-side)')
+  .option('--cursor <c>', 'continue from the nextCursor a previous page printed')
+  .option('--limit <n>', 'page size, 1..1000 (default 100)')
+  .option('--service <name>', 'storage service (default: the sole one on the branch)')
+  .option('--branch <b>', 'branch (default: current)').option('--json')
+  .action(guard((o) => storageCmd.storageList(o)))
+storage.command('get <key>').description('Download one object to disk through a short-lived presigned URL (bytes come straight from the provider)')
+  .option('-o, --output <file>', "output file (default: the key's last segment)")
+  .option('--service <name>', 'storage service (default: the sole one on the branch)')
+  .option('--branch <b>', 'branch (default: current)')
+  .option('--json', 'print the presigned URL + expiry instead of downloading')
+  .action(guard((key, o) => storageCmd.storageGet(key, o)))
+storage.command('delete <key>').description('DELETES one object from the bucket immediately — no undo, and an already-gone key still reports success (gated: storage.delete)')
+  .option('--service <name>', 'storage service (default: the sole one on the branch)')
+  .option('--branch <b>', 'branch (default: current)').option('--json')
+  .action(guard((key, o) => storageCmd.storageDelete(key, o)))
+
 // ---- manifest ----
 program.command('manifest').description('Print an agent-legible view of the project environments').option('--json').action(guard((o) => manifest(o)))
 
@@ -249,7 +270,7 @@ ob.command('sync').description('Upload findings into the project timeline').acti
 // ---- policy ----
 const pol = program.command('policy').description('Governance policy')
 pol.command('get').option('--json').action(guard((o) => govern.policyGet(o)))
-pol.command('set <action> <decision>').description('action: secrets.read|secrets.write|deploy|project.delete|branch.delete|service.add|service.remove|service.scale|service.upgrade|service.setAccess; decision: allow|deny|approve').action(guard((a, d) => govern.policySet(a, d)))
+pol.command('set <action> <decision>').description('action: secrets.read|secrets.write|deploy|project.delete|branch.delete|service.add|service.remove|service.scale|service.upgrade|service.setAccess|storage.read|storage.write|storage.delete; decision: allow|deny|approve').action(guard((a, d) => govern.policySet(a, d)))
 
 // ---- self-update ----
 program.command('upgrade').description('Update the insta CLI to the latest release (binary or npm install)')