From 4da1abaa63d9d707ced2e5a44532bffdf72ba2b2 Mon Sep 17 00:00:00 2001 From: Efi Jeremiah Date: Tue, 29 Sep 2026 22:22:37 +0300 Subject: [PATCH] feat: delta runs, signed and proven rounds, one honest progress bar, gentle by default Repeat runs recompute only what changed: - cached parses and stored results are keyed on a generated fingerprint of the parsing and analysis code, not the release number - staleness is decided on the evidence's content, not manifest mtime - the collect-time host witness refreshes the per-artifact overlay instead of fully parsing every transcript; analysis decodes events once - per-artifact content scans are memoized (authenticated with the machine key); rule-pack regexes get an exact case-folding literal prefilter and event subjects are prepared once per scope Rounds are signed with a per-machine key, anchored outside the case and proven (signature, anchor, quick verify) before the next round is added; failures are recorded in the round and exit 4. Rounds are transactional: an unsealed round is rolled back to the last seal and logged as round_aborted. The reused overlay is integrity-checked (segment and event hashes, MAC over its state). One progress display for the whole run with elapsed time and an ETA from this machine's timing history that counts down steadily. Gentle by default: lowered priority, capped workers and heap, paced reads, a pause under load and a free-disk floor. Fixes: run --sign signed before sealing; carried-forward records dropped their inode/ctime so alternate runs re-read everything; retiring excluded records forced a full re-parse on every analysis; carried-forward bytes made the collect ETA collapse to zero. Co-Authored-By: Claude --- CHANGELOG.md | 65 +++ README.md | 3 + SECURITY.md | 32 ++ docs/adfir-spec.md | 63 ++- internal/analysis/analysis.go | 224 ++++++--- internal/analysis/delta_test.go | 306 +++++++++++++ internal/analysis/incremental_equiv_test.go | 51 ++- internal/artmemo/artmemo.go | 199 ++++++++ internal/artmemo/artmemo_test.go | 87 ++++ internal/casepkg/casepkg.go | 150 +++++- internal/casepkg/pending.go | 237 ++++++++++ internal/casepkg/pending_test.go | 175 +++++++ internal/casepkg/retire.go | 18 +- internal/casepkg/retire_test.go | 22 + internal/cli/cli.go | 28 +- internal/cli/correlate_cmd.go | 5 + internal/cli/import_cmd.go | 14 +- internal/cli/main_test.go | 28 ++ internal/cli/pkgopen.go | 74 ++- internal/cli/progress.go | 133 +----- internal/cli/run_cmd.go | 441 ++++++++++++------ internal/cli/run_delta_test.go | 75 +++ internal/collector/collector.go | 45 +- internal/collector/parallel_test.go | 19 + internal/detect/content_scan.go | 26 +- internal/fingerprint/compute/compute.go | 143 ++++++ internal/fingerprint/fingerprint.go | 26 ++ internal/fingerprint/fingerprint_test.go | 34 ++ internal/fingerprint/gen/main.go | 35 ++ internal/fingerprint/zz_generated.go | 8 + internal/gentle/disk_unix.go | 23 + internal/gentle/gentle.go | 301 ++++++++++++ internal/gentle/gentle_test.go | 141 ++++++ internal/gentle/sys_darwin.go | 65 +++ internal/gentle/sys_linux.go | 91 ++++ internal/gentle/sys_windows.go | 100 ++++ internal/integrity/integrity.go | 198 ++++++++ internal/integrity/integrity_test.go | 280 ++++++++++++ internal/integrity/sha_test.go | 21 + internal/normalize/current.go | 202 ++++++++ internal/normalize/normalize.go | 3 + internal/normalize/segments.go | 164 +++++-- internal/progress/model.go | 146 ++++++ internal/progress/tracker.go | 482 ++++++++++++++++++++ internal/progress/tracker_test.go | 147 ++++++ internal/rulepack/prefilter.go | 155 +++++++ internal/rulepack/prefilter_test.go | 130 ++++++ internal/rulepack/rulepack.go | 95 ++-- internal/rulepack/subjects.go | 88 ++++ internal/seal/seal.go | 42 ++ internal/serve/serve_test.go | 8 + internal/store/keys.go | 208 +++++++++ 52 files changed, 5415 insertions(+), 441 deletions(-) create mode 100644 internal/analysis/delta_test.go create mode 100644 internal/artmemo/artmemo.go create mode 100644 internal/artmemo/artmemo_test.go create mode 100644 internal/casepkg/pending.go create mode 100644 internal/casepkg/pending_test.go create mode 100644 internal/cli/main_test.go create mode 100644 internal/cli/run_delta_test.go create mode 100644 internal/fingerprint/compute/compute.go create mode 100644 internal/fingerprint/fingerprint.go create mode 100644 internal/fingerprint/fingerprint_test.go create mode 100644 internal/fingerprint/gen/main.go create mode 100644 internal/fingerprint/zz_generated.go create mode 100644 internal/gentle/disk_unix.go create mode 100644 internal/gentle/gentle.go create mode 100644 internal/gentle/gentle_test.go create mode 100644 internal/gentle/sys_darwin.go create mode 100644 internal/gentle/sys_linux.go create mode 100644 internal/gentle/sys_windows.go create mode 100644 internal/integrity/integrity.go create mode 100644 internal/integrity/integrity_test.go create mode 100644 internal/integrity/sha_test.go create mode 100644 internal/normalize/current.go create mode 100644 internal/progress/model.go create mode 100644 internal/progress/tracker.go create mode 100644 internal/progress/tracker_test.go create mode 100644 internal/rulepack/prefilter.go create mode 100644 internal/rulepack/prefilter_test.go create mode 100644 internal/rulepack/subjects.go create mode 100644 internal/store/keys.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 013639c..6bf9a29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ and the project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. Lessons from the 2026 agent-driven cloud attacks: Microsoft's Storm-3168, Sysdig's JADEPUFFER and Sygnia's AI-assisted intrusion. +Repeat runs are a true delta, every round is signed and proven before the +next is added, and a run never chokes the machine. + ### Added - **Cloud audit logs as a second witness.** `--endpoint` (on `analyze`, `run` and `correlate`) now reads AWS CloudTrail, Azure Activity Log and @@ -46,6 +49,17 @@ Sysdig's JADEPUFFER and Sygnia's AI-assisted intrusion. pen-tester writes exactly that. - Secret formats `ALIBABA_ACCESS_KEY` (`LTAI…`) and `TENCENT_SECRET_ID` (`AKID…`). +- **Every round is signed** with a per-machine key (`--sign` for another + key, `--no-sign` to opt out), and its digest is recorded in an anchor + log outside the case and printed. Before a round is added, the previous + signature, the anchor and the sealed files are verified + (`--verify-prior full` also re-hashes every blob); a failure is recorded + in the round (`prior_integrity: FAILED`) and the run exits 4. +- Each round archives the previous signature to `seals/SEAL.sig.` and + records `prev_seal_sha256`, `signer` and `prior_integrity`. +- The reused analysis overlay is integrity-checked: segment and event + hashes, and a MAC over its state under the machine key. Anything that + does not match is rebuilt from the sealed evidence. ### Changed - `CLOUD_CREDENTIAL_EXPORT` also covers listing storage account keys and @@ -54,10 +68,61 @@ Sysdig's JADEPUFFER and Sygnia's AI-assisted intrusion. - The Protect tab's steps for a leaked secret say that deleting it from an issue, pull request or commit does not remove it from edit history, forks or caches. +- **The content scans remember each artifact's result** (rule packs over + raw transcripts and configs; credential, injection and invisible-character + scans), keyed on its content address, its record, the analysis code and + the rules, and authenticated with the machine key. An unchanged + transcript is not scanned again. Rule-pack regexes are also prefiltered on + the literals every match must contain (case-folded exactly as `(?i)` + folds), and each event's match subject is prepared once instead of once + per rule. On a real case (359,000 events) a repeat analysis went from + 111 s to 42 s, and a repeat `run` from about 5 minutes to under 1. +- **Repeat `run` recomputes only what changed.** Cached parses and stored + results are keyed on a fingerprint of the parsing and analysis code, not + on the release number, so a release that changes no parser or rule no + longer re-parses and re-analyzes every case. Staleness is decided on the + content of the evidence, not the manifest's modification time: a round + that only carried files forward rebuilds nothing, and when neither the + evidence nor the code changed, `run` reuses the stored results outright + (`--reanalyze` forces it). + The first run after upgrading re-parses once: caches written by earlier + versions carry no fingerprint and no segment hashes, so they are not + trusted. +- The host-witness step at collection time refreshes the per-artifact + overlay instead of fully parsing every transcript; analysis then finds + the overlay current. Analysis decodes the events once instead of seven + times, and writes annotated events once. +- **One progress display for the whole run**: an overall bar, elapsed time, + and a time remaining predicted from this machine's own history + (`perf.jsonl` in the home), corrected by today's pace and counting down + steadily. It never reads 0:00 while work remains. +- **Gentle by default** (`--priority gentle|background|normal`): lowered + CPU priority, at most half the CPUs, a soft heap limit + (`--max-memory-mb`), reads capped at 200 MB/s (`--max-read-mbps`), a + pause while the machine is under load (`--no-governor`), and a free-disk + floor it never crosses (`--min-free-gb`; exit 5 when a round cannot fit). ### Fixed - Re-running analysis with the same endpoint log appended the same corroboration note to an event again on every run. +- **Every other repeat run re-read the whole profile.** A carried-forward + record dropped the inode and change time it was judged unchanged on, so + the next round had nothing to compare and re-read those files — 2.7 GB on + a real machine, on alternate runs. +- **Every analysis re-parsed the whole case** once any `node_modules`/`.git` + records existed: retiring them from the scan set counted every excluded + record (and each round's own policy placeholders) as newly retired and + forced a full re-parse. It now counts only newly retired content, and + retiring is an incremental rebuild. +- **An interrupted round no longer damages the case.** A round that never + sealed (an error, Ctrl+C, a crash) left records in both hash chains that + no seal covered, so the case stopped verifying. Rounds are now + transactional: an unsealed round is rolled back to the last seal and the + next round records it as `round_aborted`. +- `run --sign` signed before sealing, so its signature covered the previous + round's `SHA256SUMS` and never verified. +- The collect-step time remaining collapsed to about zero on repeat runs: + carried-forward files counted as bytes read. ## [3.1.2] — 2026-09-29 diff --git a/README.md b/README.md index 473374c..e0e393e 100644 --- a/README.md +++ b/README.md @@ -247,6 +247,9 @@ that case instead of producing another multi-gigabyte copy. A second round: - carries forward files that are unchanged by **size, inode and ctime** — never mtime alone, which any writer can set — recorded as `carried_forward` with the round that actually read them, so carried evidence is never presented as a fresh acquisition - stores only the **new tail** of a transcript that grew, after proving the earlier bytes still hash to what was preserved - continues both hash chains from their previous last line (a chain that is already broken is refused, not extended) and archives the seal that closed the previous round +- **proves the earlier rounds first**: every round is signed (a per-machine key, or `--sign `), its digest is recorded in an anchor log outside the case and printed, and the next run checks the signature, the anchor and the sealed files before adding anything. A failure is recorded in the new round for good and the run exits 4 +- **re-analyzes only what changed**: only new or grown transcripts are parsed, and when neither the evidence nor the parsing and rule code changed, the stored results are reused outright. A new release that doesn't touch parsers or rules doesn't trigger a re-analysis +- **stays out of the way**: lowered priority, half the CPUs, a soft memory cap, paced reads, a pause while the machine is busy, and a free-disk floor it won't cross (`--priority normal` for full speed). One progress bar covers the whole run, with elapsed time and a time remaining learned from this machine's earlier runs On the same machine as above, a second run re-read 10 files, carried 8,269 forward, and added **157 KB** to disk. diff --git a/SECURITY.md b/SECURITY.md index f25c70e..d65abf2 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -40,6 +40,38 @@ passed through a session. Treat it as you would any evidence store. other case holds its own real link to the bytes it needs. - `--no-share` keeps a case's bytes entirely inside its own directory. +### Signed rounds and what they prove + +Every round is signed with this machine's key (`keys/seal.ed25519` in the +home, `0600`, created on first use) unless `--sign` names another key or +`--no-sign` is given, and each sealed round's digest is appended to +`anchors.jsonl` in the home and printed at the end of the run. Before a +new round is added, the previous signature, the anchor and the sealed +files are checked; a failure is recorded in the new round permanently and +the run exits 4. + +Limits, stated plainly: the key and the anchor log sit on the same machine, +under the same account, as the evidence. They stop anyone who can only +reach the case directory (a share, a copy, a backup restore), and they +catch accidental corruption, but someone who controls the account can +rewrite a case, re-sign it and edit the anchor log. The digest printed at +the end of each run — pasted into a ticket or case notes somewhere else — +is what that attacker cannot reach. Sign with an external key (`--sign`) +when that matters. + +The analysis overlay is reused across rounds. It is authenticated with a +key derived from the machine key and every cached file is hashed; anything +that does not match is rebuilt from the sealed evidence, never trusted. + +### Resource use + +`agentdfir run` is gentle by default: lowered CPU priority, at most half the +CPUs (4 workers), a soft heap limit, acquisition reads capped at 200 MB/s, +a pause while the machine is under load, and a free-disk floor (2 GiB, or +1% of the volume up to 5 GiB) it will not cross — it refuses to start, or +stops cleanly, instead. `--priority background` also lowers I/O priority; +`--priority normal` restores full speed. + A collect→seal cycle holds an exclusive lock on the package. If a run is killed, the next run reports and reclaims the stale lock; a lock held by a live process is never stolen. diff --git a/docs/adfir-spec.md b/docs/adfir-spec.md index 2993bf7..bc06d7b 100644 --- a/docs/adfir-spec.md +++ b/docs/adfir-spec.md @@ -20,13 +20,15 @@ case.adfir/ ├── chain-of-custody.jsonl # sealed: hash-chained custody log ├── case.json # sealed: case / operator / clock metadata, rounds ├── seals/SHA256SUMS. # sealed: the seal each earlier round was closed with +├── seals/SEAL.sig. # sealed: the signature over that seal, when it was signed ├── SHA256SUMS # sealed: covers the sealed zone exactly -├── SEAL.sig # optional: ed25519 detached signature +├── SEAL.sig # ed25519 signature over SHA256SUMS (every round, by default) ├── .lock # transient: held during a collect→seal cycle; not sealed ├── normalized/ # overlay: events / entities / relationships (JSONL) │ ├── events.jsonl # uncompressed: index/ addresses it by byte offset │ ├── events// # per-artifact segments (.jsonl.gz), the parse cache -│ └── state.json # which artifact each segment holds, and at what offset +│ ├── state.json # which artifact each segment holds, at what offset, with what hash +│ └── state.mac # HMAC of state.json under the machine's cache key, when it has one ├── detections/ # overlay: findings.json ├── index/ # overlay: events.idx, the explorer's offset index ├── reports/ # overlay: HTML/JSON/CSV/STIX/OTel @@ -34,9 +36,16 @@ case.adfir/ ``` `SHA256SUMS` covers exactly: `case.json`, the manifest, `collection.jsonl`, -`chain-of-custody.jsonl`, every `seals/SHA256SUMS.`, and every file in +`chain-of-custody.jsonl`, every file in `seals/`, and every file in `raw/`. Regenerating the overlay never changes the seal. +The overlay is derived and rebuildable, but it is reused across rounds, so +it is not trusted blindly: `state.json` records the sha256 of every cached +segment and of `events.jsonl`, and the code fingerprint and evidence digest +it was built from. A segment or `events.jsonl` that no longer matches, or a +`state.json` whose `state.mac` does not verify, is discarded and rebuilt +from the sealed zone. + Everything in the overlay is gzipped except `normalized/events.jsonl`, which stays plaintext because `index/events.idx` records a byte offset per event and a gzip stream cannot be seeked. The segments under @@ -121,11 +130,26 @@ A package may be collected into more than once. Each collection is a verify the whole existing chain before appending: extending a broken chain would hide the break behind valid-looking records. - Before writing a new `SHA256SUMS`, the previous one is copied to - `seals/SHA256SUMS.` where `n` is the round it closed. Earlier sealed - states stay provable. -- `case.json` gains a `rounds` array summarizing each round. + `seals/SHA256SUMS.` where `n` is the round it closed, and the previous + `SEAL.sig`, if any, to `seals/SEAL.sig.`. Both are covered by the new + `SHA256SUMS`, so every seal commits to all earlier seals and signatures. +- `case.json` gains a `rounds` array summarizing each round. A round records + `prev_seal_sha256` (the sha256 of the `SHA256SUMS` it replaced), `signer` + (the hex ed25519 public key it was sealed for) and `prior_integrity`: + what checking the earlier rounds found before this round was added — + `verified`, `unsigned`, or `FAILED` with `prior_integrity_problems`. - A producer MUST hold an exclusive lock (`.lock`) for a collect→seal cycle. `.lock` is not evidence and is not covered by `SHA256SUMS`. +- Rounds are transactional. Before a round writes anything, the producer + records the sealed state it starts from (the lengths of the manifest and + both chains, and `case.json`) in `.round-pending.json`. A round that is + not sealed — abandoned or crashed — is rolled back to exactly that state, + by the producer on exit or by the next round before it checks or extends + the package; the next round records each one as a `round_aborted` custody + event with what was discarded. A round is sealed once its new + `SHA256SUMS` is in place (written atomically, last); it is never rolled + back after that. `.round-pending.json` and `.round-aborted.json` are not + evidence and are not covered by `SHA256SUMS`. A file a later round did not re-read is recorded with `collection_method: "carried_forward"` and the `acquired_in_round` that @@ -134,6 +158,33 @@ acquired. A producer MUST decide "unchanged" on properties an unprivileged writer cannot forge (inode plus change time); modification time alone is not sufficient. +## Round integrity + +Before a producer adds a round to an existing package, and after taking +the lock but before writing anything, it SHOULD check the earlier rounds: + +1. `SEAL.sig` verifies over the current `SHA256SUMS`, with the public key + the last signed round recorded as its `signer`; +2. the current `SHA256SUMS` matches the producer's last recorded anchor for + this case (below); +3. a quick verification (steps 1, 2, 4, 5 of the next section) passes. + +A failure MUST NOT stop the new round, and MUST NOT be repaired: the round +is sealed with `prior_integrity: "FAILED"` and the problems listed, so the +failure is part of the record from then on. A `SEAL.sig` that does not +match when no round recorded a `signer` predates signed rounds (earlier +`run --sign` signed before sealing); it is reported and not trusted, and it +is not evidence of tampering. + +A producer SHOULD sign every round (AgentDFIR uses a per-machine key under +its home, `keys/seal.ed25519`, unless `--sign` names another or `--no-sign` +is given) and SHOULD record each sealed round's `SHA256SUMS` digest in an +anchor log outside the package (AgentDFIR: `anchors.jsonl` in its home, a +hash chain of its own) and print it. A signature proves a round was sealed +by the holder of the key; the anchor, and the printed digest kept somewhere +else, are what show a package was not rewritten and re-signed, or rolled +back to an earlier copy, by someone who controls the machine. + ## Compatibility Readers MUST accept both manifest forms: `manifest.jsonl` (0.2) and a diff --git a/internal/analysis/analysis.go b/internal/analysis/analysis.go index 70de0c7..d73ce66 100644 --- a/internal/analysis/analysis.go +++ b/internal/analysis/analysis.go @@ -8,11 +8,14 @@ package analysis import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "io" "os" "path/filepath" + "sort" "strings" "time" @@ -21,6 +24,7 @@ import ( "github.com/efij/AgentDFIR/v3/internal/correlate" "github.com/efij/AgentDFIR/v3/internal/detect" "github.com/efij/AgentDFIR/v3/internal/endpoint" + "github.com/efij/AgentDFIR/v3/internal/fingerprint" "github.com/efij/AgentDFIR/v3/internal/index" "github.com/efij/AgentDFIR/v3/internal/ioc" "github.com/efij/AgentDFIR/v3/internal/journal" @@ -101,37 +105,125 @@ func Stale(pkg string) bool { return staleReason(pkg) != "" } // staleReason says why results must be recomputed, or "" when they are // current. +// +// Results are current when they were computed by this binary's analysis +// code, on the overlay the package holds now, from the evidence the +// package holds now. None of that is the release number: a release that +// changes no parser and no rule used to throw away every case's results, +// and three releases in a day meant three full re-analyses of an unchanged +// machine. Results computed by different analysis code are still +// recomputed — on a real case the difference was once 66 HIGH findings. func staleReason(pkg string) string { - ev, err := overlay.Stat(filepath.Join(pkg, "normalized", "events.jsonl")) - if err != nil { - return "no normalized events" + st := normalize.Status(pkg) + if !st.Current { + return st.Reason } if !overlay.Exists(filepath.Join(pkg, "detections", "findings.json")) { return "no findings" } - if mt, ok := manifestModTime(pkg); ok && mt.After(ev.ModTime()) { - return "manifest newer than the normalized overlay" - } - // Results carry the version that produced them. A package analyzed by - // an older binary is that binary's opinion, not this one's: on a real - // case the difference was 66 HIGH findings that the running version - // would not have raised. - data, err := overlay.ReadFile(filepath.Join(pkg, "detections", "analysis.json")) + meta, err := readMeta(pkg) if err != nil { return "no analysis metadata" } - var meta struct { - Version string `json:"agentdfir_version"` + switch { + case meta.AnalysisFingerprint == "": + if meta.Version != "" { + return "analysis was produced by agentdfir " + meta.Version + } + return "analysis metadata carries no fingerprint" + case meta.AnalysisFingerprint != fingerprint.Analysis(): + return "analysis code changed since agentdfir " + meta.Version + " produced these results" + case meta.OverlayBuild != st.BuildID: + return "the overlay was rebuilt since the last analysis" } - if json.Unmarshal(data, &meta) != nil || meta.Version == "" { - return "analysis metadata carries no version" + man, err := casepkg.ReadManifest(pkg) + if err != nil { + return "unreadable manifest" } - if meta.Version != version.Version { - return "analysis was produced by agentdfir " + meta.Version + if meta.InputsDigest != analysisInputs(man) { + return "evidence changed since the last analysis" } return "" } +// meta is the part of analysis.json staleness reads. +type meta struct { + Events int `json:"events"` + Version string `json:"agentdfir_version"` + AnalysisFingerprint string `json:"analysis_fingerprint"` + OverlayBuild string `json:"overlay_build"` + InputsDigest string `json:"inputs_digest"` + OptionsDigest string `json:"options_digest"` +} + +func readMeta(pkg string) (*meta, error) { + data, err := overlay.ReadFile(filepath.Join(pkg, "detections", "analysis.json")) + if err != nil { + return nil, err + } + var m meta + if err := json.Unmarshal(data, &m); err != nil { + return nil, err + } + return &m, nil +} + +// analysisInputs identifies everything in the sealed zone analysis reads: +// every current record, including the ones no parser reads (the host +// witness, MCP configs read by the audit), by content address. +func analysisInputs(man *casepkg.Manifest) string { + var keys []string + for _, a := range man.Current() { + keys = append(keys, a.Status+"\x00"+a.LogicalPath+"\x00"+a.SourcePath+"\x00"+a.ArtifactID+"\x00"+a.Product) + } + sort.Strings(keys) + h := sha256.New() + for _, k := range keys { + h.Write([]byte(k)) + h.Write([]byte{'\n'}) + } + return hex.EncodeToString(h.Sum(nil)) +} + +// optionsDigest identifies the analyst-supplied inputs that change what a +// default analysis would find. A caller deciding whether earlier results +// can stand compares it along with staleness. +func (o Options) optionsDigest() string { + h := sha256.New() + fmt.Fprintf(h, "rules=%s\x00nopacks=%t\x00spawn=%d\x00", o.RulesDir, o.NoBuiltinPacks, o.SpawnThreshold) + for _, l := range [][]string{o.EndpointLogs, o.Honeytokens, o.KnownDests, o.IOCFiles, o.GatewayServers} { + fmt.Fprintf(h, "%q\x00", l) + } + fmt.Fprintf(h, "%s\x00%s\x00%s\x00%s\x00%s", o.ShellHistory, o.GatewayLog, o.GatewayMap, o.EndpointFormat, o.Window) + return hex.EncodeToString(h.Sum(nil)) +} + +// Current reports whether the package's stored results are the ones Run +// with these options would produce now, and why not when they are not. +// A caller that gets true can use them instead of re-running. +func Current(pkg string, o Options) (bool, string) { + if o.Renormalize { + return false, "re-parse requested" + } + if why := staleReason(pkg); why != "" { + return false, why + } + if o.SpawnThreshold <= 0 { + o.SpawnThreshold = 10 + } + o.Log, o.Stage = nil, nil + m, err := readMeta(pkg) + if err != nil || m.OptionsDigest != o.optionsDigest() { + return false, "analysis options differ from the last analysis" + } + if o.RetireExcluded { + if man, err := casepkg.ReadManifest(pkg); err == nil && man.RetireExcluded() > 0 { + return false, "artifacts to retire from the scan set" + } + } + return true, "" +} + // Ensure runs a default analysis only when results are missing or stale, // and says why, so the analyst knows the numbers they were about to read // were not the running version's. @@ -153,6 +245,7 @@ func Run(pkg string, o Options) (*Result, error) { } res := &Result{} dir := filepath.Join(pkg, "normalized") + var inputsDigest string detDir := filepath.Join(pkg, "detections") for _, d := range []string{dir, detDir} { if err := os.MkdirAll(d, 0o700); err != nil { @@ -171,19 +264,20 @@ func Run(pkg string, o Options) (*Result, error) { return nil, fmt.Errorf("retire excluded artifacts: %w", err) } res.StageNotes = append(res.StageNotes, fmt.Sprintf("%d artifacts under node_modules/.git objects retired from the scan set (evidence kept)", n)) - o.Renormalize = true + // No full re-parse: retiring changes the overlay's inputs, + // so the incremental rebuild drops exactly those segments. } } } + if man, err := casepkg.ReadManifest(pkg); err == nil { + inputsDigest = analysisInputs(man) + } o.stage(1, "normalize") // ---- 1. normalize (streaming) — only when the overlay is missing/stale. evPath := filepath.Join(dir, "events.jsonl") - needNorm := o.Renormalize - if fi, err := overlay.Stat(evPath); err != nil { - needNorm = true - } else if mt, ok := manifestModTime(pkg); ok && mt.After(fi.ModTime()) { - needNorm = true - } + ovStatus := normalize.Status(pkg) + needNorm := o.Renormalize || !ovStatus.Current + prevMeta, _ := readMeta(pkg) var entities []schema.Entity if needNorm { // The overlay is segmented per source artifact, so a new collection @@ -195,15 +289,12 @@ func Run(pkg string, o Options) (*Result, error) { // events.jsonl itself stays uncompressed: internal/index records a // byte offset per event so the explorer can open one without // holding all of them, and a gzip stream cannot be seeked. - sr, err := normalize.BuildOverlay(pkg, dir, normalize.OverlayOptions{Full: o.Renormalize}) + sr, err := normalize.Refresh(pkg, normalize.OverlayOptions{Full: o.Renormalize}) if err != nil { return nil, err } - if err := overlay.WriteJSONL(filepath.Join(dir, "entities.jsonl"), len(sr.Entities), func(i int) any { return sr.Entities[i] }); err != nil { - return nil, err - } - if err := overlay.WriteJSONL(filepath.Join(dir, "relationships.jsonl"), len(sr.Relationships), func(i int) any { return sr.Relationships[i] }); err != nil { - return nil, err + if sr.CacheRejected != "" { + res.StageNotes = append(res.StageNotes, "overlay cache rejected, rebuilt from the sealed evidence: "+sr.CacheRejected) } entities, res.Events, res.Renormalized = sr.Entities, sr.EventCount, true res.Reused, res.Reparsed = sr.Reused, sr.Reparsed @@ -225,6 +316,12 @@ func Run(pkg string, o Options) (*Result, error) { } entities = overlay.ReadJSONL[schema.Entity](filepath.Join(dir, "entities.jsonl")) res.Events = overlay.CountLines(evPath) + // Built since the last analysis (acquisition refreshes it before + // asking the host about the agent's claims): new evidence, so + // results computed on the previous build do not carry over. + if prevMeta == nil || prevMeta.OverlayBuild != ovStatus.BuildID { + res.Renormalized = true + } o.logf("Normalized: reusing overlay (%d events); corroboration states preserved", res.Events) // Not a StageNote: those are printed as "note:" on stderr and mean a // stage was skipped or degraded. Reclaiming disk is neither. @@ -236,16 +333,27 @@ func Run(pkg string, o Options) (*Result, error) { o.stage(2, "second witness") var findings []schema.Finding + // The events are decoded once and shared by every stage below. Each + // stage used to read the whole overlay again — seven decodes of every + // event per analysis, 323 MB each on a real machine. The stages that + // annotate events (host witness, endpoint and shell corroboration) + // change this one copy, and it is written back once, before the + // detections that stream the file read it. + var events []schema.Event + loaded, dirty := false, false + allEvents := func() []schema.Event { + if !loaded { + events, loaded = LoadEvents(pkg), true + } + return events + } // Host witness recorded during acquisition. This is the only source // that is always available: it needs no EDR, no auditd, no Sysmon, and // it is why a finding can now say CONFIRMED instead of only RECORDED. if wrec, wErr := witness.Load(pkg); wErr == nil { - events := LoadEvents(pkg) - wres, wf := witness.Apply(events, wrec) + wres, wf := witness.Apply(allEvents(), wrec) if wres.Checked > 0 { - if err := overlay.WriteJSONLPlain(evPath, len(events), func(i int) any { return events[i] }); err != nil { - return nil, err - } + dirty = true findings = append(findings, wf...) res.Witness = &wres o.logf("Host witness: %d claimed write(s) checked — %d CONFIRMED, %d DISPROVED, %d no longer present", @@ -254,7 +362,8 @@ func Run(pkg string, o Options) (*Result, error) { } // ---- 2. second witness (runs BEFORE detection so findings carry the states). if len(o.EndpointLogs) > 0 || o.ShellHistory != "" { - events := LoadEvents(pkg) + events := allEvents() + dirty = true if o.ShellHistory != "" { if cres, err := correlate.Apply(events, &correlate.ShellHistoryAdapter{Path: o.ShellHistory}); err == nil && cres.Corroborated > 0 { o.logf("Shell history: %d tool call(s) corroborated", cres.Corroborated) @@ -284,9 +393,14 @@ func Run(pkg string, o Options) (*Result, error) { cres.CloudCommands, cres.CloudCorroborated, cres.CloudRefused, cres.CloudBursts) } } + } + if dirty { if err := overlay.WriteJSONLPlain(evPath, len(events), func(i int) any { return events[i] }); err != nil { return nil, err } + if err := normalize.RecordEvents(pkg); err != nil { + return nil, err + } } // Earlier second-witness results stay part of the case as long as the @@ -357,7 +471,7 @@ func Run(pkg string, o Options) (*Result, error) { for _, d := range dropped { res.StageNotes = append(res.StageNotes, "duplicate rule id: "+d) } - extra, err := rulepack.Apply(packs, &schema.Normalized{Events: LoadEvents(pkg)}, pkg) + extra, err := rulepack.Apply(packs, &schema.Normalized{Events: allEvents()}, pkg) if err != nil { return nil, fmt.Errorf("rule packs: %w", err) } @@ -387,7 +501,7 @@ func Run(pkg string, o Options) (*Result, error) { if err != nil { return nil, fmt.Errorf("gateway log: %w", err) } - sum, gf := mcpaudit.CorrelateGateway(LoadEvents(pkg), recs, o.GatewayServers, 3) + sum, gf := mcpaudit.CorrelateGateway(allEvents(), recs, o.GatewayServers, 3) sum.Unparsed = unparsed gw = &sum mf = append(mf, gf...) @@ -424,7 +538,7 @@ func Run(pkg string, o Options) (*Result, error) { o.stage(6, "provenance") // ---- 6. instruction & memory provenance. - if prov, err := provenance.Run(pkg, LoadEvents(pkg), ""); err == nil { + if prov, err := provenance.Run(pkg, allEvents(), ""); err == nil { res.Provenance = len(prov.Files) findings = append(findings, prov.Findings...) _ = overlay.WriteJSON(filepath.Join(detDir, "provenance.json"), prov) @@ -443,7 +557,7 @@ func Run(pkg string, o Options) (*Result, error) { } chains = append(chains, extra...) } - cf := chain.Run(LoadEvents(pkg), findings, chains) + cf := chain.Run(allEvents(), findings, chains) res.Chains = len(cf) findings = append(findings, cf...) o.logf("Attack chains: %d chain(s) evaluated, %d matched", len(chains), len(cf)) @@ -452,7 +566,7 @@ func Run(pkg string, o Options) (*Result, error) { // // Confidence is computed last, over the finished set, so a verifier can // see the enrichment states the earlier stages produced. - findings = verify.Apply(findings, LoadEvents(pkg)) + findings = verify.Apply(findings, allEvents()) findings = dedupe(findings) sortBySeverity(findings) res.Findings = findings @@ -470,6 +584,12 @@ func Run(pkg string, o Options) (*Result, error) { // Which rule set decided this, by name, version and content hash — // so the question stays answerable after the binary is replaced. "rule_packs": packSrcs, "agentdfir_version": version.Version, + // What decides whether these results can be reused: the code that + // produced them, the overlay build and evidence they were computed + // on, and the analyst's options (see staleReason and Current). + "analysis_fingerprint": fingerprint.Analysis(), "parse_fingerprint": fingerprint.Parse(), + "overlay_build": normalize.Status(pkg).BuildID, "inputs_digest": inputsDigest, + "options_digest": o.optionsDigest(), }) // The explorer's offset index over the finished overlay, built here so // opening a case is instant instead of re-parsing hundreds of MB of @@ -493,6 +613,16 @@ func LoadEntities(pkg string) []schema.Entity { return overlay.ReadJSONL[schema.Entity](filepath.Join(pkg, "normalized", "entities.jsonl")) } +// PreviousEvents is how many events the last analysis of the package +// covered (0 when there was none). Progress uses it to size the analysis +// before it starts. +func PreviousEvents(pkg string) int { + if m, err := readMeta(pkg); err == nil { + return m.Events + } + return 0 +} + // LoadFindings reads the persisted findings. func LoadFindings(pkg string) []schema.Finding { var out []schema.Finding @@ -525,15 +655,3 @@ func sortBySeverity(f []schema.Finding) { } } } - -// manifestModTime returns when the package manifest last changed, in -// whichever form it is written. A later collection round appends to it, so -// this is what tells the overlay it is out of date. -func manifestModTime(pkg string) (time.Time, bool) { - for _, name := range []string{"manifest.jsonl", "manifest.json"} { - if fi, err := os.Stat(filepath.Join(pkg, name)); err == nil { - return fi.ModTime(), true - } - } - return time.Time{}, false -} diff --git a/internal/analysis/delta_test.go b/internal/analysis/delta_test.go new file mode 100644 index 0000000..11ac1e7 --- /dev/null +++ b/internal/analysis/delta_test.go @@ -0,0 +1,306 @@ +package analysis + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/efij/AgentDFIR/v3/internal/casepkg" + "github.com/efij/AgentDFIR/v3/internal/detect" + "github.com/efij/AgentDFIR/v3/internal/fingerprint" + "github.com/efij/AgentDFIR/v3/internal/normalize" + "github.com/efij/AgentDFIR/v3/internal/overlay" + "github.com/efij/AgentDFIR/v3/internal/rulepack" + "github.com/efij/AgentDFIR/v3/internal/store" +) + +// analyzedCase is a two-transcript-product case analyzed once. +func analyzedCase(t *testing.T) (pkg, root string, first *Result) { + t.Helper() + root = t.TempDir() + writeProfile(t, root, 1) + pkg = filepath.Join(t.TempDir(), "d.adfir") + b, err := casepkg.New(pkg, "DELTA", casepkg.CaseInfo{OperatorOSUser: "t"}) + if err != nil { + t.Fatal(err) + } + collectRound(t, b, root) + first, err = Run(pkg, Options{}) + if err != nil { + t.Fatal(err) + } + return pkg, root, first +} + +// A round in which nothing changed leaves the stored results current: no +// rebuild, no re-analysis. And a new release of the binary that did not +// change any parsing or analysis code does not invalidate them either — +// the release number is not part of the key any more. +func TestNothingChangedMeansNothingRecomputed(t *testing.T) { + pkg, root, _ := analyzedCase(t) + b, err := casepkg.Reopen(pkg, casepkg.CaseInfo{OperatorOSUser: "t"}) + if err != nil { + t.Fatal(err) + } + collectRound(t, b, root) // same files: every one carried forward + if ok, why := Current(pkg, Options{}); !ok { + t.Fatalf("unchanged round made results stale: %s", why) + } + // Rewrite analysis.json's version as if an older release (with the + // same code) had produced it. + p := filepath.Join(pkg, "detections", "analysis.json") + var m map[string]any + data, _ := overlay.ReadFile(p) + if err := json.Unmarshal(data, &m); err != nil { + t.Fatal(err) + } + m["agentdfir_version"] = "0.0.1" + if err := overlay.WriteJSON(p, m); err != nil { + t.Fatal(err) + } + if ok, why := Current(pkg, Options{}); !ok { + t.Fatalf("a release number alone invalidated results: %s", why) + } + // Different analysis code must invalidate them. + m["analysis_fingerprint"] = "0000" + if err := overlay.WriteJSON(p, m); err != nil { + t.Fatal(err) + } + if ok, _ := Current(pkg, Options{}); ok { + t.Fatal("results from different analysis code were treated as current") + } + if m["parse_fingerprint"] != fingerprint.Parse() { + t.Fatal("analysis.json does not record the parse fingerprint") + } +} + +// Options that change what analysis finds make earlier results unusable. +func TestDifferentOptionsAreNotCurrent(t *testing.T) { + pkg, _, _ := analyzedCase(t) + if ok, why := Current(pkg, Options{}); !ok { + t.Fatalf("fresh analysis not current: %s", why) + } + if ok, _ := Current(pkg, Options{Honeytokens: []string{"canary-123"}}); ok { + t.Fatal("results computed without a honeytoken reused for a run with one") + } +} + +// Someone deletes an event from the overlay. The overlay no longer +// hashes to what AgentDFIR last wrote, so it is rebuilt from the sealed +// evidence and the event is back. +func TestEditedEventsAreRebuiltFromEvidence(t *testing.T) { + pkg, _, first := analyzedCase(t) + evPath := filepath.Join(pkg, "normalized", "events.jsonl") + data, err := os.ReadFile(evPath) + if err != nil { + t.Fatal(err) + } + lines := strings.SplitAfter(string(data), "\n") + if err := os.WriteFile(evPath, []byte(strings.Join(lines[1:], "")), 0o600); err != nil { + t.Fatal(err) + } + if st := normalize.Status(pkg); st.Current { + t.Fatal("an overlay with an event removed was reported current") + } + again, err := Run(pkg, Options{}) + if err != nil { + t.Fatal(err) + } + if !again.Renormalized || again.Events != first.Events { + t.Fatalf("rebuilt=%t events=%d, want a rebuild back to %d", again.Renormalized, again.Events, first.Events) + } +} + +// Someone edits a cached segment. The next round that rebuilds the +// overlay replays it, the hash does not match, and the whole overlay is +// rebuilt from the sealed evidence — identical to a full re-parse. +func TestEditedSegmentIsRejected(t *testing.T) { + pkg, root, _ := analyzedCase(t) + segs, _ := filepath.Glob(filepath.Join(pkg, "normalized", "events", "*", "*.jsonl.gz")) + if len(segs) == 0 { + t.Fatal("no cached segments") + } + for _, sp := range segs { + rewriteGz(t, sp, func(b []byte) []byte { return bytes.Replace(b, []byte("curl"), []byte("true"), 1) }) + } + writeProfile(t, root, 2) // new evidence, so the overlay is rebuilt + b, err := casepkg.Reopen(pkg, casepkg.CaseInfo{OperatorOSUser: "t"}) + if err != nil { + t.Fatal(err) + } + collectRound(t, b, root) + inc, err := Run(pkg, Options{}) + if err != nil { + t.Fatal(err) + } + found := false + for _, n := range inc.StageNotes { + if strings.Contains(n, "cache rejected") { + found = true + } + } + if !found { + t.Fatalf("edited segments were not reported: %v", inc.StageNotes) + } + clone := copyPkg(t, pkg) + if _, err := Run(clone, Options{Renormalize: true}); err != nil { + t.Fatal(err) + } + if readFile(t, filepath.Join(pkg, "normalized", "events.jsonl")) != readFile(t, filepath.Join(clone, "normalized", "events.jsonl")) { + t.Fatal("overlay after rejecting the edited cache differs from a full parse") + } +} + +// With a machine key, the overlay state is authenticated: an edited +// state.json is not trusted. +func TestStateIsAuthenticatedWithTheMachineKey(t *testing.T) { + home := filepath.Join(t.TempDir(), "home") + if err := os.Mkdir(home, 0o700); err != nil { + t.Fatal(err) + } + t.Setenv(store.EnvHome, home) + if _, err := store.MachineKey(); err != nil { + t.Fatal(err) + } + pkg, _, _ := analyzedCase(t) + if _, err := os.Stat(filepath.Join(pkg, "normalized", "state.mac")); err != nil { + t.Fatalf("no MAC written with a machine key present: %v", err) + } + if st := normalize.Status(pkg); !st.Current { + t.Fatalf("authenticated overlay not current: %s", st.Reason) + } + sp := filepath.Join(pkg, "normalized", "state.json") + data, _ := os.ReadFile(sp) + if err := os.WriteFile(sp, bytes.Replace(data, []byte(`"events": `), []byte(`"events": `), 1), 0o600); err != nil { + t.Fatal(err) + } + if st := normalize.Status(pkg); st.Current { + t.Fatal("state.json edited without the key was trusted") + } +} + +func rewriteGz(t *testing.T, path string, edit func([]byte) []byte) { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + zr, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + plain, err := io.ReadAll(zr) + f.Close() + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + zw.Write(edit(plain)) + zw.Close() + _ = os.Chmod(path, 0o600) + if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil { + t.Fatal(err) + } +} + +// Retiring artifacts from the scan set rebuilds the overlay incrementally, +// and the result is what a full parse of the reduced set produces. +func TestRetirementIsIncrementalAndEquivalent(t *testing.T) { + root := t.TempDir() + writeProfile(t, root, 1) + nm := filepath.Join(root, ".claude", "projects", "p", "node_modules") + if err := os.MkdirAll(nm, 0o755); err != nil { + t.Fatal(err) + } + writeTranscript(t, filepath.Join(nm, "vendored.jsonl"), claudeLines("sv", 0, 3)) + pkg := filepath.Join(t.TempDir(), "r.adfir") + b, err := casepkg.New(pkg, "RET", casepkg.CaseInfo{OperatorOSUser: "t"}) + if err != nil { + t.Fatal(err) + } + // An older collector took the node_modules transcript. + src := filepath.Join(nm, "vendored.jsonl") + if err := b.IngestFile(src, casepkg.ArtifactRecord{SourcePath: src, LogicalPath: ".claude/projects/p/node_modules/vendored.jsonl", + Product: "claude-code", ArtifactType: "transcript", CollectorRule: "claude.sessions"}); err != nil { + t.Fatal(err) + } + collectRound(t, b, root) + before, err := Run(pkg, Options{}) + if err != nil { + t.Fatal(err) + } + inc, err := Run(pkg, Options{RetireExcluded: true}) + if err != nil { + t.Fatal(err) + } + if inc.Events >= before.Events { + t.Fatalf("events %d → %d: the retired transcript was never in the overlay; the test proves nothing", before.Events, inc.Events) + } + if inc.Reparsed != 0 { + t.Fatalf("retiring re-parsed %d artifact(s); the rest are unchanged", inc.Reparsed) + } + clone := copyPkg(t, pkg) + if _, err := Run(clone, Options{Renormalize: true}); err != nil { + t.Fatal(err) + } + for _, name := range []string{"events.jsonl", "entities.jsonl", "relationships.jsonl"} { + if readFile(t, filepath.Join(pkg, "normalized", name)) != readFile(t, filepath.Join(clone, "normalized", name)) { + t.Fatalf("%s after incremental retirement differs from a full parse", name) + } + } + if ok, why := Current(pkg, Options{RetireExcluded: true}); !ok { + t.Fatalf("after retiring once, results not current: %s", why) + } +} + +// With a machine key, the content scans remember each artifact's result: +// a second analysis reads no unchanged artifact again, and what it finds +// is exactly what a scan of everything finds. +func TestContentScansAreRememberedPerArtifact(t *testing.T) { + home := filepath.Join(t.TempDir(), "home") + if err := os.Mkdir(home, 0o700); err != nil { + t.Fatal(err) + } + t.Setenv(store.EnvHome, home) + if _, err := store.MachineKey(); err != nil { + t.Fatal(err) + } + pkg, root, _ := analyzedCase(t) + writeProfile(t, root, 2) // some transcripts grow, one is new + b, err := casepkg.Reopen(pkg, casepkg.CaseInfo{OperatorOSUser: "t"}) + if err != nil { + t.Fatal(err) + } + collectRound(t, b, root) + clone := copyPkg(t, pkg) + inc, err := Run(pkg, Options{}) + if err != nil { + t.Fatal(err) + } + if detect.ContentMemoHits == 0 || rulepack.ArtifactMemoHits == 0 { + t.Fatalf("no artifact served from the memo (content %d, rule packs %d)", detect.ContentMemoHits, rulepack.ArtifactMemoHits) + } + // The same round analyzed with no memo at all. + if err := os.RemoveAll(filepath.Join(clone, "detections", "memo")); err != nil { + t.Fatal(err) + } + full, err := Run(clone, Options{Renormalize: true}) + if err != nil { + t.Fatal(err) + } + got, want := canonicalFindings(t, inc.Findings), canonicalFindings(t, full.Findings) + if len(got) != len(want) { + t.Fatalf("findings with memo %d, without %d", len(got), len(want)) + } + for i := range got { + if got[i] != want[i] { + t.Fatalf("finding %d differs:\n memo: %s\n no memo: %s", i, got[i], want[i]) + } + } +} diff --git a/internal/analysis/incremental_equiv_test.go b/internal/analysis/incremental_equiv_test.go index eee81d1..d0ccf9e 100644 --- a/internal/analysis/incremental_equiv_test.go +++ b/internal/analysis/incremental_equiv_test.go @@ -13,6 +13,7 @@ import ( "github.com/efij/AgentDFIR/v3/internal/overlay" "github.com/efij/AgentDFIR/v3/internal/products" "github.com/efij/AgentDFIR/v3/internal/schema" + "github.com/efij/AgentDFIR/v3/internal/store" ) // geminiLines writes a gemini-cli session, which the generic chat parser @@ -85,7 +86,23 @@ func writeTranscript(t testing.TB, path, body string) { // Divergence here would not look like a crash. It would look like an agent // lineage that is subtly wrong in a case file an analyst is about to // testify from, which is why this compares bytes and not counts. -func TestIncrementalEquivalenceAcrossRounds(t *testing.T) { +func TestIncrementalEquivalenceAcrossRounds(t *testing.T) { equivalenceAcrossRounds(t) } + +// The same rounds with a machine key, so the authenticated overlay state +// and the per-artifact scan memo are in play on every round. +func TestIncrementalEquivalenceAcrossRoundsWithMemo(t *testing.T) { + home := filepath.Join(t.TempDir(), "home") + if err := os.Mkdir(home, 0o700); err != nil { + t.Fatal(err) + } + t.Setenv(store.EnvHome, home) + if _, err := store.MachineKey(); err != nil { + t.Fatal(err) + } + equivalenceAcrossRounds(t) +} + +func equivalenceAcrossRounds(t *testing.T) { root := t.TempDir() cdir := filepath.Join(root, ".claude", "projects", "p") xdir := filepath.Join(root, ".codex", "sessions") @@ -132,7 +149,12 @@ func TestIncrementalEquivalenceAcrossRounds(t *testing.T) { t.Fatal(err) } - reusedEver, renumberedEver := 0, false + reusedEver, renumberedEver, unchangedSeen := 0, false, false + man0, err := casepkg.ReadManifest(pkg) + if err != nil { + t.Fatal(err) + } + prevDigest := normalize.InputsDigest(man0) for i, m := range rounds { m.apply(t, root) nb, err := casepkg.Reopen(pkg, casepkg.CaseInfo{OperatorOSUser: "t"}) @@ -142,6 +164,10 @@ func TestIncrementalEquivalenceAcrossRounds(t *testing.T) { collectAll(t, nb, root) clone := copyPkg(t, pkg) + // The full side reads everything: no remembered scan results. + if err := os.RemoveAll(filepath.Join(clone, "detections", "memo")); err != nil { + t.Fatal(err) + } inc, err := Run(pkg, Options{}) if err != nil { t.Fatalf("round %d (%s) incremental: %v", i+1, m.name, err) @@ -150,9 +176,23 @@ func TestIncrementalEquivalenceAcrossRounds(t *testing.T) { if err != nil { t.Fatalf("round %d (%s) full: %v", i+1, m.name, err) } - if !inc.Renormalized { - t.Fatalf("round %d (%s): a new round must rebuild the overlay", i+1, m.name) + // A round that changed the case's current evidence rebuilds the + // overlay; one that did not — every file carried forward, or a + // source gone from the host while its earlier record stays current + // — must not. Rebuilding those was a full re-analysis of an + // unchanged machine on every run. + man, err := casepkg.ReadManifest(pkg) + if err != nil { + t.Fatal(err) } + digest := normalize.InputsDigest(man) + if want := digest != prevDigest; inc.Renormalized != want { + t.Fatalf("round %d (%s): overlay rebuilt=%t, want %t", i+1, m.name, inc.Renormalized, want) + } + if digest == prevDigest { + unchangedSeen = true + } + prevDigest = digest reusedEver += inc.Reused if inc.Reparsed > 0 && inc.Reused > 0 { // Something was parsed ahead of something replayed, so at least @@ -188,6 +228,9 @@ func TestIncrementalEquivalenceAcrossRounds(t *testing.T) { } } } + if !unchangedSeen { + t.Fatal("no round left the evidence unchanged; the skip path never ran") + } if reusedEver == 0 { t.Fatal("no segment was ever replayed; the test compared the full path against itself") } diff --git a/internal/artmemo/artmemo.go b/internal/artmemo/artmemo.go new file mode 100644 index 0000000..cc8f4fc --- /dev/null +++ b/internal/artmemo/artmemo.go @@ -0,0 +1,199 @@ +// Package artmemo remembers what a per-artifact scan found, so a scan that +// depends only on one artifact's bytes runs once per artifact, not once +// per collection round. +// +// The content scans — rule packs over raw transcripts and configs, the +// credential, injection and invisible-character scans — read every +// artifact in full. On a real case that was most of the analysis: minutes +// of regular expressions over 2.4 GB of transcripts, repeated on every run +// although almost none of them had changed. Their result is a function of +// the artifact (its content address and the record fields the scan reads) +// and of the scanning code and rules. That is the memo key; nothing else +// can reach the result. +// +// The memo is derived data in the analysis overlay, and it is trusted only +// when it is authenticated: it is written with an HMAC under the machine's +// cache key and read back only if the MAC verifies and it was produced by +// the same scanning code with the same rules and options. Without a key it +// is neither read nor written, and every scan runs. A tampered memo is a +// re-scan, never a hidden finding. +package artmemo + +import ( + "bytes" + "compress/gzip" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "os" + "path/filepath" + "sync" + + "github.com/efij/AgentDFIR/v3/internal/casepkg" + "github.com/efij/AgentDFIR/v3/internal/fingerprint" + "github.com/efij/AgentDFIR/v3/internal/store" +) + +// Memo is one scan's remembered results, by artifact. +type Memo[T any] struct { + path string + key []byte + scope string + + mu sync.Mutex + entries map[string]T + used map[string]bool + hits int +} + +type file[T any] struct { + Scope string `json:"scope"` + Entries map[string]T `json:"entries"` +} + +// Open loads the memo named name for the package. scope names everything +// besides the artifact the scan depends on (rules, options); the analysis +// code fingerprint is always part of it. A memo that is missing, from other +// code or rules, or fails authentication starts empty. +func Open[T any](pkgDir, name, scope string) *Memo[T] { + m := &Memo[T]{ + path: filepath.Join(pkgDir, "detections", "memo", name+".json.gz"), + key: store.CacheKey(), + scope: fingerprint.Analysis() + "\x00" + scope, + entries: map[string]T{}, + used: map[string]bool{}, + } + if m.key == nil { + return m + } + data, err := os.ReadFile(m.path) + if err != nil { + return m + } + mac, err := os.ReadFile(m.path + ".mac") + if err != nil || !hmac.Equal(bytes.TrimSpace(mac), []byte(sum(m.key, data))) { + return m + } + zr, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return m + } + plain, err := io.ReadAll(io.LimitReader(zr, 512<<20)) + if err != nil { + return m + } + var f file[T] + if json.Unmarshal(plain, &f) != nil || f.Scope != m.scope || f.Entries == nil { + return m + } + m.entries = f.Entries + return m +} + +// Enabled reports whether results can be remembered at all (this machine +// has a cache key). +func (m *Memo[T]) Enabled() bool { return m.key != nil } + +// Key names an artifact by everything a scan reads off its record. +func Key(a casepkg.ArtifactRecord) string { + h := sha256.New() + for _, f := range []string{a.ArtifactID, a.LogicalPath, a.SourcePath, a.ArtifactType, a.Product, a.Status, a.Codec} { + h.Write([]byte(f)) + h.Write([]byte{0}) + } + return hex.EncodeToString(h.Sum(nil)) +} + +// Get returns a remembered result. Safe for concurrent use. +func (m *Memo[T]) Get(a casepkg.ArtifactRecord) (T, bool) { + k := Key(a) + m.mu.Lock() + defer m.mu.Unlock() + v, ok := m.entries[k] + if ok { + m.used[k] = true + m.hits++ + } + return v, ok +} + +// Put remembers a result. Safe for concurrent use. +func (m *Memo[T]) Put(a casepkg.ArtifactRecord, v T) { + if m.key == nil { + return + } + k := Key(a) + m.mu.Lock() + m.entries[k] = v + m.used[k] = true + m.mu.Unlock() +} + +// Hits is how many results were served from the memo. +func (m *Memo[T]) Hits() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.hits +} + +// Save writes the memo, keeping only the artifacts this run touched, so it +// never grows past the case's current evidence. It does nothing without a +// key. +func (m *Memo[T]) Save() error { + if m.key == nil { + return nil + } + m.mu.Lock() + keep := make(map[string]T, len(m.used)) + for k := range m.used { + keep[k] = m.entries[k] + } + m.mu.Unlock() + plain, err := json.Marshal(file[T]{Scope: m.scope, Entries: keep}) + if err != nil { + return err + } + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + if _, err := zw.Write(plain); err != nil { + return err + } + if err := zw.Close(); err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(m.path), 0o700); err != nil { + return err + } + data := buf.Bytes() + // MAC first: a crash between the two writes leaves a memo that fails + // authentication and is re-scanned, never one trusted without a MAC. + if err := writeAtomic(m.path+".mac", []byte(sum(m.key, data)+"\n")); err != nil { + return err + } + return writeAtomic(m.path, data) +} + +func sum(key, data []byte) string { + h := hmac.New(sha256.New, key) + h.Write(data) + return hex.EncodeToString(h.Sum(nil)) +} + +func writeAtomic(path string, data []byte) error { + tmp, err := os.CreateTemp(filepath.Dir(path), ".memo-*.tmp") + if err != nil { + return err + } + if _, err := tmp.Write(data); err != nil { + tmp.Close() + os.Remove(tmp.Name()) + return err + } + if err := tmp.Close(); err != nil { + os.Remove(tmp.Name()) + return err + } + return os.Rename(tmp.Name(), path) +} diff --git a/internal/artmemo/artmemo_test.go b/internal/artmemo/artmemo_test.go new file mode 100644 index 0000000..32221cd --- /dev/null +++ b/internal/artmemo/artmemo_test.go @@ -0,0 +1,87 @@ +package artmemo + +import ( + "os" + "path/filepath" + "testing" + + "github.com/efij/AgentDFIR/v3/internal/casepkg" + "github.com/efij/AgentDFIR/v3/internal/store" +) + +func withKey(t *testing.T) { + t.Helper() + home := filepath.Join(t.TempDir(), "home") + if err := os.Mkdir(home, 0o700); err != nil { + t.Fatal(err) + } + t.Setenv(store.EnvHome, home) + if _, err := store.MachineKey(); err != nil { + t.Fatal(err) + } +} + +var art = casepkg.ArtifactRecord{ArtifactID: "abc", LogicalPath: "t.jsonl", ArtifactType: "agent_session", Status: casepkg.StatusOK} + +func TestRoundTripAndScope(t *testing.T) { + withKey(t) + pkg := t.TempDir() + m := Open[[]string](pkg, "x", "rules-v1") + m.Put(art, []string{"finding"}) + if err := m.Save(); err != nil { + t.Fatal(err) + } + again := Open[[]string](pkg, "x", "rules-v1") + if v, ok := again.Get(art); !ok || len(v) != 1 || v[0] != "finding" { + t.Fatalf("remembered result not served: %v %v", v, ok) + } + // Other rules: nothing carries over. + if _, ok := Open[[]string](pkg, "x", "rules-v2").Get(art); ok { + t.Fatal("result served under a different rule set") + } + // A different artifact record (same bytes, other path): not the same key. + other := art + other.LogicalPath = "u.jsonl" + if _, ok := again.Get(other); ok { + t.Fatal("result served for a different record") + } +} + +// An edited memo fails its MAC and is ignored: a re-scan, never a hidden +// finding. +func TestTamperedMemoIsIgnored(t *testing.T) { + withKey(t) + pkg := t.TempDir() + m := Open[[]string](pkg, "x", "s") + m.Put(art, []string{"finding"}) + if err := m.Save(); err != nil { + t.Fatal(err) + } + p := filepath.Join(pkg, "detections", "memo", "x.json.gz") + data, _ := os.ReadFile(p) + data[len(data)-1] ^= 0xff + if err := os.WriteFile(p, data, 0o600); err != nil { + t.Fatal(err) + } + if _, ok := Open[[]string](pkg, "x", "s").Get(art); ok { + t.Fatal("a memo that fails its MAC was served") + } +} + +// Without a machine key nothing is remembered or served. +func TestNoKeyNoMemo(t *testing.T) { + home := filepath.Join(t.TempDir(), "home") + t.Setenv(store.EnvHome, home) + pkg := t.TempDir() + m := Open[[]string](pkg, "x", "s") + if m.Enabled() { + t.Fatal("memo enabled without a key") + } + m.Put(art, []string{"f"}) + if err := m.Save(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(pkg, "detections", "memo")); err == nil { + t.Fatal("memo written without a key") + } +} diff --git a/internal/casepkg/casepkg.go b/internal/casepkg/casepkg.go index 03827b2..b1b4d7c 100644 --- a/internal/casepkg/casepkg.go +++ b/internal/casepkg/casepkg.go @@ -165,6 +165,20 @@ type Round struct { ArtifactsCarried int `json:"artifacts_carried_forward"` ArtifactsFailed int `json:"artifacts_not_acquired"` StoredBytes int64 `json:"stored_bytes"` + // PrevSealSHA256 is the sha256 of the SHA256SUMS this round replaced — + // the seal of the round before. The archived seal is covered by this + // round's SHA256SUMS as well; this names the link for a reader. + PrevSealSHA256 string `json:"prev_seal_sha256,omitempty"` + // Signer is the public key (hex) the round was sealed for, when it is + // signed. The signature itself is SEAL.sig, archived by the next round. + Signer string `json:"signer,omitempty"` + // PriorIntegrity is what checking the earlier rounds found before this + // one was added: "verified", "unsigned" (nothing to check a signature + // against), or "FAILED" with the problems listed. A round added to a + // case that failed is sealed like any other — the host's evidence is + // still worth taking — but it says so, permanently. + PriorIntegrity string `json:"prior_integrity,omitempty"` + PriorProblems []string `json:"prior_integrity_problems,omitempty"` } // CaseInfo is case.json. @@ -199,16 +213,27 @@ type Sharer interface { // Builder accumulates evidence into a package directory and seals it. type Builder struct { - Dir string - Shared Sharer // optional cross-case blob sharing - NoCodec bool // store plaintext (used by tests and --no-compress) - manifest Manifest - caseInfo CaseInfo - coll *hashchain.Writer // collection.jsonl - custody *hashchain.Writer // chain-of-custody.jsonl - mf *os.File // manifest.jsonl, append mode - lock *lockHandle - sealed bool + Dir string + Shared Sharer // optional cross-case blob sharing + NoCodec bool // store plaintext (used by tests and --no-compress) + // Signer, PriorIntegrity and PriorProblems are recorded in this round's + // entry in case.json when it is sealed (see Round). + Signer string + PriorIntegrity string + PriorProblems []string + manifest Manifest + caseInfo CaseInfo + coll *hashchain.Writer // collection.jsonl + custody *hashchain.Writer // chain-of-custody.jsonl + mf *os.File // manifest.jsonl, append mode + lock *lockHandle + sealed bool + // pending: this round of an existing package recorded the sealed state + // it started from and is rolled back to it unless sealed. created: + // this builder made the package directory, and removes it unless the + // first round is sealed. + pending bool + created bool round int started time.Time @@ -253,7 +278,8 @@ func New(dir, caseID string, info CaseInfo) (*Builder, error) { info.UTCOffsetSeconds = offset b := &Builder{ - Dir: dir, + Dir: dir, + created: true, manifest: Manifest{ ADFIRVersion: version.ADFIRVersion, CaseID: caseID, @@ -309,18 +335,40 @@ func New(dir, caseID string, info CaseInfo) (*Builder, error) { // appended: resuming a broken chain would hide the break behind a // valid-looking tail. func Reopen(dir string, info CaseInfo) (*Builder, error) { + return ReopenChecked(dir, info, nil) +} + +// ReopenChecked is Reopen with a check that runs once the package is +// locked and before anything is written to it. That is the only point at +// which the earlier rounds can be proven as they were sealed: before the +// lock another run could still be adding a round, and after Reopen the +// custody chain already carries this round's first record. +func ReopenChecked(dir string, info CaseInfo, check func()) (*Builder, error) { + lk, err := acquireLock(dir) + if err != nil { + return nil, err + } + // A round a previous process left unsealed is rolled back first, so + // the check below sees the package exactly as it was last sealed. + ab, err := rollbackPending(dir, "crashed") + if err != nil { + lk.release() + return nil, err + } + noteAborted(dir, ab) + if check != nil { + check() + } man, err := ReadManifest(dir) if err != nil { + lk.release() return nil, fmt.Errorf("reopen %s: %w", dir, err) } prevCase, err := ReadCaseInfo(dir) if err != nil { + lk.release() return nil, fmt.Errorf("reopen %s: %w", dir, err) } - lk, err := acquireLock(dir) - if err != nil { - return nil, err - } // Carry the operator's new assertions into the existing case record; // identity, creation time and host stay as first recorded. @@ -390,6 +438,10 @@ func Reopen(dir string, info CaseInfo) (*Builder, error) { } } + if err := beginPending(dir, round); err != nil { + return fail(fmt.Errorf("record the sealed state: %w", err)) + } + b.pending = true if b.coll, err = hashchain.NewAppender(filepath.Join(dir, "collection.jsonl")); err != nil { return fail(fmt.Errorf("collection log: %w", err)) } @@ -402,6 +454,16 @@ func Reopen(dir string, info CaseInfo) (*Builder, error) { if err := b.startRound(); err != nil { return fail(err) } + // Rounds rolled back since the last seal are part of the record. + for _, a := range takeAborted(dir) { + if err := b.custody.Append(map[string]any{ + "event": "round_aborted", "case_id": b.manifest.CaseID, "round": a.Round, + "started_utc": a.StartedUTC, "how": a.How, + "discarded_blobs": a.DiscardedBlobs, "discarded_bytes": a.DiscardedBytes, + }); err != nil { + return fail(err) + } + } return b, nil } @@ -446,6 +508,9 @@ func countOK(m *Manifest) int { // Round returns the round this builder is writing. func (b *Builder) Round() int { return b.round } +// CaseID is the case this builder is writing. +func (b *Builder) CaseID() string { return b.manifest.CaseID } + // startRound records the opening custody event for this round. func (b *Builder) startRound() error { return b.custody.Append(map[string]any{ @@ -554,6 +619,10 @@ func (b *Builder) carryRecord(prev ArtifactRecord, rec ArtifactRecord) ArtifactR rec.StoredSHA = prev.StoredSHA rec.StoredSize = prev.StoredSize rec.Chunks = prev.Chunks + // The identity the file was judged unchanged on travels with the + // record, or the next round has nothing to compare and re-reads it. + rec.Inode = prev.Inode + rec.CTimeUTC = prev.CTimeUTC rec.Status = StatusOK rec.Method = MethodCarriedForward rec.CollectedUTC = time.Now().UTC().Format(time.RFC3339Nano) @@ -1030,6 +1099,11 @@ func (b *Builder) Seal() error { } b.coll, b.custody, b.mf = nil, nil, nil + prevSeal := "" + if data, err := os.ReadFile(filepath.Join(b.Dir, sumsFile)); err == nil { + sum := sha256.Sum256(data) + prevSeal = hex.EncodeToString(sum[:]) + } b.caseInfo.Rounds = append(b.caseInfo.Rounds, Round{ Round: b.round, StartedUTC: b.started.UTC().Format(time.RFC3339), @@ -1040,6 +1114,10 @@ func (b *Builder) Seal() error { ArtifactsCarried: b.stats.Carried, ArtifactsFailed: b.stats.Failed, StoredBytes: b.stats.StoredBytes, + PrevSealSHA256: prevSeal, + Signer: b.Signer, + PriorIntegrity: b.PriorIntegrity, + PriorProblems: b.PriorProblems, }) if err := writeJSON(filepath.Join(b.Dir, "case.json"), b.caseInfo); err != nil { return err @@ -1050,6 +1128,7 @@ func (b *Builder) Seal() error { if err := b.writeSums(); err != nil { return err } + endPending(b.Dir) b.sealed = true b.lock.release() return nil @@ -1081,6 +1160,25 @@ func (b *Builder) Close() { _ = b.mf.Close() b.mf = nil } + // Nothing unsealed survives a builder that is abandoned: an added + // round is rolled back to the last seal, and a package whose first + // round never sealed is removed. + switch { + case b.pending: + b.pending = false + if ab, err := rollbackPending(b.Dir, "abandoned"); err == nil { + noteAborted(b.Dir, ab) + } + case b.created: + b.created = false + _ = filepath.WalkDir(b.Dir, func(p string, _ os.DirEntry, _ error) error { _ = os.Chmod(p, 0o700); return nil }) + if b.lock != nil { + b.lock.release() + b.lock = nil + } + _ = os.RemoveAll(b.Dir) + return + } if b.lock != nil { b.lock.release() } @@ -1101,6 +1199,18 @@ func (b *Builder) archivePreviousSeal() error { if err := os.MkdirAll(filepath.Join(b.Dir, sealsDir), 0o700); err != nil { return err } + // The signature over that seal goes with it. Before this, SEAL.sig was + // overwritten by the next round's and every earlier signature was lost. + if sig, err := os.ReadFile(filepath.Join(b.Dir, sigFileName)); err == nil { + sdst := filepath.Join(b.Dir, sealsDir, fmt.Sprintf("%s.%d", sigFileName, b.round-1)) + if _, err := os.Stat(sdst); errors.Is(err, os.ErrNotExist) { + if err := os.WriteFile(sdst, sig, 0o400); err != nil { + return err + } + } + } else if !errors.Is(err, os.ErrNotExist) { + return err + } dst := filepath.Join(b.Dir, sealsDir, fmt.Sprintf("%s.%d", sumsFile, b.round-1)) if _, err := os.Stat(dst); err == nil { return nil // already archived @@ -1108,6 +1218,11 @@ func (b *Builder) archivePreviousSeal() error { return os.WriteFile(dst, data, 0o400) } +// sigFileName is the package signature (internal/seal writes it). It sits +// outside the sealed zone because it signs SHA256SUMS; each round archives +// the previous one into seals/, where the next seal covers it. +const sigFileName = "SEAL.sig" + // sealedFiles lists the non-raw files covered by SHA256SUMS. manifest.json // is listed for packages that still carry the legacy array form. var sealedFiles = []string{"case.json", manifestJSONL, manifestJSON, "collection.jsonl", "chain-of-custody.jsonl"} @@ -1156,8 +1271,9 @@ func (b *Builder) writeSums() error { lines = append(lines, h+" raw/"+e.Name()) } sort.Strings(lines) - return os.WriteFile(filepath.Join(b.Dir, sumsFile), - []byte(strings.Join(lines, "\n")+"\n"), 0o600) + // Atomically: the new SHA256SUMS is the moment the round becomes + // sealed, and a half-written one must never exist. + return writeAtomic(filepath.Join(b.Dir, sumsFile), []byte(strings.Join(lines, "\n")+"\n"), 0o600) } // VerifyResult reports the outcome of package verification. diff --git a/internal/casepkg/pending.go b/internal/casepkg/pending.go new file mode 100644 index 0000000..af4c47e --- /dev/null +++ b/internal/casepkg/pending.go @@ -0,0 +1,237 @@ +package casepkg + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// Rounds are transactional. +// +// A round appends to the manifest and both hash chains as it goes and +// writes blobs into raw/, and only Seal makes any of it part of the sealed +// package. A round that never reaches Seal — an error, a refused disk +// floor, Ctrl+C, a crash — used to leave that unsealed tail behind: the +// next round then extended chains whose last records no seal covered, and +// the case no longer verified. Every later round inherited the failure. +// +// So before a round writes anything, the sealed state it starts from is +// recorded in .round-pending.json: the lengths of the three append-only +// files and a copy of case.json. A round that is abandoned is rolled back +// to exactly that state — by Close, or by the next round when the process +// died — and the next round's custody chain records what was discarded: +// nothing is lost silently, and nothing unsealed is ever presented as part +// of a seal. The file is removed once the round is sealed. +// +// A round counts as sealed as soon as a new SHA256SUMS is in place (Seal +// writes it last, atomically). A crash after that point is a completed +// round and is not rolled back. + +const ( + pendingFile = ".round-pending.json" + abortedFile = ".round-aborted.json" +) + +// appendOnlyFiles are the files a round appends to before it is sealed. +var appendOnlyFiles = []string{manifestJSONL, "collection.jsonl", "chain-of-custody.jsonl"} + +type pendingRound struct { + Round int `json:"round"` + StartedUTC string `json:"started_utc"` + Lengths map[string]int64 `json:"lengths"` // -1: the file did not exist + CaseJSON []byte `json:"case_json"` + SumsSHA256 string `json:"sums_sha256"` +} + +// AbortedRound describes a round that was rolled back. +type AbortedRound struct { + Round int `json:"round"` + StartedUTC string `json:"started_utc"` + DiscardedBlobs int `json:"discarded_blobs"` + DiscardedBytes int64 `json:"discarded_bytes"` + How string `json:"how"` // "abandoned" (rolled back on exit) or "crashed" (found by the next round) +} + +func sumsDigest(dir string) string { + data, err := os.ReadFile(filepath.Join(dir, sumsFile)) + if err != nil { + return "" + } + s := sha256.Sum256(data) + return hex.EncodeToString(s[:]) +} + +// beginPending records the sealed state a round of an existing package +// starts from. It must run before the round writes anything. +func beginPending(dir string, round int) error { + p := pendingRound{Round: round, StartedUTC: time.Now().UTC().Format(time.RFC3339), Lengths: map[string]int64{}, SumsSHA256: sumsDigest(dir)} + for _, f := range appendOnlyFiles { + fi, err := os.Stat(filepath.Join(dir, f)) + switch { + case errors.Is(err, os.ErrNotExist): + p.Lengths[f] = -1 + case err != nil: + return err + default: + p.Lengths[f] = fi.Size() + } + } + cj, err := os.ReadFile(filepath.Join(dir, "case.json")) + if err != nil { + return err + } + p.CaseJSON = cj + data, err := json.Marshal(p) + if err != nil { + return err + } + return writeAtomic(filepath.Join(dir, pendingFile), data, 0o600) +} + +// endPending marks the round sealed. +func endPending(dir string) { + _ = os.Remove(filepath.Join(dir, pendingFile)) +} + +// rollbackPending restores the sealed state recorded by beginPending, if +// a round was left unsealed. It returns nil when there was nothing to roll +// back. +func rollbackPending(dir, how string) (*AbortedRound, error) { + data, err := os.ReadFile(filepath.Join(dir, pendingFile)) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var p pendingRound + if err := json.Unmarshal(data, &p); err != nil { + return nil, fmt.Errorf("%s: %w", pendingFile, err) + } + if sumsDigest(dir) != p.SumsSHA256 { + // The new seal was written: the round completed. + endPending(dir) + return nil, nil + } + sealed, err := readSums(filepath.Join(dir, sumsFile)) + if err != nil { + return nil, fmt.Errorf("roll back round %d: the seal it started from is unreadable: %w", p.Round, err) + } + // Every blob the seal lists must still be there before anything is + // removed: a seal that does not describe raw/ is not one to restore to. + for rel := range sealed { + if strings.HasPrefix(rel, "raw/") { + if _, err := os.Lstat(filepath.Join(dir, filepath.FromSlash(rel))); err != nil { + return nil, fmt.Errorf("roll back round %d: sealed %s is missing; leaving the package for verify to examine", p.Round, rel) + } + } + } + ab := &AbortedRound{Round: p.Round, StartedUTC: p.StartedUTC, How: how} + for _, f := range appendOnlyFiles { + path := filepath.Join(dir, f) + want, ok := p.Lengths[f] + if !ok { + continue + } + if want < 0 { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + continue + } + fi, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("roll back round %d: %s: %w", p.Round, f, err) + } + if fi.Size() < want { + return nil, fmt.Errorf("roll back round %d: %s is shorter than when the round began; leaving the package for verify to examine", p.Round, f) + } + if err := os.Truncate(path, want); err != nil { + return nil, err + } + } + if err := writeAtomic(filepath.Join(dir, "case.json"), p.CaseJSON, 0o600); err != nil { + return nil, err + } + // Blobs and archived seals the round wrote are not in the seal it + // started from. + for _, sub := range []string{"raw", sealsDir} { + entries, err := os.ReadDir(filepath.Join(dir, sub)) + if err != nil { + continue + } + for _, e := range entries { + if _, ok := sealed[sub+"/"+e.Name()]; ok { + continue + } + path := filepath.Join(dir, sub, e.Name()) + if fi, err := os.Lstat(path); err == nil && sub == "raw" { + ab.DiscardedBlobs++ + ab.DiscardedBytes += fi.Size() + } + _ = os.Chmod(path, 0o600) + if err := os.Remove(path); err != nil { + return nil, err + } + } + } + endPending(dir) + return ab, nil +} + +// noteAborted leaves the record of a rolled-back round for the next round +// to put in its custody chain. +func noteAborted(dir string, ab *AbortedRound) { + if ab == nil { + return + } + var list []AbortedRound + if data, err := os.ReadFile(filepath.Join(dir, abortedFile)); err == nil { + _ = json.Unmarshal(data, &list) + } + list = append(list, *ab) + if data, err := json.Marshal(list); err == nil { + _ = writeAtomic(filepath.Join(dir, abortedFile), data, 0o600) + } +} + +// takeAborted returns and clears the rounds rolled back since the last +// round began. +func takeAborted(dir string) []AbortedRound { + path := filepath.Join(dir, abortedFile) + data, err := os.ReadFile(path) + if err != nil { + return nil + } + var list []AbortedRound + _ = json.Unmarshal(data, &list) + _ = os.Remove(path) + return list +} + +func writeAtomic(path string, data []byte, mode os.FileMode) error { + tmp, err := os.CreateTemp(filepath.Dir(path), ".tmp-"+filepath.Base(path)+"-*") + if err != nil { + return err + } + if _, err := tmp.Write(data); err != nil { + tmp.Close() + os.Remove(tmp.Name()) + return err + } + if err := tmp.Close(); err != nil { + os.Remove(tmp.Name()) + return err + } + if err := os.Chmod(tmp.Name(), mode); err != nil { + os.Remove(tmp.Name()) + return err + } + return os.Rename(tmp.Name(), path) +} diff --git a/internal/casepkg/pending_test.go b/internal/casepkg/pending_test.go new file mode 100644 index 0000000..5875554 --- /dev/null +++ b/internal/casepkg/pending_test.go @@ -0,0 +1,175 @@ +package casepkg + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// sealedCase is a package with one sealed round. +func sealedCase(t *testing.T) (pkg, src string) { + t.Helper() + pkg = filepath.Join(t.TempDir(), "p.adfir") + src = filepath.Join(t.TempDir(), "a.jsonl") + writeFile(t, src, "round one\n") + b, err := New(pkg, "PEND", CaseInfo{OperatorOSUser: "t"}) + if err != nil { + t.Fatal(err) + } + ingest(t, b, src, "a.jsonl") + if err := b.Seal(); err != nil { + t.Fatal(err) + } + return pkg, src +} + +func mustVerify(t *testing.T, pkg, when string) { + t.Helper() + res, err := Verify(pkg) + if err != nil || len(res.Problems) > 0 { + t.Fatalf("%s: package does not verify: %v %v", when, err, res.Problems) + } +} + +func custody(t *testing.T, pkg string) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(pkg, "chain-of-custody.jsonl")) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +// A round that is abandoned (an error, a refused disk floor, Ctrl+C that +// reaches Close) leaves the package exactly as it was sealed, and the next +// round records that it happened. +func TestAbandonedRoundRollsBack(t *testing.T) { + pkg, src := sealedCase(t) + writeFile(t, src, "round one\nround two\n") + b, err := Reopen(pkg, CaseInfo{OperatorOSUser: "t"}) + if err != nil { + t.Fatal(err) + } + other := filepath.Join(t.TempDir(), "b.jsonl") + writeFile(t, other, "new file\n") + ingest(t, b, src, "a.jsonl") + ingest(t, b, other, "b.jsonl") + b.Close() // never sealed + mustVerify(t, pkg, "after an abandoned round") + if _, err := os.Stat(filepath.Join(pkg, pendingFile)); err == nil { + t.Fatal("pending state left behind") + } + + b2, err := Reopen(pkg, CaseInfo{OperatorOSUser: "t"}) + if err != nil { + t.Fatal(err) + } + if err := b2.Seal(); err != nil { + t.Fatal(err) + } + mustVerify(t, pkg, "after the next round") + c := custody(t, pkg) + if !strings.Contains(c, `"event":"round_aborted"`) || !strings.Contains(c, `"how":"abandoned"`) { + t.Fatal("the next round did not record the abandoned one") + } +} + +// A process that dies mid-round leaves its lock and pending state. The +// next round rolls the package back before checking or extending it. +func TestCrashedRoundIsRolledBackByTheNextRound(t *testing.T) { + pkg, src := sealedCase(t) + writeFile(t, src, "round one\nround two\n") + b, err := Reopen(pkg, CaseInfo{OperatorOSUser: "t"}) + if err != nil { + t.Fatal(err) + } + ingest(t, b, src, "a.jsonl") + // Simulate the process dying: files closed by the OS, nothing rolled + // back, the lock left as a stale lock of a dead process. + b.pending = false + b.coll.Close() + b.custody.Close() + b.mf.Close() + b.coll, b.custody, b.mf = nil, nil, nil + b.lock.release() + b.lock = nil + if res, _ := VerifyQuick(pkg); len(res.Problems) == 0 { + t.Fatal("setup: a crashed round should leave the package not matching its seal") + } + + var sawClean bool + b2, err := ReopenChecked(pkg, CaseInfo{OperatorOSUser: "t"}, func() { + res, err := VerifyQuick(pkg) + sawClean = err == nil && len(res.Problems) == 0 + }) + if err != nil { + t.Fatal(err) + } + if !sawClean { + t.Fatal("the check before the next round saw the crashed round's tail") + } + if err := b2.Seal(); err != nil { + t.Fatal(err) + } + mustVerify(t, pkg, "after recovery") + if !strings.Contains(custody(t, pkg), `"how":"crashed"`) { + t.Fatal("the crashed round was not recorded") + } +} + +// A crash after the new seal was written is a completed round: nothing is +// rolled back. +func TestCrashAfterTheSealKeepsTheRound(t *testing.T) { + pkg, src := sealedCase(t) + writeFile(t, src, "round one\nround two\n") + b, err := Reopen(pkg, CaseInfo{OperatorOSUser: "t"}) + if err != nil { + t.Fatal(err) + } + ingest(t, b, src, "a.jsonl") + pending, err := os.ReadFile(filepath.Join(pkg, pendingFile)) + if err != nil { + t.Fatal(err) + } + if err := b.Seal(); err != nil { + t.Fatal(err) + } + // Put the pending file back, as if the process died right after the + // seal was written. + if err := os.WriteFile(filepath.Join(pkg, pendingFile), pending, 0o600); err != nil { + t.Fatal(err) + } + b2, err := Reopen(pkg, CaseInfo{OperatorOSUser: "t"}) + if err != nil { + t.Fatal(err) + } + if err := b2.Seal(); err != nil { + t.Fatal(err) + } + ci, err := ReadCaseInfo(pkg) + if err != nil { + t.Fatal(err) + } + if len(ci.Rounds) != 3 { + t.Fatalf("rounds = %d, want 3: the sealed round 2 was rolled back", len(ci.Rounds)) + } + mustVerify(t, pkg, "after a crash past the seal") +} + +// A package whose first round never seals is removed: there is no sealed +// state to keep, and a half-built package would only be mistaken for one. +func TestUnsealedFirstRoundIsRemoved(t *testing.T) { + pkg := filepath.Join(t.TempDir(), "new.adfir") + src := filepath.Join(t.TempDir(), "a.jsonl") + writeFile(t, src, "x\n") + b, err := New(pkg, "NEW", CaseInfo{OperatorOSUser: "t"}) + if err != nil { + t.Fatal(err) + } + ingest(t, b, src, "a.jsonl") + b.Close() + if _, err := os.Stat(pkg); !os.IsNotExist(err) { + t.Fatalf("unsealed first-round package left behind: %v", err) + } +} diff --git a/internal/casepkg/retire.go b/internal/casepkg/retire.go index 8fe1f60..0305892 100644 --- a/internal/casepkg/retire.go +++ b/internal/casepkg/retire.go @@ -33,22 +33,28 @@ func ExcludedByPolicy(path string) bool { // removes a record, and a later round that collects the path again wins. const retiredFile = "retired.json" -// RetireExcluded marks every current record ExcludedByPolicy would have -// skipped. Returns how many records left the scan set. Evidence records -// stay in Artifacts; only Current() changes. +// RetireExcluded marks every acquired record ExcludedByPolicy would have +// skipped. Returns how many records newly left the scan set — zero when +// nothing changed since the last call persisted by WriteRetired. Evidence +// records stay in Artifacts; only Current() changes. +// +// It used to count every excluded record, retired before or not, and the +// policy's own SKIPPED_BY_POLICY placeholders with them. Every round writes +// fresh placeholders, so the count was never zero, and analysis took it as +// a reason to re-parse the whole case — on every run. func (m *Manifest) RetireExcluded() int { if m.retired == nil { m.retired = map[string]int{} } n := 0 for _, a := range m.Artifacts { - if !ExcludedByPolicy(a.SourcePath) { - continue + if a.Status != StatusOK || !ExcludedByPolicy(a.SourcePath) { + continue // placeholders and failures carry no content to scan } if r, ok := m.retired[a.SourcePath]; !ok || a.Round > r { m.retired[a.SourcePath] = a.Round + n++ } - n++ } return n } diff --git a/internal/casepkg/retire_test.go b/internal/casepkg/retire_test.go index d509b34..42d2f3b 100644 --- a/internal/casepkg/retire_test.go +++ b/internal/casepkg/retire_test.go @@ -61,3 +61,25 @@ func TestRetiredListPersistsAcrossReadManifest(t *testing.T) { t.Fatalf("after reopen current = %+v, want only SKILL.md", cur) } } + +// Retiring is reported once. The count used to include records retired +// before and the policy's own SKIPPED_BY_POLICY placeholders, which every +// round writes afresh, so it was never zero and analysis re-parsed the +// whole case on every run. +func TestRetireCountsOnlyNewlyRetiredContent(t *testing.T) { + junk := "/h/.claude/plugins/cache/x/node_modules/a.js" + man := &Manifest{Artifacts: []ArtifactRecord{ + {SourcePath: junk, Round: 1, Status: StatusOK}, + {SourcePath: "/h/.claude/plugins/cache/y/.git/objects", Round: 1, Status: StatusSkippedPolicy}, + }} + if n := man.RetireExcluded(); n != 1 { + t.Fatalf("first pass retired %d, want 1 (the content, not the placeholder)", n) + } + if n := man.RetireExcluded(); n != 0 { + t.Fatalf("second pass retired %d, want 0: nothing changed", n) + } + man.Artifacts = append(man.Artifacts, ArtifactRecord{SourcePath: "/h/.claude/plugins/cache/y/.git/objects", Round: 2, Status: StatusSkippedPolicy}) + if n := man.RetireExcluded(); n != 0 { + t.Fatalf("a new round's placeholder counted as retired content: %d", n) + } +} diff --git a/internal/cli/cli.go b/internal/cli/cli.go index a86ef8c..aa39541 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -13,6 +13,7 @@ import ( "github.com/efij/AgentDFIR/v3/internal/casepkg" "github.com/efij/AgentDFIR/v3/internal/collector" + "github.com/efij/AgentDFIR/v3/internal/integrity" "github.com/efij/AgentDFIR/v3/internal/live" "github.com/efij/AgentDFIR/v3/internal/products" "github.com/efij/AgentDFIR/v3/internal/sanitize" @@ -244,14 +245,16 @@ func cmdCollect(args []string) int { noShare := fs.Bool("no-share", false, "do not share identical blobs with other cases on this machine") fullPlugins := fs.Bool("full-plugins", false, "also collect node_modules/.git subtrees (large, third-party)") liveMode := fs.Bool("live", false, "collect volatile evidence first (RFC 3227 order)") - signKey := fs.String("sign", "", "sign the sealed package with this ed25519 private key") + signKey := fs.String("sign", "", "sign the sealed round with this ed25519 private key (default: this machine's key)") + noSign := fs.Bool("no-sign", false, "do not sign the sealed round") + verifyPrior := fs.String("verify-prior", "quick", "before adding a round, check earlier rounds: quick | full (re-hash every blob)") importTree := fs.String("import", "", "KAPE/Velociraptor/CyLR/image tree: collect every product for every user profile found") dockerRef := fs.String("docker", "", "container id/name (docker export, read-only) or a saved export .tar") archive := fs.String("archive", "", "zip / tar / tar.gz: GitHub Actions artifact, support bundle, vendor data export") if err := fs.Parse(args); err != nil { return 2 } - base := importOpts{out: *out, caseID: *caseID, operator: *operator, authz: *authz, signKey: *signKey, maxFileMB: *maxFileMB, args: args} + base := importOpts{out: *out, caseID: *caseID, operator: *operator, authz: *authz, signKey: *signKey, noSign: *noSign, maxFileMB: *maxFileMB, args: args} switch { case *importTree != "": base.tree = *importTree @@ -359,13 +362,17 @@ func cmdCollect(args []string) int { } } - b, reopened, err := openPackage(dest, id, info, !*noShare) + var prior integrity.Prior + b, reopened, err := openPackage(dest, id, info, !*noShare, func() { + prior = checkPrior(os.Stdout, dest, *verifyPrior) + }) if err != nil { fmt.Fprintln(os.Stderr, "error:", err) return 1 } defer b.Close() if reopened { + prior.Apply(b) fmt.Printf("Adding round %d to the existing package %s\n", b.Round(), sanitize.Terminal(dest)) } host, _ := os.Hostname() @@ -404,15 +411,15 @@ func cmdCollect(args []string) int { "duration_ms": time.Since(start).Milliseconds(), }) roundStats := b.Stats() + signing := prepareSigning(b, *signKey, *noSign) if err := b.Seal(); err != nil { fmt.Fprintln(os.Stderr, "seal error:", err) return 1 } - if *signKey != "" { - if err := seal.Sign(dest, *signKey); err != nil { - fmt.Fprintln(os.Stderr, "sign error:", err) - return 1 - } + sealed, err := finishSeal(os.Stdout, dest, signing, b.CaseID(), b.Round(), true) + if err != nil { + fmt.Fprintln(os.Stderr, "sign error:", err) + return 1 } if runErr != nil { fmt.Fprintln(os.Stderr, "collection error (package sealed with partial evidence):", runErr) @@ -431,13 +438,16 @@ func cmdCollect(args []string) int { if liveStats != nil { fmt.Printf("Volatile: %d collected, %d failed (live mode)\n", liveStats.Collected, liveStats.Failed) } - if *signKey != "" { + if sealed.Signed { fmt.Println("Signed: SEAL.sig written (ed25519).") } fmt.Println("Sealed: SHA256SUMS written; run `agentdfir verify` to confirm integrity.") if runErr != nil { return 1 } + if prior.Status == integrity.Failed { + return exitPriorFailed + } return 0 } diff --git a/internal/cli/correlate_cmd.go b/internal/cli/correlate_cmd.go index cb8bd41..f914194 100644 --- a/internal/cli/correlate_cmd.go +++ b/internal/cli/correlate_cmd.go @@ -10,6 +10,7 @@ import ( "github.com/efij/AgentDFIR/v3/internal/correlate" "github.com/efij/AgentDFIR/v3/internal/endpoint" + "github.com/efij/AgentDFIR/v3/internal/normalize" "github.com/efij/AgentDFIR/v3/internal/overlay" "github.com/efij/AgentDFIR/v3/internal/sanitize" "github.com/efij/AgentDFIR/v3/internal/schema" @@ -102,6 +103,10 @@ func runEndpointCorrelation(pkg string, logs []string, f endpoint.Format, opts c fmt.Fprintln(os.Stderr, "error:", err) return nil, nil, 1 } + if err := normalize.RecordEvents(pkg); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + return nil, nil, 1 + } detDir := filepath.Join(pkg, "detections") _ = os.MkdirAll(detDir, 0o700) out := struct { diff --git a/internal/cli/import_cmd.go b/internal/cli/import_cmd.go index 7683942..e7b950b 100644 --- a/internal/cli/import_cmd.go +++ b/internal/cli/import_cmd.go @@ -11,12 +11,12 @@ import ( "github.com/efij/AgentDFIR/v3/internal/collector" "github.com/efij/AgentDFIR/v3/internal/products" "github.com/efij/AgentDFIR/v3/internal/sanitize" - "github.com/efij/AgentDFIR/v3/internal/seal" ) // importOpts carries the collect flags that apply to tree import. type importOpts struct { tree, out, caseID, operator, authz, signKey string + noSign bool maxFileMB int64 args []string notes map[string]string // extra case notes (docker/archive provenance) @@ -141,15 +141,15 @@ func collectImport(o importOpts) int { fmt.Printf(" no profile layout found — preserved %d loose JSON/JSONL file(s) as archive.sessions\n", st.Acquired) } _ = b.Log("import_finished", map[string]any{"duration_ms": time.Since(start).Milliseconds()}) + signing := prepareSigning(b, o.signKey, o.noSign) if err := b.Seal(); err != nil { fmt.Fprintln(os.Stderr, "seal error:", err) return 1 } - if o.signKey != "" { - if err := seal.Sign(dest, o.signKey); err != nil { - fmt.Fprintln(os.Stderr, "sign error:", err) - return 1 - } + sealed, err := finishSeal(os.Stdout, dest, signing, id, b.Round(), true) + if err != nil { + fmt.Fprintln(os.Stderr, "sign error:", err) + return 1 } fmt.Printf("Case: %s\n", id) fmt.Printf("Package: %s\n", dest) @@ -157,7 +157,7 @@ func collectImport(o importOpts) int { fmt.Printf("Acquired: %d artifacts (%d bytes)\n", total.Acquired, total.TotalBytes) fmt.Printf("Symlinks: %d recorded (never followed)\n", total.Symlinks) fmt.Printf("Skipped: %d Failed: %d\n", total.Skipped, total.Failed) - if o.signKey != "" { + if sealed.Signed { fmt.Println("Signed: SEAL.sig written (ed25519).") } fmt.Println("Sealed: SHA256SUMS written; run `agentdfir verify` to confirm integrity.") diff --git a/internal/cli/main_test.go b/internal/cli/main_test.go new file mode 100644 index 0000000..99ba255 --- /dev/null +++ b/internal/cli/main_test.go @@ -0,0 +1,28 @@ +package cli + +import ( + "os" + "testing" + + "github.com/efij/AgentDFIR/v3/internal/store" +) + +// TestMain points the AgentDFIR home at a throwaway directory. Sealing now +// signs with the machine key and records an anchor in the home; a test must +// never create a key in, or write anchors to, the developer's real one. +func TestMain(m *testing.M) { + dir, err := os.MkdirTemp("", "agentdfir-home-*") + if err != nil { + panic(err) + } + if err := os.Chmod(dir, 0o700); err != nil { + panic(err) + } + os.Setenv(store.EnvHome, dir) + // The free-disk floor protects real machines; tests must not depend + // on how full the machine running them is. + os.Setenv("AGENTDFIR_MIN_FREE_GB", "1") + code := m.Run() + os.RemoveAll(dir) + os.Exit(code) +} diff --git a/internal/cli/pkgopen.go b/internal/cli/pkgopen.go index 950313a..ca85716 100644 --- a/internal/cli/pkgopen.go +++ b/internal/cli/pkgopen.go @@ -3,9 +3,13 @@ package cli import ( "errors" "fmt" + "io" "os" "github.com/efij/AgentDFIR/v3/internal/casepkg" + "github.com/efij/AgentDFIR/v3/internal/integrity" + "github.com/efij/AgentDFIR/v3/internal/sanitize" + "github.com/efij/AgentDFIR/v3/internal/seal" "github.com/efij/AgentDFIR/v3/internal/store" ) @@ -17,9 +21,13 @@ import ( // identical evidence. It is append-only: the existing manifest and both // hash chains are continued, the previous seal is archived, and nothing // already recorded is rewritten. -func openPackage(dest, caseID string, info casepkg.CaseInfo, share bool) (b *casepkg.Builder, reopened bool, err error) { +// +// beforeRound, when set, runs on an existing package once it is locked and +// before this round writes anything to it — where the earlier rounds are +// checked. +func openPackage(dest, caseID string, info casepkg.CaseInfo, share bool, beforeRound func()) (b *casepkg.Builder, reopened bool, err error) { if _, statErr := os.Stat(dest); statErr == nil { - b, err = casepkg.Reopen(dest, info) + b, err = casepkg.ReopenChecked(dest, info, beforeRound) reopened = true } else if errors.Is(statErr, os.ErrNotExist) { b, err = casepkg.New(dest, caseID, info) @@ -38,3 +46,65 @@ func openPackage(dest, caseID string, info casepkg.CaseInfo, share bool) (b *cas } return b, reopened, nil } + +// exitPriorFailed is the exit status when an earlier round of the case +// failed its integrity check. The new round is still sealed. +const exitPriorFailed = 4 + +// checkPrior proves an existing case's earlier rounds before a round is +// added and prints what it found. The caller records it on the round +// (Prior.Apply) once the builder exists. +func checkPrior(w io.Writer, dest string, depth string) integrity.Prior { + p := integrity.CheckPrior(dest, depth == "full") + switch p.Status { + case integrity.Verified: + how := "signature valid" + if p.Anchored { + how += ", matches the anchor" + } + fmt.Fprintf(w, " Earlier rounds verified (%s, %s check)\n", how, p.Depth) + case integrity.Unsigned: + fmt.Fprintf(w, " Earlier rounds verified (%s check; unsigned, so tampering by someone who could rewrite the seal is not excluded)\n", p.Depth) + default: + fmt.Fprintf(w, " WARNING: earlier rounds of this case FAILED their integrity check (%s):\n", p.Depth) + for _, pr := range p.Problems { + fmt.Fprintf(w, " - %s\n", sanitize.Terminal(pr)) + } + fmt.Fprintln(w, " The new round is collected and sealed, and records this failure permanently. Nothing earlier is repaired or re-signed as clean.") + } + for _, n := range p.Notes { + fmt.Fprintln(w, " note:", sanitize.Terminal(n)) + } + return p +} + +// prepareSigning chooses the round's key before sealing and records it on +// the round, so case.json names the key its seal was made for. +func prepareSigning(b *casepkg.Builder, explicitKey string, noSign bool) integrity.Signing { + s := integrity.Prepare(explicitKey, noSign) + if s.KeyPath != "" { + b.Signer = s.Public + } + return s +} + +// finishSeal signs the sealed round and anchors it, then prints the one +// line an analyst can paste into a ticket to pin this exact state. +func finishSeal(w io.Writer, dest string, s integrity.Signing, caseID string, round int, anchor bool) (integrity.Sealed, error) { + res, err := integrity.Finish(dest, s, caseID, round, anchor) + if err != nil { + return res, err + } + line := fmt.Sprintf(" Seal %s · round %d", res.Digest[:16], round) + if res.Signed { + line += " · signed " + seal.Fingerprint(s.Public) + } + if res.Anchored { + line += " · anchored" + } + fmt.Fprintln(w, line) + for _, n := range res.Notes { + fmt.Fprintln(w, " note:", sanitize.Terminal(n)) + } + return res, nil +} diff --git a/internal/cli/progress.go b/internal/cli/progress.go index 2325890..07b2deb 100644 --- a/internal/cli/progress.go +++ b/internal/cli/progress.go @@ -2,118 +2,11 @@ package cli import ( "fmt" - "io" - "os" - "sync" "time" ) -// progress is the one status line a long step keeps redrawing on a terminal -// (" claude-code 3,120 artifacts · 412 MB · 14s"). It is silent when stdout -// is not a terminal, and it is an io.Writer so ordinary log lines from the -// analysis stages print cleanly above it instead of tearing through it. -type progress struct { - mu sync.Mutex - out io.Writer - tty bool - label string - detail string - start time.Time - shown bool - stop chan struct{} - done chan struct{} -} - -func newProgress() *progress { - p := &progress{out: os.Stdout} - if fi, err := os.Stdout.Stat(); err == nil && fi.Mode()&os.ModeCharDevice != 0 && os.Getenv("NO_COLOR") == "" && os.Getenv("TERM") != "dumb" { - p.tty = true - } - return p -} - -// Start begins a status line for one step; the elapsed time ticks on its own. -func (p *progress) Start(label string) { - p.Stop() - p.mu.Lock() - p.label, p.detail, p.start = label, "", time.Now() - p.stop, p.done = make(chan struct{}), make(chan struct{}) - p.mu.Unlock() - if !p.tty { - return - } - go func() { - defer close(p.done) - t := time.NewTicker(250 * time.Millisecond) - defer t.Stop() - for { - select { - case <-p.stop: - return - case <-t.C: - p.mu.Lock() - p.draw() - p.mu.Unlock() - } - } - }() -} - -// Set updates the variable part of the line (counts, bytes). -func (p *progress) Set(detail string) { - p.mu.Lock() - p.detail = detail - p.mu.Unlock() -} - -// Stop ends the step and clears the line so the final summary can take its place. -func (p *progress) Stop() { - p.mu.Lock() - if p.stop == nil { - p.mu.Unlock() - return - } - close(p.stop) - p.stop = nil - done := p.done - p.mu.Unlock() - if p.tty { - <-done - } - p.mu.Lock() - p.clear() - p.label = "" - p.mu.Unlock() -} - -// Write lets stage log lines print above the status line. -func (p *progress) Write(b []byte) (int, error) { - p.mu.Lock() - defer p.mu.Unlock() - p.clear() - n, err := p.out.Write(b) - p.draw() - return n, err -} - -func (p *progress) clear() { - if p.tty && p.shown { - fmt.Fprint(p.out, "\r\033[2K") - p.shown = false - } -} - -func (p *progress) draw() { - if !p.tty || p.label == "" { - return - } - line := p.label - if p.detail != "" { - line += " · " + p.detail - } - fmt.Fprintf(p.out, "\r\033[2K%s · %s", line, elapsed(time.Since(p.start))) - p.shown = true -} +// The run command's status display is internal/progress. These are the +// small formatting helpers the commands share. func elapsed(d time.Duration) string { s := int(d.Seconds()) @@ -134,25 +27,3 @@ func humanBytes(n int64) string { } return fmt.Sprintf("%d B", n) } - -// eta renders a real time-remaining from measured throughput. -// -// It is only ever shown for acquisition, where a metadata-only pre-walk has -// already established exactly how many bytes there are. The analysis stages -// deliberately show no ETA: their costs differ by an order of magnitude and -// a fabricated number is worse than none. -func eta(done, total int64, since time.Duration) string { - if done <= 0 || total <= 0 || done >= total || since < 2*time.Second { - return "" - } - rate := float64(done) / since.Seconds() - if rate <= 0 { - return "" - } - return " · ~" + elapsed(time.Duration(float64(total-done)/rate)*time.Second) + " left" -} - -// step prints a completed step's heading with the time it took. -func stepDone(label string, since time.Duration) { - fmt.Printf(" ✓ %s in %s\n", label, elapsed(since)) -} diff --git a/internal/cli/run_cmd.go b/internal/cli/run_cmd.go index fa4286f..89afacf 100644 --- a/internal/cli/run_cmd.go +++ b/internal/cli/run_cmd.go @@ -3,16 +3,21 @@ package cli import ( "flag" "fmt" + "io" "os" "os/user" "path/filepath" + "strconv" "time" "github.com/efij/AgentDFIR/v3/internal/analysis" "github.com/efij/AgentDFIR/v3/internal/casepkg" "github.com/efij/AgentDFIR/v3/internal/collector" + "github.com/efij/AgentDFIR/v3/internal/gentle" + "github.com/efij/AgentDFIR/v3/internal/integrity" "github.com/efij/AgentDFIR/v3/internal/normalize" "github.com/efij/AgentDFIR/v3/internal/products" + "github.com/efij/AgentDFIR/v3/internal/progress" "github.com/efij/AgentDFIR/v3/internal/sanitize" "github.com/efij/AgentDFIR/v3/internal/schema" "github.com/efij/AgentDFIR/v3/internal/seal" @@ -26,6 +31,13 @@ import ( // sealed package, analyze it, open the case explorer. Each step calls the // same code the individual commands use; nothing is skipped or approximated. // `detect`, `collect`, `analyze` and `serve` remain for every other case. +// +// A repeat run is a delta: earlier rounds are proven intact, unchanged +// files are carried forward, only changed transcripts are parsed, and when +// neither the evidence nor the analysis code changed the stored results +// are reused outright. It is gentle on the machine by default — lowered +// priority, capped workers and heap, paced reads, a pause while the host is +// busy, and a free-disk floor it never crosses. func cmdRun(args []string) int { fs := flag.NewFlagSet("run", flag.ContinueOnError) product := fs.String("product", "", "collect only this product (default: every detected agent)") @@ -34,13 +46,21 @@ func cmdRun(args []string) int { operator := fs.String("operator", "", "asserted operator name") authz := fs.String("authorization", "", "authorization reference") maxFileMB := fs.Int64("max-file-mb", 0, "per-artifact size bound (MiB)") - jobs := fs.Int("jobs", 0, "parallel acquisition workers (default: CPUs, max 8)") + jobs := fs.Int("jobs", 0, "parallel acquisition workers (default: half the CPUs, max 4; --priority normal: CPUs, max 8)") newCase := fs.Bool("new", false, "start a fresh case instead of adding a round to the existing one") recollect := fs.Bool("recollect", false, "re-read every file, even one an earlier round already preserved") noWitness := fs.Bool("no-witness", false, "do not ask the host whether the files the agent claimed to write exist") noShare := fs.Bool("no-share", false, "do not share identical blobs with other cases on this machine") fullPlugins := fs.Bool("full-plugins", false, "also collect node_modules/.git subtrees (large, third-party)") - signKey := fs.String("sign", "", "sign the sealed package with this ed25519 private key") + signKey := fs.String("sign", "", "sign the sealed round with this ed25519 private key (default: this machine's key)") + noSign := fs.Bool("no-sign", false, "do not sign the sealed round") + verifyPrior := fs.String("verify-prior", "quick", "before adding a round, check earlier rounds: quick | full (re-hash every blob)") + priority := fs.String("priority", "gentle", "how much to yield to the rest of the machine: gentle | background | normal") + maxMemMB := fs.Int64("max-memory-mb", 0, "soft heap limit in MiB (default: gentle/background 25% of RAM, 512 MiB–4 GiB)") + readMBps := fs.Int("max-read-mbps", -1, "cap acquisition reads (MB/s; default gentle 200, background 50, normal unlimited; 0 = unlimited)") + minFreeGB := fs.Int("min-free-gb", envInt("AGENTDFIR_MIN_FREE_GB"), "never leave less free disk than this on the evidence volume (default: 2 GB, or 1% of the volume up to 5 GB; env AGENTDFIR_MIN_FREE_GB)") + noGovernor := fs.Bool("no-governor", false, "do not pause acquisition while the machine is busy") + reanalyze := fs.Bool("reanalyze", false, "re-run the analysis even when the stored results are current") var endpointLogs multiFlag fs.Var(&endpointLogs, "endpoint", "OS telemetry log (auditd, Sysmon XML, JSONL/CSV export); repeatable") gwLog := fs.String("gateway-log", "", "MCP gateway log (JSONL) to check MCP calls against") @@ -50,29 +70,117 @@ func cmdRun(args []string) int { noOpen := fs.Bool("no-open", false, "print the URL but do not open the browser") noServe := fs.Bool("no-serve", false, "stop after analysis and print the findings (scripts, CI)") if err := fs.Parse(args); err != nil || fs.NArg() != 0 { - fmt.Fprintln(os.Stderr, "usage: agentdfir run [--product

] [--out

] [--endpoint ]... [--new] [--jobs N] [--no-open] [--no-serve]") + fmt.Fprintln(os.Stderr, "usage: agentdfir run [--product

] [--out

] [--endpoint ]... [--new] [--priority gentle|background|normal] [--no-open] [--no-serve]") + return 2 + } + mode, err := gentle.ParseMode(*priority) + if err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + return 2 + } + if *verifyPrior != "quick" && *verifyPrior != "full" { + fmt.Fprintln(os.Stderr, "error: --verify-prior must be quick or full") return 2 } + settings := gentle.Apply(mode, *jobs, *maxMemMB<<20) home, err := os.UserHomeDir() if err != nil { fmt.Fprintln(os.Stderr, "error:", err) return 1 } - runStart := time.Now() + + // Where the case lives: one per host/user at a predictable path, so + // running from a different directory adds a round to the same case + // instead of copying every byte of evidence again. + id := *caseID + if id == "" { + id = generateCaseID() + } + host, _ := os.Hostname() + osUser := "" + if u, err := user.Current(); err == nil { + osUser = u.Username + } + dest := *out + if dest == "" { + d, err := store.CaseDir(host, osUser) + if err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + return 1 + } + dest = d + if *newCase { + dest = filepath.Join(filepath.Dir(d), id+".adfir") + } + } + if *newCase { + if _, err := os.Stat(dest); err == nil { + fmt.Fprintf(os.Stderr, "error: --new was given but %s already exists; remove it or choose another --out\n", sanitize.Terminal(dest)) + return 1 + } + } + _, statErr := os.Stat(dest) + reopening := statErr == nil + + // One display for the whole run, sized from how long each step took on + // this machine before. + perfPath := "" + if h, err := store.Home(); err == nil { + perfPath = filepath.Join(h, "perf.jsonl") + } + prevEvents := 0 + if reopening { + prevEvents = analysis.PreviousEvents(dest) + } + steps := []progress.Step{ + {Name: "detect", Label: "Detect — which AI agents are on this machine", Rate: 0.5}, + {Name: "verify-prior", Label: "Verify — earlier rounds of this case", Rate: 2}, + {Name: "survey", Label: "Collect — sizing the collection", Rate: 2}, + {Name: "collect", Label: "Collect — acquiring evidence", Units: 1, Rate: 1.0 / (120 << 20)}, + {Name: "witness", Label: "Collect — asking the host about the agent's claims", Units: 1, Rate: 1.0 / (40 << 20)}, + {Name: "seal", Label: "Collect — sealing and signing", Rate: 1}, + {Name: "analyze", Label: "Analyze — detections, MCP audit, provenance", Units: float64(max(prevEvents, 1)), Rate: 0.0008}, + {Name: "serve", Label: "Look — loading the case explorer", Rate: 3}, + } + tr := progress.New(steps, progress.Options{Key: dest, Model: progress.LoadModel(perfPath)}) + if !reopening { + tr.Skip("verify-prior") + } + if *noWitness { + tr.Skip("witness") + } + if *noServe { + tr.Skip("serve") + } + tr.Run() + stopped := false + stop := func() { + if !stopped { + stopped = true + tr.Stop() + } + } + defer stop() + fail := func(code int, format string, a ...any) int { + stop() + fmt.Fprintf(os.Stderr, "error: "+format+"\n", a...) + return code + } + w := io.Writer(tr) // 1. detect - stepStart := time.Now() - fmt.Println("Step 1/4 Detect — which AI agents are on this machine (none is executed)") + fmt.Fprintf(w, "Step 1/4 Detect — which AI agents are on this machine (none is executed)\n") + fmt.Fprintf(w, " Priority: %s · %d worker(s)\n", settings.Priority, settings.Workers) + tr.Begin("detect") var targets []string if *product != "" { targets = []string{canonicalProductID(*product)} - fmt.Printf(" %s (requested)\n", targets[0]) + fmt.Fprintf(w, " %s (requested)\n", targets[0]) } else { dets, err := products.DetectAll(home) if err != nil { - fmt.Fprintln(os.Stderr, "error:", err) - return 1 + return fail(1, "%v", err) } for _, d := range dets { if !d.Detected { @@ -80,111 +188,122 @@ func cmdRun(args []string) int { } man, mErr := products.Manifest(d.Product.ID) if mErr != nil || man == nil { - fmt.Printf(" %-16s detected (no collector yet — skipped)\n", d.Product.Name) + fmt.Fprintf(w, " %-16s detected (no collector yet — skipped)\n", d.Product.Name) continue } - fmt.Printf(" %-16s detected\n", d.Product.Name) + fmt.Fprintf(w, " %-16s detected\n", d.Product.Name) targets = append(targets, d.Product.ID) } } - stepDone("detected "+fmt.Sprint(len(targets))+" agent(s)", time.Since(stepStart)) + tr.End() if len(targets) == 0 { - fmt.Fprintln(os.Stderr, "no AI agents found for this user. Evidence somewhere else? agentdfir collect --path | --import | --docker | --archive ") - return 1 + return fail(1, "no AI agents found for this user. Evidence somewhere else? agentdfir collect --path | --import | --docker | --archive ") } // 2. collect - id := *caseID - if id == "" { - id = generateCaseID() - } - host, _ := os.Hostname() - osUser := "" - if u, err := user.Current(); err == nil { - osUser = u.Username - } - // One case per host/user at a predictable path, so running from a - // different directory adds a round to the same case instead of copying - // every byte of evidence again. - dest := *out - if dest == "" { - d, err := store.CaseDir(host, osUser) - if err != nil { - fmt.Fprintln(os.Stderr, "error:", err) - return 1 - } - dest = d - if *newCase { - dest = filepath.Join(filepath.Dir(d), id+".adfir") - } - } info := casepkg.CaseInfo{ OperatorOSUser: osUser, OperatorAsserted: *operator, Authorization: *authz, CollectionArgs: append([]string{"run"}, args...), Notes: map[string]string{"mode": "current-user", "run": "detect+collect+analyze"}, } - if *newCase { - if _, err := os.Stat(dest); err == nil { - fmt.Fprintf(os.Stderr, "error: --new was given but %s already exists; remove it or choose another --out\n", sanitize.Terminal(dest)) - return 1 - } - } - b, reopened, err := openPackage(dest, id, info, !*noShare) + var prior integrity.Prior + b, reopened, err := openPackage(dest, id, info, !*noShare, func() { + fmt.Fprintf(w, "\nStep 2/4 Collect — adding a round to the existing case %s\n", sanitize.Terminal(dest)) + tr.Begin("verify-prior") + prior = checkPrior(w, dest, *verifyPrior) + }) if err != nil { - fmt.Fprintln(os.Stderr, "error:", err) - return 1 + return fail(1, "%v", err) } defer b.Close() if reopened { - fmt.Printf("\nStep 2/4 Collect — round %d of existing case %s\n", b.Round(), sanitize.Terminal(dest)) - fmt.Println(" Unchanged files are carried forward, not re-read; files that only grew store just the new tail.") + prior.Apply(b) + fmt.Fprintf(w, " Round %d. Unchanged files are carried forward, not re-read; files that only grew store just the new tail.\n", b.Round()) } else { - fmt.Printf("\nStep 2/4 Collect — sealed evidence package %s\n", sanitize.Terminal(dest)) + fmt.Fprintf(w, "\nStep 2/4 Collect — sealed evidence package %s\n", sanitize.Terminal(dest)) } - stepStart = time.Now() - tune := collectTuning{MaxFileMB: *maxFileMB, Jobs: *jobs, Recollect: *recollect, FullContent: *fullPlugins} - // Metadata-only pre-walk: one or two seconds buys an honest percentage - // and time-remaining instead of a spinner with no end in sight. - prog := newProgress() - prog.Start(" sizing the collection") + // Metadata-only pre-walk: counts the bytes that will really be read — + // files an earlier round already holds are not work — so the time + // remaining is about this run, not about the size of the profile. + tr.Begin("survey") + tune := collectTuning{MaxFileMB: *maxFileMB, Jobs: settings.Workers, Recollect: *recollect, FullContent: *fullPlugins} var plan collector.Survey for _, pid := range targets { - s, err := surveyCurrentUser(pid, home, host, osUser, tune) + s, err := surveyCurrentUser(pid, home, host, osUser, tune, b) if err == nil { plan.Files += s.Files plan.Bytes += s.Bytes plan.Skipped += s.Skipped + plan.Carried += s.Carried + plan.ReadBytes += s.ReadBytes } } - prog.Stop() + tr.End() if plan.Files > 0 { - fmt.Printf(" %d files · %s to acquire\n", plan.Files, humanBytes(plan.Bytes)) + if plan.Carried > 0 { + fmt.Fprintf(w, " %d files · %s to read · %d unchanged since the last round\n", plan.Files-plan.Carried, humanBytes(plan.ReadBytes), plan.Carried) + } else { + fmt.Fprintf(w, " %d files · %s to acquire\n", plan.Files, humanBytes(plan.ReadBytes)) + } + } + + // Never take the last of the disk: refuse up front when the round + // cannot fit above the floor, and stop cleanly if the floor is reached + // anyway. Evidence compresses about 5x; the overlay needs headroom. + volume := dest + if !reopened { + volume = filepath.Dir(dest) + } + floor := gentle.Floor(volume, *minFreeGB) + if err := gentle.Preflight(volume, plan.ReadBytes/3+(256<<20), floor); err != nil { + return fail(exitDiskFloor, "%v", err) + } + rate := *readMBps + if rate < 0 { + switch mode { + case gentle.Gentle: + rate = 200 + case gentle.Background: + rate = 50 + default: + rate = 0 + } } + gov := gentle.NewGovernor(gentle.GovernorOptions{ + ReadMBps: rate, DiskPath: volume, FloorByte: floor, + Adaptive: mode != gentle.Normal && !*noGovernor, + }) + defer gov.Close() + tune.Pace = gov.Pace + // A carried-forward file costs a stat, not a read: count it as a small + // fixed amount of work so progress moves through a round of unchanged + // files without pretending they were gigabytes read. + const statCost = 64 << 10 + tr.SetUnits("collect", float64(plan.ReadBytes+int64(plan.Files)*statCost)) + tr.Begin("collect") var total collector.Stats var collectErr error for _, pid := range targets { - prog.Start(fmt.Sprintf(" %-16s collecting", pid)) st, err := collectCurrentUser(b, pid, home, host, osUser, tune, func(s collector.Stats) { - done := total.TotalBytes + s.TotalBytes - line := fmt.Sprintf("%d new · %d carried · %s", s.Acquired, s.Carried, humanBytes(s.TotalBytes)) - if plan.Bytes > 0 { - line = fmt.Sprintf("%d%% · %s/%s · %d new · %d carried", - min(100, int(100*done/plan.Bytes)), humanBytes(done), humanBytes(plan.Bytes), s.Acquired, s.Carried) - line += eta(done, plan.Bytes, time.Since(stepStart)) + tr.Progress(float64(total.ReadBytes + s.ReadBytes + int64(total.Processed+s.Processed)*statCost)) + busy := "" + if gov.Busy() { + busy = " · paused while the machine is busy" } - prog.Set(line) + tr.Detail("%s · %s read · %d new · %d carried%s", pid, humanBytes(total.ReadBytes+s.ReadBytes), s.Acquired, s.Carried, busy) }) - prog.Stop() total.Acquired += st.Acquired total.Carried += st.Carried total.Symlinks += st.Symlinks total.Skipped += st.Skipped total.Failed += st.Failed total.TotalBytes += st.TotalBytes + total.ReadBytes += st.ReadBytes + total.Processed += st.Processed if err != nil { - fmt.Fprintf(os.Stderr, " %s: %v (continuing)\n", sanitize.Terminal(pid), err) + fmt.Fprintf(w, " %s: %v (continuing)\n", sanitize.Terminal(pid), err) if collectErr == nil { collectErr = err } @@ -195,94 +314,108 @@ func cmdRun(args []string) int { // Detected but empty. Printing a bare "0 artifacts" tells an // analyst nothing about whether the product stores nothing or // the collector is aimed at the wrong path. - fmt.Printf(" %-16s nothing collected — %d manifest path(s) checked, none present on this host\n", pid, st.NotPresent) - fmt.Printf(" %-16s the paths are recorded as NOT_PRESENT in the manifest; `agentdfir inspect ` lists them\n", "") + fmt.Fprintf(w, " %-16s nothing collected — %d manifest path(s) checked, none present on this host\n", pid, st.NotPresent) + fmt.Fprintf(w, " %-16s the paths are recorded as NOT_PRESENT in the manifest; `agentdfir inspect ` lists them\n", "") case st.Carried > 0: - fmt.Printf(" %-16s %d new · %d carried forward · %s\n", pid, st.Acquired, st.Carried, humanBytes(st.TotalBytes)) + fmt.Fprintf(w, " %-16s %d new · %d carried forward · %s\n", pid, st.Acquired, st.Carried, humanBytes(st.TotalBytes)) default: - fmt.Printf(" %-16s %d artifacts · %s\n", pid, st.Acquired, humanBytes(st.TotalBytes)) + fmt.Fprintf(w, " %-16s %d artifacts · %s\n", pid, st.Acquired, humanBytes(st.TotalBytes)) } } + if pauses, paused := gov.Stats(); pauses > 0 { + fmt.Fprintf(w, " paused %d time(s), %s in all, while the machine was busy\n", pauses, elapsed(paused)) + } // The host witness is acquisition, not analysis: what the filesystem and // the repositories said at the moment the evidence was taken. Asking // later would describe a different host. + events := 0 if !*noWitness { - prog.Start(" asking the host about the agent's claims") - gathered := gatherWitness(dest, b, host) - prog.Stop() + tr.SetUnits("witness", float64(total.ReadBytes+(1<<20))) + tr.Begin("witness") + var gathered int + gathered, events = gatherWitness(dest, b, host) if gathered > 0 { - fmt.Printf(" %d claimed write(s) checked against the filesystem\n", gathered) + fmt.Fprintf(w, " %d claimed write(s) checked against the filesystem\n", gathered) } } - if *signKey != "" { - if err := seal.Sign(dest, *signKey); err != nil { - fmt.Fprintln(os.Stderr, "sign error:", err) - return 1 - } - } - prog.Start(" sealing") + tr.Begin("seal") + signing := prepareSigning(b, *signKey, *noSign) roundStats := b.Stats() - sealErr := b.Seal() - prog.Stop() - if sealErr != nil { - fmt.Fprintln(os.Stderr, "seal error:", sealErr) - return 1 + if err := b.Seal(); err != nil { + return fail(1, "seal: %v", err) + } + if _, err := finishSeal(w, dest, signing, b.CaseID(), b.Round(), true); err != nil { + return fail(1, "%v", err) } - fmt.Printf(" Sealed round %d: %d artifacts (%s evidence, %s added to disk), SHA256SUMS written", + tr.End() + fmt.Fprintf(w, " Sealed round %d: %d artifacts (%s evidence, %s added to disk), SHA256SUMS written", b.Round(), total.Acquired+total.Carried, humanBytes(total.TotalBytes), humanBytes(roundStats.StoredBytes)) if collectErr != nil { - fmt.Print(" — partial evidence, see errors above") + fmt.Fprint(w, " — partial evidence, see errors above") } - fmt.Println() - stepDone("collected", time.Since(stepStart)) + fmt.Fprintln(w) - // 3. analyze - stepStart = time.Now() - fmt.Println("\nStep 3/4 Analyze — detections, MCP audit, provenance") - prog.Start(" analyzing") - res, err := analysis.Run(dest, analysis.Options{ + // 3. analyze — or reuse, when nothing it depends on changed. + fmt.Fprintln(w, "\nStep 3/4 Analyze — detections, MCP audit, provenance") + aopts := analysis.Options{ EndpointLogs: endpointLogs, GatewayLog: *gwLog, - RulesDir: *rulesDir, NoBuiltinPacks: *noPacks, Log: prog, + RulesDir: *rulesDir, NoBuiltinPacks: *noPacks, Log: w, RetireExcluded: !*fullPlugins, Stage: func(n, total int, name string) { - prog.Set(fmt.Sprintf("stage %d/%d · %s", n, total, name)) + tr.Detail("stage %d/%d · %s", n, total, name) }, - }) - prog.Stop() - if err != nil { - fmt.Fprintln(os.Stderr, "error:", err) - return 1 } - for _, n := range res.StageNotes { - fmt.Fprintln(os.Stderr, "note:", n) + var findings []schema.Finding + if current, _ := analysis.Current(dest, aopts); current && !*reanalyze { + tr.Skip("analyze") + findings = analysis.LoadFindings(dest) + fmt.Fprintln(w, " Results are current: no evidence, parser or rule changed since the last analysis — reused (--reanalyze to force)") + } else { + if events > 0 { + tr.SetUnits("analyze", float64(events)) + } + tr.Begin("analyze") + res, err := analysis.Run(dest, aopts) + if err != nil { + return fail(1, "%v", err) + } + tr.End() + for _, n := range res.StageNotes { + fmt.Fprintln(w, "note:", n) + } + findings = res.Findings + } + fmt.Fprintf(w, " %s\n", severitySummary(findings)) + exit := exitFor(findings) + if prior.Status == integrity.Failed { + exit = exitPriorFailed } - stepDone("analyzed", time.Since(stepStart)) - fmt.Printf(" %s\n", severitySummary(res.Findings)) if *noServe { - fmt.Printf(" Total %s\n", elapsed(time.Since(runStart))) - printTriageFindings(res.Findings) + stop() + fmt.Printf(" Total %s (%s)\n", elapsed(tr.Elapsed()), tr.Summary()) + printTriageFindings(findings) fmt.Printf("\nPackage: %s (open it later: agentdfir serve %s)\n", dest, dest) - return exitFor(res.Findings) + return exit } // 4. serve - stepStart = time.Now() - fmt.Println("\nStep 4/4 Look — case explorer in your browser") - prog.Start(" loading the explorer") + fmt.Fprintln(w, "\nStep 4/4 Look — case explorer in your browser") + tr.Begin("serve") s, err := serve.Load(dest, serve.Options{Port: *port}) - prog.Stop() if err != nil { - fmt.Fprintln(os.Stderr, "error:", err) - return 1 + return fail(1, "%v", err) } ln, url, err := s.Listen(*port) if err != nil { - fmt.Fprintln(os.Stderr, "error:", err) - return 1 + return fail(1, "%v", err) } - fmt.Printf(" Ready in %s (total %s)\n", elapsed(time.Since(stepStart)), elapsed(time.Since(runStart))) + stop() + fmt.Printf(" Ready — total %s (%s)\n", elapsed(tr.Elapsed()), tr.Summary()) fmt.Printf(" Open %s (127.0.0.1 only · read-only · Ctrl+C to stop)\n", url) fmt.Printf(" Package: %s (later: agentdfir serve %s)\n", dest, dest) + if prior.Status == integrity.Failed { + fmt.Fprintln(os.Stderr, " WARNING: earlier rounds of this case failed their integrity check (see above).") + } if !*noOpen { openBrowser(url) } @@ -290,25 +423,46 @@ func cmdRun(args []string) int { fmt.Fprintln(os.Stderr, "server:", err) return 1 } - return 0 + return exit } +// envInt reads a whole-number setting from the environment (0 when unset +// or not a number). +func envInt(name string) int { + n, err := strconv.Atoi(os.Getenv(name)) + if err != nil { + return 0 + } + return n +} + +// exitDiskFloor is the exit status when a run would have taken the +// evidence volume below its free-space floor. +const exitDiskFloor = 5 + // collectTuning carries the acquisition knobs from the command line. type collectTuning struct { MaxFileMB int64 Jobs int Recollect bool FullContent bool + Pace func(int64) error } // surveyCurrentUser measures what collectCurrentUser would acquire, using // the same manifest, overrides and bounds, so the progress total matches // the work that follows. -func surveyCurrentUser(productID, home, host, osUser string, tune collectTuning) (collector.Survey, error) { +func surveyCurrentUser(productID, home, host, osUser string, tune collectTuning, b *casepkg.Builder) (collector.Survey, error) { man, opts, err := collectPlan(productID, home, host, osUser, tune) if err != nil { return collector.Survey{}, err } + if b != nil { + opts.Unchanged = func(path string, info os.FileInfo) bool { + _, ok := b.Unchanged(path, info) + return ok + } + } return collector.SurveyRun(man, opts), nil } @@ -342,6 +496,7 @@ func collectPlan(productID, home, host, osUser string, tune collectTuning) (*pro ProfileRoot: home, ConfigRoot: configRoot, SystemRoot: "/", Host: host, User: osUser, Product: productID, Jobs: tune.Jobs, Recollect: tune.Recollect, FullContent: tune.FullContent, + Pace: tune.Pace, } if tune.MaxFileMB > 0 { opts.MaxFileBytes = tune.MaxFileMB << 20 @@ -408,28 +563,34 @@ func severitySummary(f []schema.Finding) string { return s } -// gatherWitness parses what has just been collected, asks the host about -// every file the agent claimed to write, and seals the answer into the -// package. Returns how many paths were checked. +// gatherWitness asks the host about every file the agent claimed to write, +// and seals the answer into the package. It returns how many paths were +// checked and how many events the evidence holds. // -// It runs on the package's own evidence rather than on the live profile, so -// it inspects exactly what was preserved, and it runs before Seal so the -// record is covered by SHA256SUMS and the custody chain like anything else. -func gatherWitness(pkg string, b *casepkg.Builder, host string) int { - var events []schema.Event - if _, err := normalize.ParseStream(pkg, func(ev schema.Event) error { - events = append(events, ev) - return nil - }); err != nil { - return 0 +// It reads the claims from the package's own evidence rather than from the +// live profile, so it inspects exactly what was preserved, and it runs +// before Seal so the record is covered by SHA256SUMS and the custody chain +// like anything else. +// +// The events come from the normalized overlay, refreshed here — only the +// transcripts this round added or changed are parsed — instead of a full +// parse of every transcript on every run. The analysis that follows finds +// the overlay current and parses nothing again. +func gatherWitness(pkg string, b *casepkg.Builder, host string) (checked, events int) { + if !normalize.Status(pkg).Current { + if _, err := normalize.Refresh(pkg, normalize.OverlayOptions{}); err != nil { + fmt.Fprintln(os.Stderr, "note: host witness not gathered:", err) + return 0, 0 + } } - rec := witness.Gather(events, host, b.Round(), witness.DefaultLimits) + evs := analysis.LoadEvents(pkg) + rec := witness.Gather(evs, host, b.Round(), witness.DefaultLimits) if len(rec.Files) == 0 && len(rec.Repos) == 0 { - return 0 + return 0, len(evs) } if err := witness.Write(b, rec); err != nil { fmt.Fprintln(os.Stderr, "note: host witness not recorded:", err) - return 0 + return 0, len(evs) } - return len(rec.Files) + return len(rec.Files), len(evs) } diff --git a/internal/cli/run_delta_test.go b/internal/cli/run_delta_test.go new file mode 100644 index 0000000..edeabbf --- /dev/null +++ b/internal/cli/run_delta_test.go @@ -0,0 +1,75 @@ +package cli + +import ( + "path/filepath" + "testing" + + "github.com/efij/AgentDFIR/v3/internal/casepkg" + "github.com/efij/AgentDFIR/v3/internal/integrity" + "github.com/efij/AgentDFIR/v3/internal/seal" + "github.com/efij/AgentDFIR/v3/internal/simulate" +) + +func simHome(t *testing.T) { + t.Helper() + home := t.TempDir() + if err := simulate.OrphanAgent(home); err != nil { + t.Fatal(err) + } + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv("CLAUDE_CONFIG_DIR", "") +} + +// `run --sign` used to sign before sealing, so the signature covered the +// previous round's SHA256SUMS and never verified. It must verify now, and +// so must the machine-key signature every round gets by default. +func TestRunSignaturesVerify(t *testing.T) { + simHome(t) + key := filepath.Join(t.TempDir(), "k") + if err := seal.GenerateKey(key, key+".pub"); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + args []string + }{ + {"explicit key", []string{"--sign", key}}, + {"machine key", nil}, + } { + out := filepath.Join(t.TempDir(), "c.adfir") + args := append([]string{"run", "--no-serve", "--no-witness", "--out", out, "--case-id", "SIGN"}, tc.args...) + if code := Main(args); code != 0 && code != 3 { + t.Fatalf("%s: run exit %d", tc.name, code) + } + res, err := seal.Verify(out, "") + if err != nil || !res.Present || !res.Valid { + t.Fatalf("%s: signature after run: %+v %v", tc.name, res, err) + } + } +} + +// A second run proves the first round before adding its own, and a run +// in which nothing changed reuses the analysis. +func TestSecondRunVerifiesAndReuses(t *testing.T) { + simHome(t) + out := filepath.Join(t.TempDir(), "c.adfir") + for i := 0; i < 2; i++ { + if code := Main([]string{"run", "--no-serve", "--out", out, "--case-id", "TWICE"}); code != 0 && code != 3 { + t.Fatalf("run %d exit %d", i+1, code) + } + } + ci, err := casepkg.ReadCaseInfo(out) + if err != nil { + t.Fatal(err) + } + if len(ci.Rounds) != 2 { + t.Fatalf("rounds = %d, want 2", len(ci.Rounds)) + } + if got := ci.Rounds[1].PriorIntegrity; got != integrity.Verified { + t.Fatalf("round 2 prior integrity %q, want %q", got, integrity.Verified) + } + if p := integrity.CheckPrior(out, true); p.Status != integrity.Verified { + t.Fatalf("after two runs: %+v", p) + } +} diff --git a/internal/collector/collector.go b/internal/collector/collector.go index 19232cc..65aa0ec 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -45,6 +45,16 @@ type Options struct { Recollect bool // re-read every file even when an earlier round already preserved it FullContent bool // collect dependency/VCS subtrees too (--full-plugins) Progress func(Stats) // optional; called after every acquired artifact (UI status lines) + // Unchanged, for SurveyRun only, answers whether an earlier round + // already holds a file (the builder's carry-forward test), so the + // survey counts the bytes that will really be read. + Unchanged func(path string, info os.FileInfo) bool + // Pace, when set, is called by a worker before it reads a file of the + // given size. It may block (to keep the host responsive) and may + // refuse (the disk reached its free-space floor); a refused file is + // recorded as not acquired, with the reason. It never changes which + // files are collected or in what order — only when. + Pace func(size int64) error } // Defaults for size bounds. @@ -88,6 +98,12 @@ type Stats struct { Failed int NotPresent int // manifest paths checked that do not exist on this host TotalBytes int64 // plaintext bytes the package now accounts for + // ReadBytes is what was actually read from disk this round. Carried- + // forward files add to TotalBytes without being read, so a throughput + // computed from TotalBytes is a fiction on every repeat run — it made + // the time-remaining collapse to zero while real reads were still going. + ReadBytes int64 + Processed int // files handled so far, read or carried } // candidate is one discovered file awaiting acquisition. @@ -175,6 +191,12 @@ func newRunner(b *casepkg.Builder, opts Options, st *Stats) *runner { func (r *runner) worker() { defer r.wg.Done() for c := range r.jobs { + if r.opts.Pace != nil { + if err := r.opts.Pace(c.info.Size()); err != nil { + r.results <- result{idx: c.idx, pending: r.b.PrepareRecord(failedRecord(c.rec, err))} + continue + } + } p, err := r.b.PrepareFile(c.path, c.rec) r.results <- result{idx: c.idx, pending: p, err: err} } @@ -218,6 +240,7 @@ func (r *runner) commit(res result) { r.fail(err) return } + r.st.Processed++ switch { case carried: r.st.Carried++ @@ -225,6 +248,7 @@ func (r *runner) commit(res result) { case rec.Status == casepkg.StatusOK: r.st.Acquired++ r.st.TotalBytes += rec.Size + r.st.ReadBytes += rec.Size case rec.Status == casepkg.StatusSymlink: r.st.Symlinks++ case rec.Status == casepkg.StatusNotPresent: @@ -540,9 +564,14 @@ func IngestLooseSessions(b *casepkg.Builder, root string, opts Options) (*Stats, // symlinks, irregular files, over-bound files and policy-excluded subtrees // are counted as skipped, not as work. type Survey struct { - Files int // regular files that would be read + Files int // regular files that would be acquired (read or carried) Bytes int64 // their total size Skipped int // symlinks, irregular, bound-exceeded, policy-excluded + // Carried and ReadBytes are filled when Options.Unchanged is set: + // files an earlier round already holds, and the bytes of the rest — + // what acquisition will actually read. + Carried int + ReadBytes int64 } // SurveyRun measures a collection without performing it. @@ -629,4 +658,18 @@ func (s *surveyor) file(path string) { } s.out.Files++ s.out.Bytes += info.Size() + if s.opts.Unchanged != nil && !s.opts.Recollect && s.opts.Unchanged(path, info) { + s.out.Carried++ + return + } + s.out.ReadBytes += info.Size() +} + +// failedRecord is a file discovery committed to but a worker was not +// allowed to read (Options.Pace refused it). It stays in the manifest as +// not acquired, with the reason — never silently dropped. +func failedRecord(rec casepkg.ArtifactRecord, err error) casepkg.ArtifactRecord { + rec.Status = casepkg.StatusError + rec.Error = err.Error() + return rec } diff --git a/internal/collector/parallel_test.go b/internal/collector/parallel_test.go index f0f6729..62b064e 100644 --- a/internal/collector/parallel_test.go +++ b/internal/collector/parallel_test.go @@ -235,6 +235,25 @@ func TestSecondRoundCarriesForwardUnchangedFiles(t *testing.T) { if res, err := casepkg.Verify(pkg); err != nil || len(res.Problems) != 0 { t.Fatalf("package does not verify after two rounds: %v %v", err, res.Problems) } + + // A third round must carry forward too. A carried record that dropped + // the inode and change time it was judged on left the next round no + // identity to compare, so every other run re-read every file — on a + // real machine, 2.7 GB re-read on alternate runs. + b3, err := casepkg.Reopen(pkg, casepkg.CaseInfo{OperatorOSUser: "tester"}) + if err != nil { + t.Fatal(err) + } + third, err := Run(b3, man, opts) + if err != nil { + t.Fatal(err) + } + if err := b3.Seal(); err != nil { + t.Fatal(err) + } + if third.Carried != first.Acquired || third.Acquired != 0 { + t.Fatalf("third round: carried %d, re-read %d of %d unchanged artifacts", third.Carried, third.Acquired, first.Acquired) + } } // TestAbsentManifestPathsAreRecorded: "we looked here and it was not there" diff --git a/internal/detect/content_scan.go b/internal/detect/content_scan.go index 6c61a85..b6fd7fb 100644 --- a/internal/detect/content_scan.go +++ b/internal/detect/content_scan.go @@ -9,6 +9,7 @@ import ( "unicode" "unicode/utf8" + "github.com/efij/AgentDFIR/v3/internal/artmemo" "github.com/efij/AgentDFIR/v3/internal/casepkg" "github.com/efij/AgentDFIR/v3/internal/schema" ) @@ -231,6 +232,18 @@ type contentFindings struct { secret, injection, unicode, honey []schema.Finding } +// memoContent is contentFindings as the artifact memo stores it. +type memoContent struct { + Secret []schema.Finding `json:"secret,omitempty"` + Injection []schema.Finding `json:"injection,omitempty"` + Unicode []schema.Finding `json:"unicode,omitempty"` + Honey []schema.Finding `json:"honey,omitempty"` +} + +// ContentMemoHits is how many artifacts the last content scan served from +// the memo instead of reading (for tests and progress reporting). +var ContentMemoHits int + // contentScans runs POTENTIAL_SECRET_EXPOSURE, the injection-surface // rules, INVISIBLE_UNICODE_INSTRUCTION and SECRET_ACCESS (honeytokens) // with one read of each artifact. @@ -238,9 +251,20 @@ func contentScans(man *casepkg.Manifest, pkgDir string, honeytokens []string) [] store := casepkg.NewStore(pkgDir, man) cur := man.Current() results := make([]contentFindings, len(cur)) + // Each artifact's result depends only on its bytes, its record and the + // honeytokens, so an unchanged artifact is not read again. + memo := artmemo.Open[memoContent](pkgDir, "detect-content", fmt.Sprintf("%q", honeytokens)) forEachParallel(len(cur), func(i int) { - results[i] = scanArtifactContent(store, cur[i], honeytokens) + if m, ok := memo.Get(cur[i]); ok { + results[i] = contentFindings{m.Secret, m.Injection, m.Unicode, m.Honey} + return + } + r := scanArtifactContent(store, cur[i], honeytokens) + results[i] = r + memo.Put(cur[i], memoContent{r.secret, r.injection, r.unicode, r.honey}) }) + _ = memo.Save() // advisory: a memo that cannot be written costs the next run a re-scan + ContentMemoHits = memo.Hits() var secret, injection, uni, honey []schema.Finding for _, r := range results { secret = append(secret, r.secret...) diff --git a/internal/fingerprint/compute/compute.go b/internal/fingerprint/compute/compute.go new file mode 100644 index 0000000..0eb78a3 --- /dev/null +++ b/internal/fingerprint/compute/compute.go @@ -0,0 +1,143 @@ +// Package compute derives the code fingerprints the fingerprint package +// embeds. It is shared by the generator and the test that proves the +// embedded values are current. +package compute + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" +) + +// Roots are the packages whose dependency closures are fingerprinted. +var Roots = map[string]string{ + "parse": "internal/normalize", + "analysis": "internal/analysis", +} + +// excluded packages never contribute: version changes every release +// without changing what any parser or rule does (SchemaVersion is keyed +// separately), and fingerprint would otherwise hash its own output. +var excluded = []string{"internal/version", "internal/fingerprint"} + +type pkg struct { + ImportPath string + Dir string + Module *struct{ Path string } + EmbedFiles []string +} + +// Compute returns the fingerprint of rel's in-module dependency closure, +// run from the module root. +// +// It must be the same on every OS and architecture, so the closure is +// listed for one fixed target and every non-test .go file in each +// package directory is hashed regardless of build tags, with line endings +// normalized (a Windows checkout with autocrlf must not change it). +func Compute(root, rel string) (string, error) { + cmd := exec.Command("go", "list", "-deps", "-json", "./"+rel) + cmd.Dir = root + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0", "GOFLAGS=-mod=mod") + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("go list %s: %v: %s", rel, err, stderr.String()) + } + modPath, err := modulePath(root) + if err != nil { + return "", err + } + var pkgs []pkg + dec := json.NewDecoder(bytes.NewReader(out)) + for { + var p pkg + if err := dec.Decode(&p); err == io.EOF { + break + } else if err != nil { + return "", err + } + if p.Module == nil || p.Module.Path != modPath { + continue + } + relPkg := strings.TrimPrefix(strings.TrimPrefix(p.ImportPath, modPath), "/") + skip := false + for _, x := range excluded { + if relPkg == x || strings.HasPrefix(relPkg, x+"/") { + skip = true + } + } + if !skip { + p.ImportPath = relPkg + pkgs = append(pkgs, p) + } + } + sort.Slice(pkgs, func(i, j int) bool { return pkgs[i].ImportPath < pkgs[j].ImportPath }) + + h := sha256.New() + for _, p := range pkgs { + files, err := filepath.Glob(filepath.Join(p.Dir, "*.go")) + if err != nil { + return "", err + } + for _, e := range p.EmbedFiles { + files = append(files, filepath.Join(p.Dir, filepath.FromSlash(e))) + } + names := make([]string, 0, len(files)) + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + r, err := filepath.Rel(p.Dir, f) + if err != nil { + return "", err + } + names = append(names, filepath.ToSlash(r)) + } + sort.Strings(names) + for _, n := range names { + data, err := os.ReadFile(filepath.Join(p.Dir, filepath.FromSlash(n))) + if err != nil { + return "", err + } + data = bytes.ReplaceAll(data, []byte("\r\n"), []byte("\n")) + fmt.Fprintf(h, "%s/%s\x00%d\x00", p.ImportPath, n, len(data)) + h.Write(data) + } + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func modulePath(root string) (string, error) { + data, err := os.ReadFile(filepath.Join(root, "go.mod")) + if err != nil { + return "", err + } + for _, line := range strings.Split(string(data), "\n") { + if f := strings.Fields(line); len(f) == 2 && f[0] == "module" { + return f[1], nil + } + } + return "", fmt.Errorf("no module line in go.mod") +} + +// Source renders the generated Go file. +func Source(parse, analysis string) string { + return fmt.Sprintf(`// Code generated by go generate ./internal/fingerprint; DO NOT EDIT. + +package fingerprint + +const ( + parse = %q + analysis = %q +) +`, parse, analysis) +} diff --git a/internal/fingerprint/fingerprint.go b/internal/fingerprint/fingerprint.go new file mode 100644 index 0000000..0a8f342 --- /dev/null +++ b/internal/fingerprint/fingerprint.go @@ -0,0 +1,26 @@ +// Package fingerprint names the code that produced a derived result. +// +// A cached overlay or a stored analysis is only reusable if it came from +// the same parsing and detection code the running binary carries. Keying +// on the release number made every release — including ones that touched +// only the explorer or the installer — throw away every cached parse and +// re-analyze every case from scratch. Keying on the code itself does not: +// a release that leaves the parsers alone keeps every overlay, and one +// that changes a parser invalidates all of them, which is the safe side. +// +// The values are generated (go generate ./internal/fingerprint) and a test +// recomputes them, so a parser or rule change cannot ship under a stale +// fingerprint and quietly keep serving results the new code would not +// produce. +package fingerprint + +//go:generate go run ./gen + +// Parse identifies the code that turns evidence into normalized events: +// every in-module package internal/normalize depends on. +func Parse() string { return parse } + +// Analysis identifies the code that turns normalized events into +// findings: every in-module package internal/analysis depends on, +// embedded rule packs included. It is a superset of Parse. +func Analysis() string { return analysis } diff --git a/internal/fingerprint/fingerprint_test.go b/internal/fingerprint/fingerprint_test.go new file mode 100644 index 0000000..d9f8308 --- /dev/null +++ b/internal/fingerprint/fingerprint_test.go @@ -0,0 +1,34 @@ +package fingerprint + +import ( + "os/exec" + "path/filepath" + "testing" + + "github.com/efij/AgentDFIR/v3/internal/fingerprint/compute" +) + +// TestCurrent fails when a parser, rule or analysis stage changed without +// regenerating the fingerprints. Shipping that would let every cached +// overlay and stored analysis survive a change that alters their content. +func TestCurrent(t *testing.T) { + if _, err := exec.LookPath("go"); err != nil { + t.Skip("go toolchain not on PATH") + } + root, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + for name, want := range map[string]string{"parse": Parse(), "analysis": Analysis()} { + got, err := compute.Compute(root, compute.Roots[name]) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("%s fingerprint is stale: run `go generate ./internal/fingerprint` and commit zz_generated.go", name) + } + } + if Parse() == Analysis() { + t.Error("parse and analysis fingerprints must differ: analysis is a strict superset") + } +} diff --git a/internal/fingerprint/gen/main.go b/internal/fingerprint/gen/main.go new file mode 100644 index 0000000..e313a50 --- /dev/null +++ b/internal/fingerprint/gen/main.go @@ -0,0 +1,35 @@ +// Command gen regenerates internal/fingerprint/zz_generated.go. +// Run it through `go generate ./internal/fingerprint`. +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/efij/AgentDFIR/v3/internal/fingerprint/compute" +) + +func main() { + // go generate runs in the package directory: internal/fingerprint. + root, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + fail(err) + } + parse, err := compute.Compute(root, compute.Roots["parse"]) + if err != nil { + fail(err) + } + analysis, err := compute.Compute(root, compute.Roots["analysis"]) + if err != nil { + fail(err) + } + if err := os.WriteFile("zz_generated.go", []byte(compute.Source(parse, analysis)), 0o644); err != nil { + fail(err) + } +} + +func fail(err error) { + fmt.Fprintln(os.Stderr, "fingerprint:", err) + os.Exit(1) +} diff --git a/internal/fingerprint/zz_generated.go b/internal/fingerprint/zz_generated.go new file mode 100644 index 0000000..1909404 --- /dev/null +++ b/internal/fingerprint/zz_generated.go @@ -0,0 +1,8 @@ +// Code generated by go generate ./internal/fingerprint; DO NOT EDIT. + +package fingerprint + +const ( + parse = "dea9ecafee34f2b3c54ec88676ca673691c061ccadbed29c13ab646125312485" + analysis = "790b815b30c5ba2714b83c89e3ab8fa15303ebdb226e4a7fc2b9f46e5e10daa0" +) diff --git a/internal/gentle/disk_unix.go b/internal/gentle/disk_unix.go new file mode 100644 index 0000000..82e70ac --- /dev/null +++ b/internal/gentle/disk_unix.go @@ -0,0 +1,23 @@ +//go:build !windows + +package gentle + +import "syscall" + +// FreeBytes is the space available to this user on the volume holding path. +func FreeBytes(path string) (int64, error) { + var st syscall.Statfs_t + if err := syscall.Statfs(path, &st); err != nil { + return 0, err + } + return int64(st.Bavail) * int64(st.Bsize), nil +} + +// TotalBytes is the size of the volume holding path. +func TotalBytes(path string) (int64, error) { + var st syscall.Statfs_t + if err := syscall.Statfs(path, &st); err != nil { + return 0, err + } + return int64(st.Blocks) * int64(st.Bsize), nil +} diff --git a/internal/gentle/gentle.go b/internal/gentle/gentle.go new file mode 100644 index 0000000..7ef6586 --- /dev/null +++ b/internal/gentle/gentle.go @@ -0,0 +1,301 @@ +// Package gentle keeps AgentDFIR from ever being the reason a machine is +// slow. +// +// It runs on developers' machines, often while they work, and on fleets +// where an SRE watches every host. A forensic sweep that saturates the CPU, +// stalls the disk, balloons memory or fills the volume is itself an +// incident. So by default the process yields: it lowers its own CPU (and, +// in background mode, I/O) priority, caps its parallelism and its heap, +// paces its reads, pauses while the machine is busy, and refuses to take +// the last of the free disk space. +// +// None of this hides anything. Priority and pacing change how fast the +// work is done, never what is collected or in what order, and every pause +// and refusal is counted and reported. +package gentle + +import ( + "errors" + "fmt" + "runtime" + "runtime/debug" + "sync" + "sync/atomic" + "time" +) + +// Mode is how much the process yields to everything else on the machine. +type Mode string + +const ( + // Gentle lowers CPU priority (nice 10 / below normal): the process + // yields to anything interactive and loses nothing on an idle machine. + Gentle Mode = "gentle" + // Background also puts I/O at the lowest priority (macOS background + // QoS, Linux idle I/O class, Windows background mode). For fleets. + Background Mode = "background" + // Normal leaves priority alone: full speed, for an analyst's own box. + Normal Mode = "normal" +) + +// ParseMode validates a --priority value. +func ParseMode(s string) (Mode, error) { + switch Mode(s) { + case Gentle, Background, Normal: + return Mode(s), nil + case "": + return Gentle, nil + } + return "", fmt.Errorf("unknown priority %q (gentle | background | normal)", s) +} + +// Settings is what Apply put in place. +type Settings struct { + Mode Mode + Priority string // what the OS was asked for, or why it could not be + Workers int + GoMaxProcs int + MemoryLimit int64 +} + +// Apply lowers this process's priority for the mode and caps its +// parallelism and heap. jobs and memLimit are the operator's explicit +// values (0: the mode's default). +func Apply(m Mode, jobs int, memLimit int64) Settings { + s := Settings{Mode: m} + if err := setPriority(m); err != nil { + s.Priority = "unchanged (" + err.Error() + ")" + } else { + s.Priority = priorityName(m) + } + s.Workers = Workers(m, jobs) + if m != Normal { + // The Go scheduler too: half the cores, at least two. + s.GoMaxProcs = max(2, runtime.NumCPU()/2) + runtime.GOMAXPROCS(s.GoMaxProcs) + } else { + s.GoMaxProcs = runtime.GOMAXPROCS(0) + } + if memLimit == 0 && m != Normal { + memLimit = DefaultMemoryLimit() + } + if memLimit > 0 { + debug.SetMemoryLimit(memLimit) + s.MemoryLimit = memLimit + } + return s +} + +// Workers is the acquisition parallelism for a mode: an explicit value +// wins; normal uses up to 8 as before; the yielding modes use half the +// cores, at most 4. +func Workers(m Mode, jobs int) int { + if jobs > 0 { + return jobs + } + if m == Normal { + return min(runtime.NumCPU(), 8) + } + return max(1, min(runtime.NumCPU()/2, 4)) +} + +// DefaultMemoryLimit is a soft heap goal: a quarter of physical memory, +// between 512 MiB and 4 GiB. It is soft — the Go runtime collects harder +// as the heap approaches it and caps its own GC effort — so it steers +// memory without risking a failure. +func DefaultMemoryLimit() int64 { + const lo, hi = 512 << 20, 4 << 30 + total := totalMemory() + if total <= 0 { + return 2 << 30 + } + return min(hi, max(lo, total/4)) +} + +// ErrDiskFloor is returned by Pace once the volume holding the evidence +// reaches its free-space floor. +var ErrDiskFloor = errors.New("stopped: free disk space reached the safety floor") + +// Governor paces acquisition: a read-rate cap, a pause while the machine +// is busy, and the disk floor. +type Governor struct { + rate float64 // bytes/s, 0 = unlimited + floor int64 + diskPath string + loadHigh float64 + loadLow float64 + checkDisk int64 + + mu sync.Mutex + tokens float64 + last time.Time + sinceDk int64 + full bool + + paused atomic.Bool + pauses atomic.Int64 + pausedNano atomic.Int64 + stop chan struct{} + stopOnce sync.Once +} + +// GovernorOptions configure a Governor. +type GovernorOptions struct { + ReadMBps int // 0 = unlimited + DiskPath string // the volume the evidence is written to + FloorByte int64 // free space never to go below (0 = none) + Adaptive bool // pause while the machine is busy +} + +// NewGovernor starts a governor. Close stops its sampler. +func NewGovernor(o GovernorOptions) *Governor { + g := &Governor{ + rate: float64(o.ReadMBps) * (1 << 20), floor: o.FloorByte, diskPath: o.DiskPath, + loadHigh: 0.8 * float64(runtime.NumCPU()), loadLow: 0.6 * float64(runtime.NumCPU()), + checkDisk: 64 << 20, last: time.Now(), stop: make(chan struct{}), + } + g.tokens = g.rate // one second of burst + if o.Adaptive { + go g.sample() + } + return g +} + +// Close stops the load sampler. +func (g *Governor) Close() { g.stopOnce.Do(func() { close(g.stop) }) } + +// Busy reports whether the governor is holding work back right now. +func (g *Governor) Busy() bool { return g.paused.Load() } + +// Stats reports how often and how long the governor paused work. +func (g *Governor) Stats() (pauses int64, paused time.Duration) { + return g.pauses.Load(), time.Duration(g.pausedNano.Load()) +} + +func (g *Governor) sample() { + t := time.NewTicker(2 * time.Second) + defer t.Stop() + for { + select { + case <-g.stop: + return + case <-t.C: + } + l, ok := loadAverage() + pressure := memoryPressure() + switch { + case (ok && l > g.loadHigh) || pressure: + if !g.paused.Swap(true) { + g.pauses.Add(1) + } + case (!ok || l < g.loadLow) && !pressure: + g.paused.Store(false) + } + } +} + +// maxPause bounds one wait: on a machine that is always busy the work +// still finishes, at low priority, instead of never. +const maxPause = 30 * time.Second + +// Pace is collector.Options.Pace: it blocks while the machine is busy and +// while the read-rate budget is spent, and refuses once the disk floor is +// reached. +func (g *Governor) Pace(size int64) error { + if g.paused.Load() { + start := time.Now() + for g.paused.Load() && time.Since(start) < maxPause { + select { + case <-g.stop: + return nil + case <-time.After(250 * time.Millisecond): + } + } + g.pausedNano.Add(int64(time.Since(start))) + } + g.mu.Lock() + if g.full { + g.mu.Unlock() + return ErrDiskFloor + } + if g.floor > 0 && g.diskPath != "" { + g.sinceDk += size + if g.sinceDk >= g.checkDisk { + g.sinceDk = 0 + if free, err := FreeBytes(g.diskPath); err == nil && free < g.floor { + g.full = true + g.mu.Unlock() + return ErrDiskFloor + } + } + } + var wait time.Duration + if g.rate > 0 { + now := time.Now() + g.tokens = min(g.rate, g.tokens+now.Sub(g.last).Seconds()*g.rate) + g.last = now + g.tokens -= float64(size) + if g.tokens < 0 { + wait = time.Duration(-g.tokens / g.rate * float64(time.Second)) + } + } + g.mu.Unlock() + if wait > 0 { + time.Sleep(wait) + } + return nil +} + +// Floor is the free space never to go below on a volume: 2 GiB, raised to +// 1% of a large volume but never past 5 GiB. It protects the machine from +// this tool, not from the state it was found in: a volume that is already +// fuller than a percentage-based rule would like still takes a small +// round, and one that is truly nearly full does not. minGB overrides it +// when positive. +func Floor(path string, minGB int) int64 { + if minGB > 0 { + return int64(minGB) << 30 + } + floor := int64(2 << 30) + if total, err := TotalBytes(path); err == nil { + floor = max(floor, min(total/100, 5<<30)) + } + return floor +} + +// Preflight refuses to start when writing need more bytes to the volume +// holding path would leave less than the floor. +func Preflight(path string, need, floor int64) error { + free, err := FreeBytes(path) + if err != nil { + return nil // cannot tell; the in-run check still applies + } + if free-need < floor { + return fmt.Errorf("not enough free disk space on the evidence volume: %s free, about %s needed, and %s must stay free (--min-free-gb to change)", + HumanBytes(free), HumanBytes(need), HumanBytes(floor)) + } + return nil +} + +// HumanBytes renders a byte count. +func HumanBytes(n int64) string { + switch { + case n >= 1<<30: + return fmt.Sprintf("%.1f GB", float64(n)/(1<<30)) + case n >= 1<<20: + return fmt.Sprintf("%.0f MB", float64(n)/(1<<20)) + case n >= 1<<10: + return fmt.Sprintf("%.0f KB", float64(n)/(1<<10)) + } + return fmt.Sprintf("%d B", n) +} + +func priorityName(m Mode) string { + switch m { + case Background: + return "background (lowest CPU and I/O priority)" + case Gentle: + return "lowered CPU priority" + } + return "normal" +} diff --git a/internal/gentle/gentle_test.go b/internal/gentle/gentle_test.go new file mode 100644 index 0000000..2fa9a33 --- /dev/null +++ b/internal/gentle/gentle_test.go @@ -0,0 +1,141 @@ +package gentle + +import ( + "errors" + "runtime" + "testing" + "time" +) + +// The host probes must return real numbers on the platforms that have +// them, or the governor and the disk floor silently do nothing. +func TestHostProbes(t *testing.T) { + if runtime.GOOS != "windows" { + l, ok := loadAverage() + if !ok || l < 0 { + t.Fatalf("load average unavailable: %v %v", l, ok) + } + } + if m := totalMemory(); m < 256<<20 { + t.Fatalf("total memory %d: implausible", m) + } + dir := t.TempDir() + free, err := FreeBytes(dir) + if err != nil || free <= 0 { + t.Fatalf("free bytes: %d %v", free, err) + } + total, err := TotalBytes(dir) + if err != nil || total < free { + t.Fatalf("total bytes %d < free %d (%v)", total, free, err) + } +} + +func TestMemoryLimitBounds(t *testing.T) { + l := DefaultMemoryLimit() + if l < 512<<20 || l > 4<<30 { + t.Fatalf("default memory limit %d outside [512 MiB, 4 GiB]", l) + } +} + +func TestWorkers(t *testing.T) { + if got := Workers(Gentle, 3); got != 3 { + t.Fatalf("explicit jobs ignored: %d", got) + } + if got := Workers(Gentle, 0); got < 1 || got > 4 || got > max(1, runtime.NumCPU()/2) { + t.Fatalf("gentle workers %d: want 1..min(NumCPU/2, 4)", got) + } + if got := Workers(Normal, 0); got != min(runtime.NumCPU(), 8) { + t.Fatalf("normal workers %d", got) + } +} + +// The read-rate cap holds: 3 MB at 1 MB/s (with 1 s of burst) takes ~2 s. +func TestPaceCapsReadRate(t *testing.T) { + g := NewGovernor(GovernorOptions{ReadMBps: 1}) + defer g.Close() + start := time.Now() + for i := 0; i < 12; i++ { + if err := g.Pace(256 << 10); err != nil { + t.Fatal(err) + } + } + if d := time.Since(start); d < 1500*time.Millisecond || d > 3500*time.Millisecond { + t.Fatalf("3 MB at 1 MB/s took %v, want ~2s", d) + } +} + +// Once free space is below the floor, every further read is refused. +func TestPaceStopsAtTheDiskFloor(t *testing.T) { + dir := t.TempDir() + free, err := FreeBytes(dir) + if err != nil { + t.Skip(err) + } + g := NewGovernor(GovernorOptions{DiskPath: dir, FloorByte: free + 1<<40}) + defer g.Close() + var got error + for i := 0; i < 4 && got == nil; i++ { + got = g.Pace(64 << 20) + } + if !errors.Is(got, ErrDiskFloor) { + t.Fatalf("pace past the floor: %v, want ErrDiskFloor", got) + } + if err := g.Pace(1); !errors.Is(err, ErrDiskFloor) { + t.Fatalf("after the floor: %v, want every read refused", err) + } +} + +func TestPreflight(t *testing.T) { + dir := t.TempDir() + free, err := FreeBytes(dir) + if err != nil { + t.Skip(err) + } + if err := Preflight(dir, 1<<20, 0); err != nil { + t.Fatalf("1 MB with no floor refused: %v", err) + } + if err := Preflight(dir, free, 1<<30); err == nil { + t.Fatal("a run needing all free space plus a floor was allowed") + } +} + +// A busy machine pauses work, but never forever. +func TestPauseIsBounded(t *testing.T) { + g := NewGovernor(GovernorOptions{}) + defer g.Close() + g.paused.Store(true) + go func() { time.Sleep(600 * time.Millisecond); g.paused.Store(false) }() + start := time.Now() + if err := g.Pace(1); err != nil { + t.Fatal(err) + } + if d := time.Since(start); d < 500*time.Millisecond || d > 2*time.Second { + t.Fatalf("paused %v, want until the machine was idle (~0.6s)", d) + } + if _, paused := g.Stats(); paused < 500*time.Millisecond { + t.Fatalf("pause not accounted: %v", paused) + } +} + +func TestParseMode(t *testing.T) { + for in, want := range map[string]Mode{"": Gentle, "gentle": Gentle, "background": Background, "normal": Normal} { + if got, err := ParseMode(in); err != nil || got != want { + t.Errorf("ParseMode(%q) = %q, %v", in, got, err) + } + } + if _, err := ParseMode("turbo"); err == nil { + t.Error("unknown mode accepted") + } +} + +// The default floor never demands more than 5 GiB: a developer machine +// that is already fairly full must still be able to take a small round. +func TestFloorIsBounded(t *testing.T) { + f := Floor(t.TempDir(), 0) + if f < 2<<30 || f > 5<<30 { + t.Fatalf("floor %d outside [2 GiB, 5 GiB]", f) + } + if got := Floor(t.TempDir(), 7); got != 7<<30 { + t.Fatalf("--min-free-gb 7 gave %d", got) + } +} diff --git a/internal/gentle/sys_darwin.go b/internal/gentle/sys_darwin.go new file mode 100644 index 0000000..92599f4 --- /dev/null +++ b/internal/gentle/sys_darwin.go @@ -0,0 +1,65 @@ +package gentle + +import ( + "encoding/binary" + "syscall" +) + +const ( + prioDarwinProcess = 4 // PRIO_DARWIN_PROCESS + prioDarwinBG = 0x1000 // PRIO_DARWIN_BG: background CPU and throttled I/O +) + +func setPriority(m Mode) error { + switch m { + case Gentle: + return syscall.Setpriority(syscall.PRIO_PROCESS, 0, 10) + case Background: + if err := syscall.Setpriority(syscall.PRIO_PROCESS, 0, 10); err != nil { + return err + } + return syscall.Setpriority(prioDarwinProcess, 0, prioDarwinBG) + } + return nil +} + +// sysctlRaw returns a sysctl's bytes. syscall.Sysctl drops one trailing +// NUL, which in a binary value is a zero high byte; pad it back. +func sysctlRaw(name string, size int) ([]byte, bool) { + s, err := syscall.Sysctl(name) + if err != nil { + return nil, false + } + b := []byte(s) + for len(b) < size { + b = append(b, 0) + } + return b, len(b) == size +} + +// loadAverage is the 1-minute load average (struct loadavg: three +// fixed-point values and their scale). +func loadAverage() (float64, bool) { + b, ok := sysctlRaw("vm.loadavg", 24) + if !ok { + return 0, false + } + l := binary.LittleEndian.Uint32(b[0:4]) + scale := binary.LittleEndian.Uint64(b[16:24]) + if scale == 0 { + return 0, false + } + return float64(l) / float64(scale), true +} + +func totalMemory() int64 { + b, ok := sysctlRaw("hw.memsize", 8) + if !ok { + return 0 + } + return int64(binary.LittleEndian.Uint64(b)) +} + +// memoryPressure is left to the load average on macOS: the kernel's own +// pressure level needs a dispatch source, not a syscall. +func memoryPressure() bool { return false } diff --git a/internal/gentle/sys_linux.go b/internal/gentle/sys_linux.go new file mode 100644 index 0000000..874c82a --- /dev/null +++ b/internal/gentle/sys_linux.go @@ -0,0 +1,91 @@ +package gentle + +import ( + "os" + "strconv" + "strings" + "syscall" +) + +const ( + ioprioWhoProcess = 1 + ioprioClassBE = 2 + ioprioClassIdle = 3 + ioprioShift = 13 +) + +// setPriority applies to every thread: on Linux, nice and I/O priority are +// per thread, and the Go runtime has already started several. Threads +// created later inherit from the one that creates them. +func setPriority(m Mode) error { + if m == Normal { + return nil + } + io := uintptr(ioprioClassBE< 0 && avail > 0 && avail*10 < total +} diff --git a/internal/gentle/sys_windows.go b/internal/gentle/sys_windows.go new file mode 100644 index 0000000..46456fd --- /dev/null +++ b/internal/gentle/sys_windows.go @@ -0,0 +1,100 @@ +package gentle + +import ( + "syscall" + "unsafe" +) + +var ( + kernel32 = syscall.NewLazyDLL("kernel32.dll") + procSetPriorityClass = kernel32.NewProc("SetPriorityClass") + procGlobalMemoryStatus = kernel32.NewProc("GlobalMemoryStatusEx") + procGetDiskFreeSpaceExW = kernel32.NewProc("GetDiskFreeSpaceExW") +) + +const ( + belowNormalPriorityClass = 0x00004000 + processModeBackgroundBegin = 0x00100000 +) + +func setPriority(m Mode) error { + cls := uintptr(0) + switch m { + case Gentle: + cls = belowNormalPriorityClass + case Background: + cls = processModeBackgroundBegin + default: + return nil + } + h, err := syscall.GetCurrentProcess() + if err != nil { + return err + } + if r, _, e := procSetPriorityClass.Call(uintptr(h), cls); r == 0 { + return e + } + return nil +} + +type memoryStatusEx struct { + Length uint32 + MemoryLoad uint32 + TotalPhys uint64 + AvailPhys uint64 + TotalPageFile uint64 + AvailPageFile uint64 + TotalVirtual uint64 + AvailVirtual uint64 + AvailExtendedVirtual uint64 +} + +func memStatus() (memoryStatusEx, bool) { + var m memoryStatusEx + m.Length = uint32(unsafe.Sizeof(m)) + r, _, _ := procGlobalMemoryStatus.Call(uintptr(unsafe.Pointer(&m))) + return m, r != 0 +} + +// loadAverage does not exist on Windows; memory load stands in for it +// (see memoryPressure). +func loadAverage() (float64, bool) { return 0, false } + +func totalMemory() int64 { + if m, ok := memStatus(); ok { + return int64(m.TotalPhys) + } + return 0 +} + +// memoryPressure: 90% or more of physical memory in use. +func memoryPressure() bool { + m, ok := memStatus() + return ok && m.MemoryLoad >= 90 +} + +func diskSpace(path string) (free, total int64, err error) { + p, err := syscall.UTF16PtrFromString(path) + if err != nil { + return 0, 0, err + } + var avail, tot, totalFree uint64 + r, _, e := procGetDiskFreeSpaceExW.Call(uintptr(unsafe.Pointer(p)), + uintptr(unsafe.Pointer(&avail)), uintptr(unsafe.Pointer(&tot)), uintptr(unsafe.Pointer(&totalFree))) + if r == 0 { + return 0, 0, e + } + return int64(avail), int64(tot), nil +} + +// FreeBytes is the space available to this user on the volume holding path. +func FreeBytes(path string) (int64, error) { + f, _, err := diskSpace(path) + return f, err +} + +// TotalBytes is the size of the volume holding path. +func TotalBytes(path string) (int64, error) { + _, t, err := diskSpace(path) + return t, err +} diff --git a/internal/integrity/integrity.go b/internal/integrity/integrity.go new file mode 100644 index 0000000..c2ec341 --- /dev/null +++ b/internal/integrity/integrity.go @@ -0,0 +1,198 @@ +// Package integrity proves a case's earlier rounds are intact before a new +// round is added to it, and signs and anchors each round once it is +// sealed. +// +// The order matters. A new round's seal re-states every earlier blob's +// hash, and a new signature covers that seal. Signing without checking +// first would put this machine's signature on whatever someone left in the +// case since the last run. So before collecting, the previous signature is +// checked against the current SHA256SUMS, the seal is checked against the +// anchor recorded outside the case, and the sealed files, both hash chains +// and every blob's presence and length are verified. A full re-hash of +// every blob is available on request; it is not needed for this guarantee, +// because the new seal carries the old hashes forward rather than +// re-deriving them, so a swapped blob stays detectable by any later +// verify. +package integrity + +import ( + "fmt" + "path/filepath" + + "github.com/efij/AgentDFIR/v3/internal/casepkg" + "github.com/efij/AgentDFIR/v3/internal/seal" + "github.com/efij/AgentDFIR/v3/internal/store" +) + +// Status values recorded in a round's prior_integrity. +const ( + Verified = "verified" // signature and anchor matched, quick verify clean + Unsigned = "unsigned" // nothing to check a signature against; quick verify clean + Failed = "FAILED" +) + +// Prior is what checking a case's earlier rounds found. +type Prior struct { + Status string + Problems []string + Signer string // public key the previous round was signed with, if any + Anchored bool // the anchor log had a record for this case and it matched + Depth string // "quick" or "full" + Notes []string +} + +// CheckPrior verifies an existing case before a round is added. full also +// re-hashes every stored blob. +func CheckPrior(pkg string, full bool) Prior { + p := Prior{Depth: "quick"} + fail := func(format string, a ...any) { p.Problems = append(p.Problems, fmt.Sprintf(format, a...)) } + + ci, err := casepkg.ReadCaseInfo(pkg) + if err != nil { + fail("case.json: %v", err) + p.Status = Failed + return p + } + expected := "" + for _, r := range ci.Rounds { + if r.Signer != "" { + expected = r.Signer + } + } + + sig, err := seal.Verify(pkg, expected) + switch { + case err != nil: + fail("signature: %v", err) + case sig.Present && !sig.Valid && expected == "": + // No round recorded a signer, so this SEAL.sig predates signed + // rounds. `run --sign` in those versions signed before sealing, so + // its signature never matched the seal it sat next to. It proves + // nothing either way; it is reported, not trusted, and not counted + // as tampering. + p.Notes = append(p.Notes, "SEAL.sig from an earlier version does not match the seal ("+sig.Reason+"); not trusted") + case sig.Present && !sig.Valid: + fail("signature: %s", sig.Reason) + case !sig.Present && expected != "": + fail("signature: %s is missing, but an earlier round was sealed for key %s", seal.SigFile, seal.Fingerprint(expected)) + case sig.Present && sig.Valid: + p.Signer = sig.PublicKey + } + + abs, _ := filepath.Abs(pkg) + if a, err := store.LatestAnchor(ci.CaseID, abs); err != nil { + fail("anchor log: %v", err) + } else if a != nil { + digest, err := seal.Digest(pkg) + switch { + case err != nil: + fail("seal: %v", err) + case digest != a.SealsDigest: + fail("seal: SHA256SUMS does not match the anchor recorded when round %d was sealed (%s) — the case was changed or rolled back since", a.Round, a.TimeUTC) + default: + p.Anchored = true + } + } + + verify := casepkg.VerifyQuick + if full { + verify, p.Depth = casepkg.Verify, "full" + } + if res, err := verify(pkg); err != nil { + fail("verify: %v", err) + } else { + for _, pr := range res.Problems { + fail("verify: %s", pr) + } + } + + switch { + case len(p.Problems) > 0: + p.Status = Failed + case p.Signer != "": + p.Status = Verified + default: + p.Status = Unsigned + } + return p +} + +// Apply records the result on the round about to be collected. +func (p Prior) Apply(b *casepkg.Builder) { + b.PriorIntegrity = p.Status + b.PriorProblems = p.Problems + _ = b.Log("prior_integrity_checked", map[string]any{ + "status": p.Status, "problems": p.Problems, "depth": p.Depth, + "anchored": p.Anchored, "signer": p.Signer, + }) +} + +// Signing is how a round will be signed: the key file, its public half, +// and why there is none when there is none. +type Signing struct { + KeyPath string + Public string + Skipped string +} + +// Prepare picks the key a round will be signed with: an explicit key +// file, else this machine's key (created on first use). noSign turns +// signing off and says so. Call it before sealing and set b.Signer to the +// Public half, so the round records which key it was sealed for. +func Prepare(explicitKey string, noSign bool) Signing { + if noSign { + return Signing{Skipped: "--no-sign"} + } + key := explicitKey + if key == "" { + k, err := store.MachineKey() + if err != nil { + return Signing{Skipped: "no machine key: " + err.Error()} + } + key = k + } + pub, err := seal.PublicKeyHex(key) + if err != nil { + return Signing{Skipped: "signing key: " + err.Error()} + } + return Signing{KeyPath: key, Public: pub} +} + +// Sealed is what Finish did. +type Sealed struct { + Digest string // sha256 of SHA256SUMS + Signed bool + Anchored bool + Notes []string +} + +// Finish signs a freshly sealed package and records its anchor. It runs +// after Seal, because the signature covers the SHA256SUMS Seal writes. +// anchor is false for runs that must leave nothing on the machine. +func Finish(pkg string, s Signing, caseID string, round int, anchor bool) (Sealed, error) { + var out Sealed + if s.KeyPath != "" { + if err := seal.Sign(pkg, s.KeyPath); err != nil { + return out, fmt.Errorf("sign: %w", err) + } + out.Signed = true + } else if s.Skipped != "" { + out.Notes = append(out.Notes, "round not signed ("+s.Skipped+")") + } + digest, err := seal.Digest(pkg) + if err != nil { + return out, err + } + out.Digest = digest + if anchor { + abs, _ := filepath.Abs(pkg) + if err := store.RecordAnchor(store.Anchor{ + CaseID: caseID, Round: round, SealsDigest: digest, Signer: s.Public, Package: abs, + }); err != nil { + out.Notes = append(out.Notes, "anchor not recorded: "+err.Error()) + } else { + out.Anchored = true + } + } + return out, nil +} diff --git a/internal/integrity/integrity_test.go b/internal/integrity/integrity_test.go new file mode 100644 index 0000000..c3b1b4f --- /dev/null +++ b/internal/integrity/integrity_test.go @@ -0,0 +1,280 @@ +package integrity + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/efij/AgentDFIR/v3/internal/casepkg" + "github.com/efij/AgentDFIR/v3/internal/seal" + "github.com/efij/AgentDFIR/v3/internal/store" +) + +func setHome(t *testing.T) { + t.Helper() + dir := filepath.Join(t.TempDir(), "home") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + t.Setenv(store.EnvHome, dir) +} + +// round seals one round with a file of the given content, signed with +// explicitKey (or the machine key when empty). +func round(t *testing.T, pkg, content, explicitKey string) { + t.Helper() + src := filepath.Join(t.TempDir(), "t.jsonl") + if err := os.WriteFile(src, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + var b *casepkg.Builder + var err error + if _, statErr := os.Stat(pkg); statErr == nil { + b, err = casepkg.Reopen(pkg, casepkg.CaseInfo{OperatorOSUser: "t"}) + } else { + b, err = casepkg.New(pkg, "CASE-1", casepkg.CaseInfo{OperatorOSUser: "t"}) + } + if err != nil { + t.Fatal(err) + } + defer b.Close() + if err := b.IngestFile(src, casepkg.ArtifactRecord{SourcePath: src, LogicalPath: "t.jsonl", ArtifactType: "transcript"}); err != nil { + t.Fatal(err) + } + s := Prepare(explicitKey, false) + if s.KeyPath == "" { + t.Fatalf("no signing key: %s", s.Skipped) + } + b.Signer = s.Public + if err := b.Seal(); err != nil { + t.Fatal(err) + } + res, err := Finish(pkg, s, "CASE-1", b.Round(), true) + if err != nil { + t.Fatal(err) + } + if !res.Signed || !res.Anchored { + t.Fatalf("round not signed/anchored: %+v", res) + } +} + +func newCase(t *testing.T) string { + setHome(t) + pkg := filepath.Join(t.TempDir(), "c.adfir") + round(t, pkg, "one\n", "") + return pkg +} + +func TestCleanCaseVerifies(t *testing.T) { + pkg := newCase(t) + p := CheckPrior(pkg, false) + if p.Status != Verified || !p.Anchored || len(p.Problems) > 0 { + t.Fatalf("clean signed case: %+v", p) + } + if full := CheckPrior(pkg, true); full.Status != Verified || full.Depth != "full" { + t.Fatalf("full check: %+v", full) + } +} + +// Someone rewrites a sealed file and regenerates SHA256SUMS to match. +// Every hash in the package agrees again; the signature and the anchor +// do not. +func TestRewrittenSealFails(t *testing.T) { + pkg := newCase(t) + casePath := filepath.Join(pkg, "case.json") + data, err := os.ReadFile(casePath) + if err != nil { + t.Fatal(err) + } + _ = os.Chmod(casePath, 0o600) + forged := strings.Replace(string(data), `"operator_os_user": "t"`, `"operator_os_user": "someone-else"`, 1) + if forged == string(data) { + t.Fatal("forgery did not change case.json") + } + if err := os.WriteFile(casePath, []byte(forged), 0o600); err != nil { + t.Fatal(err) + } + resealSums(t, pkg) + p := CheckPrior(pkg, false) + if p.Status != Failed { + t.Fatalf("rewritten case passed: %+v", p) + } + if !contains(p.Problems, "signature") || !contains(p.Problems, "anchor") { + t.Fatalf("want both the signature and the anchor to object: %v", p.Problems) + } +} + +// The forger also re-signs, with their own key. The case recorded which +// key its rounds were sealed for, so that fails too. +func TestResignedWithAnotherKeyFails(t *testing.T) { + pkg := newCase(t) + other := filepath.Join(t.TempDir(), "other.key") + if err := seal.GenerateKey(other, other+".pub"); err != nil { + t.Fatal(err) + } + if err := seal.Sign(pkg, other); err != nil { + t.Fatal(err) + } + if p := CheckPrior(pkg, false); p.Status != Failed || !contains(p.Problems, "pinned key") { + t.Fatalf("re-signed with another key: %+v", p) + } +} + +// A case restored from an older copy verifies internally — every file +// matches its own seal — but not against the anchor of the newer round. +func TestRollbackToAnEarlierRoundFails(t *testing.T) { + pkg := newCase(t) + backup := filepath.Join(t.TempDir(), "backup") + if err := os.CopyFS(backup, os.DirFS(pkg)); err != nil { + t.Fatal(err) + } + round(t, pkg, "one\ntwo\n", "") + if p := CheckPrior(pkg, false); p.Status != Verified { + t.Fatalf("after round 2: %+v", p) + } + chmodAll(t, pkg) + if err := os.RemoveAll(pkg); err != nil { + t.Fatal(err) + } + if err := os.CopyFS(pkg, os.DirFS(backup)); err != nil { + t.Fatal(err) + } + if p := CheckPrior(pkg, false); p.Status != Failed || !contains(p.Problems, "anchor") { + t.Fatalf("rolled-back case: %+v", p) + } +} + +// A blob swapped on disk is caught by the quick check's size test when +// the size changes, and by the full check always. +func TestSwappedBlobFails(t *testing.T) { + pkg := newCase(t) + raws, _ := filepath.Glob(filepath.Join(pkg, "raw", "*")) + if len(raws) == 0 { + t.Fatal("no blob") + } + info, _ := os.Stat(raws[0]) + _ = os.Chmod(raws[0], 0o600) + junk := make([]byte, info.Size()) + for i := range junk { + junk[i] = 'x' + } + if err := os.WriteFile(raws[0], junk, 0o600); err != nil { + t.Fatal(err) + } + if p := CheckPrior(pkg, true); p.Status != Failed { + t.Fatalf("same-size swapped blob passed the full check: %+v", p) + } +} + +// Before rounds were signed, `run --sign` signed before sealing, so its +// SEAL.sig never matched. That is reported and not trusted, but it is not +// evidence of tampering. +func TestLegacyMismatchedSignatureIsANoteNotAFailure(t *testing.T) { + setHome(t) + pkg := filepath.Join(t.TempDir(), "legacy.adfir") + src := filepath.Join(t.TempDir(), "t.jsonl") + _ = os.WriteFile(src, []byte("x\n"), 0o600) + b, err := casepkg.New(pkg, "LEGACY", casepkg.CaseInfo{OperatorOSUser: "t"}) + if err != nil { + t.Fatal(err) + } + _ = b.IngestFile(src, casepkg.ArtifactRecord{SourcePath: src, LogicalPath: "t.jsonl"}) + if err := b.Seal(); err != nil { + t.Fatal(err) + } + key := filepath.Join(t.TempDir(), "k") + _ = seal.GenerateKey(key, key+".pub") + if err := seal.Sign(pkg, key); err != nil { + t.Fatal(err) + } + // Rewrite SHA256SUMS after signing, as the old ordering did (the + // round's seal was written after the signature). + sums := filepath.Join(pkg, "SHA256SUMS") + _ = os.Chmod(sums, 0o600) + data, _ := os.ReadFile(sums) + if err := os.WriteFile(sums, append([]byte("\n"), data...), 0o600); err != nil { + t.Fatal(err) + } + if sig, _ := seal.Verify(pkg, ""); sig.Valid { + t.Fatal("setup: the legacy signature still verifies") + } + p := CheckPrior(pkg, false) + if p.Status == Failed || len(p.Notes) == 0 { + t.Fatalf("legacy mismatch: %+v", p) + } +} + +// Each round archives the signature of the round before, and the new +// seal covers it: no signature is ever lost. +func TestSignaturesAreArchivedAndSealed(t *testing.T) { + pkg := newCase(t) + first, err := os.ReadFile(filepath.Join(pkg, seal.SigFile)) + if err != nil { + t.Fatal(err) + } + round(t, pkg, "one\ntwo\n", "") + archived, err := os.ReadFile(filepath.Join(pkg, "seals", seal.SigFile+".1")) + if err != nil { + t.Fatalf("round 1 signature not archived: %v", err) + } + if string(archived) != string(first) { + t.Fatal("archived signature differs from round 1's") + } + sums, _ := os.ReadFile(filepath.Join(pkg, "SHA256SUMS")) + if !strings.Contains(string(sums), "seals/"+seal.SigFile+".1") { + t.Fatal("round 2's seal does not cover round 1's signature") + } + ci, err := casepkg.ReadCaseInfo(pkg) + if err != nil { + t.Fatal(err) + } + r2 := ci.Rounds[len(ci.Rounds)-1] + if r2.PrevSealSHA256 == "" || r2.Signer == "" { + t.Fatalf("round 2 record lacks its links: %+v", r2) + } +} + +// resealSums regenerates SHA256SUMS over the sealed zone — what someone +// covering their tracks would do. +func resealSums(t *testing.T, pkg string) { + t.Helper() + var lines []string + err := filepath.WalkDir(pkg, func(p string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + rel, _ := filepath.Rel(pkg, p) + rel = filepath.ToSlash(rel) + if rel == "SHA256SUMS" || rel == seal.SigFile || strings.HasPrefix(rel, "normalized/") || strings.HasPrefix(rel, ".") { + return nil + } + sum, err := fileSHA(p) + if err != nil { + return err + } + lines = append(lines, sum+" "+rel) + return nil + }) + if err != nil { + t.Fatal(err) + } + sums := filepath.Join(pkg, "SHA256SUMS") + _ = os.Chmod(sums, 0o600) + if err := os.WriteFile(sums, []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil { + t.Fatal(err) + } +} + +func chmodAll(t *testing.T, dir string) { + _ = filepath.WalkDir(dir, func(p string, _ os.DirEntry, _ error) error { return os.Chmod(p, 0o700) }) +} + +func contains(list []string, sub string) bool { + for _, s := range list { + if strings.Contains(s, sub) { + return true + } + } + return false +} diff --git a/internal/integrity/sha_test.go b/internal/integrity/sha_test.go new file mode 100644 index 0000000..278a08d --- /dev/null +++ b/internal/integrity/sha_test.go @@ -0,0 +1,21 @@ +package integrity + +import ( + "crypto/sha256" + "encoding/hex" + "io" + "os" +) + +func fileSHA(p string) (string, error) { + f, err := os.Open(p) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/normalize/current.go b/internal/normalize/current.go new file mode 100644 index 0000000..ca8c671 --- /dev/null +++ b/internal/normalize/current.go @@ -0,0 +1,202 @@ +package normalize + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/efij/AgentDFIR/v3/internal/casepkg" + "github.com/efij/AgentDFIR/v3/internal/fingerprint" + "github.com/efij/AgentDFIR/v3/internal/overlay" + "github.com/efij/AgentDFIR/v3/internal/store" + "github.com/efij/AgentDFIR/v3/internal/version" +) + +// hostWitnessType is the artifact type of the host witness record. It is +// read by analysis, not by any parser, so it is not an overlay input: a +// round that only re-asks the host must not rebuild 300 MB of events. +const hostWitnessType = "host_witness" + +// InputsDigest identifies the evidence an overlay is built from: every +// current, acquired artifact a parser could read, by everything a parser +// reads off its record (the same fields segKey covers). Two manifests with +// the same digest produce the same overlay. +// +// This replaces comparing the manifest's modification time with the +// overlay's. Every round appends to the manifest — also a round that only +// carried unchanged files forward — so the old test rebuilt the overlay +// and re-ran every detection on a machine where nothing had changed. +func InputsDigest(man *casepkg.Manifest) string { + var keys []string + for _, a := range man.Current() { + if a.Status != casepkg.StatusOK || a.ArtifactType == hostWitnessType { + continue + } + keys = append(keys, segKey(a)+"\x00"+a.SourcePath) + } + sort.Strings(keys) + h := sha256.New() + for _, k := range keys { + h.Write([]byte(k)) + h.Write([]byte{'\n'}) + } + return hex.EncodeToString(h.Sum(nil)) +} + +// OverlayStatus describes the overlay a package carries. +type OverlayStatus struct { + Current bool + Reason string // why it is not current + BuildID string // changes every time events.jsonl is rebuilt +} + +// Status reports whether the package's overlay was built by this binary's +// parsing code from the package's current evidence, and is still exactly +// what that build (and the analysis stages that annotate it) wrote. +// +// The overlay is derived data outside the seal. Reusing it round after +// round is what makes a repeat run cheap, and it is also what would let +// an edit to it — an event deleted from events.jsonl — hide from every +// later analysis. So reuse is conditional on the state's MAC (when this +// machine has a key) and on events.jsonl still hashing to what was +// recorded. Hashing it costs a fraction of a second; parsing it again +// costs minutes, and trusting it unchecked costs the analysis. +func Status(pkgDir string) OverlayStatus { + dir := filepath.Join(pkgDir, "normalized") + evPath := filepath.Join(dir, "events.jsonl") + if _, err := overlay.Stat(evPath); err != nil { + return OverlayStatus{Reason: "no normalized events"} + } + st, err := readState(dir, store.CacheKey()) + if err != nil { + return OverlayStatus{Reason: err.Error()} + } + switch { + case st.SchemaVersion != version.SchemaVersion: + return OverlayStatus{Reason: "normalized schema changed"} + case st.ParseFingerprint != fingerprint.Parse(): + return OverlayStatus{Reason: "parsing code changed since the overlay was built"} + } + man, err := casepkg.ReadManifest(pkgDir) + if err != nil { + return OverlayStatus{Reason: "unreadable manifest"} + } + if st.CaseID != man.CaseID || st.Host != man.Host { + return OverlayStatus{Reason: "overlay belongs to another case"} + } + if st.InputsDigest != InputsDigest(man) { + return OverlayStatus{Reason: "evidence changed since the overlay was built"} + } + if sum, err := hashOverlayFile(evPath); err != nil || sum != st.EventsSHA256 { + return OverlayStatus{Reason: "normalized events were changed outside AgentDFIR; rebuilding them from the sealed evidence"} + } + return OverlayStatus{Current: true, BuildID: st.BuildID} +} + +// readState reads state.json and, when this machine has a cache key, +// requires its MAC. +func readState(dir string, key []byte) (*overlayState, error) { + data, err := os.ReadFile(filepath.Join(dir, stateFileName)) + if err != nil { + return nil, errors.New("no overlay state") + } + if len(key) > 0 { + mac, err := os.ReadFile(filepath.Join(dir, macFileName)) + if err != nil || !hmac.Equal([]byte(strings.TrimSpace(string(mac))), []byte(stateMAC(key, data))) { + return nil, errors.New("overlay state failed its authentication check; rebuilding from the sealed evidence") + } + } + var st overlayState + if err := json.Unmarshal(data, &st); err != nil { + return nil, errors.New("unreadable overlay state") + } + return &st, nil +} + +// RecordEvents re-records events.jsonl's hash after a stage of this +// package's own code rewrote it (the host witness and endpoint +// corroboration annotate events in place), so the rewrite is not mistaken +// for tampering. It does nothing when there is no overlay state. +func RecordEvents(pkgDir string) error { + dir := filepath.Join(pkgDir, "normalized") + key := store.CacheKey() + st, err := readState(dir, key) + if err != nil { + return nil + } + sum, err := hashOverlayFile(filepath.Join(dir, "events.jsonl")) + if err != nil { + return err + } + st.EventsSHA256 = sum + return writeState(dir, *st, key) +} + +func hashOverlayFile(path string) (string, error) { + f, err := overlay.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// writeFileAtomic replaces path with data so a reader never sees a +// partial file. +func writeFileAtomic(path string, data []byte) error { + tmp, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)+".*.tmp") + if err != nil { + return err + } + if _, err := tmp.Write(data); err != nil { + tmp.Close() + os.Remove(tmp.Name()) + return err + } + if err := tmp.Close(); err != nil { + os.Remove(tmp.Name()) + return err + } + if err := os.Chmod(tmp.Name(), 0o600); err != nil { + os.Remove(tmp.Name()) + return err + } + return os.Rename(tmp.Name(), path) +} + +// Refresh brings the package's overlay up to date — re-parsing only what +// changed — and writes the entity and relationship files with it. It is +// what analysis runs, and what acquisition runs before asking the host +// about the agent's claims, so the two share one build instead of each +// parsing the evidence. +func Refresh(pkgDir string, opt OverlayOptions) (*StreamResult, error) { + dir := filepath.Join(pkgDir, "normalized") + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + if opt.MACKey == nil { + opt.MACKey = store.CacheKey() + } + sr, err := BuildOverlay(pkgDir, dir, opt) + if err != nil { + return nil, err + } + if err := overlay.WriteJSONL(filepath.Join(dir, "entities.jsonl"), len(sr.Entities), func(i int) any { return sr.Entities[i] }); err != nil { + return nil, err + } + if err := overlay.WriteJSONL(filepath.Join(dir, "relationships.jsonl"), len(sr.Relationships), func(i int) any { return sr.Relationships[i] }); err != nil { + return nil, err + } + return sr, nil +} diff --git a/internal/normalize/normalize.go b/internal/normalize/normalize.go index bd439cd..ff3fcf7 100644 --- a/internal/normalize/normalize.go +++ b/internal/normalize/normalize.go @@ -52,6 +52,9 @@ type StreamResult struct { // segments versus read again, when the events came from BuildOverlay. Reused int Reparsed int + // CacheRejected says why the cached segments were not used, when the + // reason was an integrity failure rather than ordinary invalidation. + CacheRejected string } // ParseStream runs all parsers and calls sink for every normalized event diff --git a/internal/normalize/segments.go b/internal/normalize/segments.go index 2f1aa75..30a0ed1 100644 --- a/internal/normalize/segments.go +++ b/internal/normalize/segments.go @@ -3,17 +3,21 @@ package normalize import ( "bufio" "compress/gzip" + "crypto/hmac" + "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" + "hash" "io" "os" "path/filepath" "strings" "github.com/efij/AgentDFIR/v3/internal/casepkg" + "github.com/efij/AgentDFIR/v3/internal/fingerprint" "github.com/efij/AgentDFIR/v3/internal/overlay" "github.com/efij/AgentDFIR/v3/internal/parsers/segment" "github.com/efij/AgentDFIR/v3/internal/schema" @@ -70,6 +74,11 @@ type OverlayOptions struct { // Full forces every artifact to be re-parsed, ignoring (and then // replacing) any cached segments. This is what --renormalize sets. Full bool + // MACKey authenticates state.json. With a key, a state file whose MAC + // is missing or wrong is not trusted and every artifact is re-parsed; + // without one (a case opened on a machine that never sealed it) the + // per-segment hashes are still checked. + MACKey []byte } // segState is one artifact's cached contribution, as persisted. @@ -79,6 +88,7 @@ type segState struct { LogicalPath string `json:"logical_path"` Base int `json:"base"` // sequence number of its first event when written Events int `json:"events"` // events in the segment file + SHA256 string `json:"sha256"` // of the segment's decompressed bytes Entities []schema.Entity `json:"entities,omitempty"` Relationships []schema.Relationship `json:"relationships,omitempty"` } @@ -91,25 +101,49 @@ type parserState struct { // overlayState is normalized/state.json. // -// Every field outside Parsers is an invalidation key. The binary's own -// version is one of them on purpose: a parser change between releases -// would silently keep producing the old events for every unchanged -// artifact, and a cached wrong answer is worse than a slow right one. The -// cost is one full re-analysis after an upgrade, which is what an analyst -// wants anyway. +// SchemaVersion, ParseFingerprint, CaseID and Host are invalidation keys. +// The parse fingerprint is a hash of the parsing code itself: a parser +// change between releases must not keep serving the old events for every +// unchanged artifact, because a cached wrong answer is worse than a slow +// right one. It used to be the release number, which made every release — +// including ones that never touched a parser — re-parse every case. +// +// ToolVersion is kept for the reader and is not a key. InputsDigest and +// BuildID describe the build rather than gate it: the digest says which +// evidence the overlay was built from (Current reports whether that is +// still the package's evidence), and the build id changes every time +// events.jsonl is rewritten, so results computed on an earlier build can +// tell they are out of date. type overlayState struct { - SchemaVersion string `json:"schema_version"` - ToolVersion string `json:"agentdfir_version"` - CaseID string `json:"case_id"` - Host string `json:"host"` - Parsers []parserState `json:"parsers"` + SchemaVersion string `json:"schema_version"` + ParseFingerprint string `json:"parse_fingerprint"` + ToolVersion string `json:"agentdfir_version"` + CaseID string `json:"case_id"` + Host string `json:"host"` + InputsDigest string `json:"inputs_digest"` + BuildID string `json:"build_id"` + // EventsSHA256 is the hash of events.jsonl as last written by this + // package's own code (a build, or an analysis stage annotating it). A + // file that no longer matches was changed by something else and is + // not reused. + EventsSHA256 string `json:"events_sha256"` + Parsers []parserState `json:"parsers"` } +// macFileName holds the HMAC of state.json when the build had a key. +const macFileName = "state.mac" + // errNoSegment means the cache entry exists but its file cannot be opened — // deleted, truncated, or no longer a readable gzip stream. The artifact is // parsed fresh instead of failing the analysis. var errNoSegment = errors.New("segment file missing") +// ErrSegmentAltered means a cached segment's bytes no longer hash to what +// was recorded when it was written. Its events have already been copied +// into the overlay being built by then, so the build is abandoned and +// redone from the sealed evidence; BuildOverlay reports the reason. +var ErrSegmentAltered = errors.New("cached segment does not match its recorded hash") + // BuildOverlay writes normalized/events.jsonl and its segments, re-parsing // only the artifacts whose content is new to the overlay, and returns the // entities and relationships for the whole package. @@ -123,8 +157,13 @@ func BuildOverlay(pkgDir, dir string, opt OverlayOptions) (*StreamResult, error) // A damaged or half-written overlay must not make the package // un-analyzable: fall back to the full parse once, which also // rewrites every segment. + reason := err.Error() opt.Full = true - return buildOverlay(pkgDir, dir, opt) + res, err = buildOverlay(pkgDir, dir, opt) + if res != nil { + res.CacheRejected = reason + } + return res, err } return res, err } @@ -150,16 +189,18 @@ func buildOverlay(pkgDir, dir string, opt OverlayOptions) (*StreamResult, error) os.Remove(tmpName) // no-op once renamed }() + evHash := sha256.New() ov := &segCache{ segDir: segDir, - buf: bufio.NewWriterSize(tmp, 256<<10), + buf: bufio.NewWriterSize(io.MultiWriter(tmp, evHash), 256<<10), prev: map[string]*segState{}, live: map[string]bool{}, taken: map[string]bool{}, } ov.enc = json.NewEncoder(ov.buf) + var rejected string if !opt.Full { - ov.load(dir, man) + rejected = ov.load(dir, man, opt.MACKey) } merged := &schema.Normalized{} @@ -197,7 +238,8 @@ func buildOverlay(pkgDir, dir string, opt OverlayOptions) (*StreamResult, error) if err := os.Remove(evPath + overlay.Suffix); err != nil && !os.IsNotExist(err) { return nil, err } - if err := ov.save(dir, man); err != nil { + ov.eventsSHA = hex.EncodeToString(evHash.Sum(nil)) + if err := ov.save(dir, man, opt.MACKey); err != nil { return nil, err } ov.prune() @@ -208,6 +250,7 @@ func buildOverlay(pkgDir, dir string, opt OverlayOptions) (*StreamResult, error) EventCount: ov.count, Reused: ov.reused, Reparsed: ov.parsed, + CacheRejected: rejected, }, nil } @@ -231,35 +274,60 @@ type segCache struct { curKey string curFile *os.File curZW *gzip.Writer + curHash hash.Hash curEnc *json.Encoder curTmp string curWritten int - count int - reused int - parsed int + count int + reused int + parsed int + eventsSHA string } -func (o *segCache) load(dir string, man *casepkg.Manifest) { +// load reads the cache left by the previous build. It returns why the +// cache was refused when that is worth telling the analyst — an +// authentication failure, not an ordinary invalidation. +func (o *segCache) load(dir string, man *casepkg.Manifest, key []byte) string { data, err := os.ReadFile(filepath.Join(dir, stateFileName)) if err != nil { - return + return "" + } + if len(key) > 0 { + mac, err := os.ReadFile(filepath.Join(dir, macFileName)) + if err != nil { + return "analysis cache carries no MAC; re-parsed from the sealed evidence" + } + if !hmac.Equal([]byte(strings.TrimSpace(string(mac))), []byte(stateMAC(key, data))) { + return "analysis cache failed its MAC check; re-parsed from the sealed evidence" + } } var st overlayState if json.Unmarshal(data, &st) != nil { - return + return "" } - if st.SchemaVersion != version.SchemaVersion || st.ToolVersion != version.Version || + if st.SchemaVersion != version.SchemaVersion || st.ParseFingerprint != fingerprint.Parse() || st.CaseID != man.CaseID || st.Host != man.Host { - return + return "" } for i := range st.Parsers { ps := &st.Parsers[i] for j := range ps.Segments { s := &ps.Segments[j] + if s.SHA256 == "" { + continue // written before segments were hashed: parse again + } o.prev[ps.Name+"\x00"+s.Key] = s } } + return "" +} + +// stateMAC authenticates state.json's exact bytes. +func stateMAC(key, data []byte) string { + m := hmac.New(sha256.New, key) + m.Write(data) + return hex.EncodeToString(m.Sum(nil)) } func (o *segCache) begin(pe parserEntry) { @@ -337,12 +405,13 @@ func (o *segCache) End(art casepkg.ArtifactRecord, base, count int, ents []schem o.curKey = "" return fmt.Errorf("%s: parser advanced %d event(s) but wrote %d", art.LogicalPath, count, o.curWritten) } + sum := hex.EncodeToString(o.curHash.Sum(nil)) if err := o.closeFresh(true); err != nil { return err } o.record(segState{ Key: o.curKey, ArtifactID: art.ArtifactID, LogicalPath: art.LogicalPath, - Base: base, Events: count, Entities: ents, Relationships: rels, + Base: base, Events: count, SHA256: sum, Entities: ents, Relationships: rels, }) o.live[o.segRel(o.curKey)] = true o.taken[o.curKey] = true @@ -380,16 +449,20 @@ func (o *segCache) replay(st *segState, base int) (*segment.Replay, error) { } defer f.Close() + // The segment is hashed as it is copied — the bytes have to be read + // anyway — so an edited cache costs one rebuild, never a wrong answer. + h := sha256.New() + src := io.TeeReader(f, h) delta := base - st.Base n := 0 if delta == 0 { cw := &lineCounter{w: o.buf} - if _, err := io.Copy(cw, f); err != nil { + if _, err := io.Copy(cw, src); err != nil { return nil, fmt.Errorf("segment %s: %w", st.LogicalPath, err) } n = cw.lines } else { - sc := bufio.NewScanner(f) + sc := bufio.NewScanner(src) sc.Buffer(make([]byte, 0, 64<<10), 16<<20) for sc.Scan() { var ev schema.Event @@ -406,6 +479,9 @@ func (o *segCache) replay(st *segState, base int) (*segment.Replay, error) { return nil, fmt.Errorf("segment %s: %w", st.LogicalPath, err) } } + if got := hex.EncodeToString(h.Sum(nil)); got != st.SHA256 { + return nil, fmt.Errorf("segment %s: %w", st.LogicalPath, ErrSegmentAltered) + } if n != st.Events { return nil, fmt.Errorf("segment %s: holds %d event(s), state says %d", st.LogicalPath, n, st.Events) } @@ -435,14 +511,15 @@ func (o *segCache) fresh(key string) error { } o.curKey, o.curFile, o.curTmp = key, f, f.Name() o.curZW = gzip.NewWriter(f) - o.curEnc = json.NewEncoder(o.curZW) + o.curHash = sha256.New() + o.curEnc = json.NewEncoder(io.MultiWriter(o.curZW, o.curHash)) o.curWritten = 0 return nil } func (o *segCache) closeFresh(keep bool) error { f, zw, tmp := o.curFile, o.curZW, o.curTmp - o.curFile, o.curZW, o.curEnc, o.curTmp = nil, nil, nil, "" + o.curFile, o.curZW, o.curEnc, o.curTmp, o.curHash = nil, nil, nil, "", nil if f == nil { return nil } @@ -472,16 +549,39 @@ func (o *segCache) segPath(key string) string { return filepath.Join(o.segDir, o.name, key+".jsonl") } -func (o *segCache) save(dir string, man *casepkg.Manifest) error { +func (o *segCache) save(dir string, man *casepkg.Manifest, key []byte) error { + var id [16]byte + if _, err := rand.Read(id[:]); err != nil { + return err + } st := overlayState{ - SchemaVersion: version.SchemaVersion, ToolVersion: version.Version, - CaseID: man.CaseID, Host: man.Host, Parsers: o.out, + SchemaVersion: version.SchemaVersion, ParseFingerprint: fingerprint.Parse(), + ToolVersion: version.Version, CaseID: man.CaseID, Host: man.Host, + InputsDigest: InputsDigest(man), BuildID: hex.EncodeToString(id[:]), + EventsSHA256: o.eventsSHA, Parsers: o.out, } + return writeState(dir, st, key) +} + +// writeState persists the overlay state with its MAC. +func writeState(dir string, st overlayState, key []byte) error { data, err := json.MarshalIndent(st, "", " ") if err != nil { return err } - return os.WriteFile(filepath.Join(dir, stateFileName), append(data, '\n'), 0o600) + data = append(data, '\n') + macPath := filepath.Join(dir, macFileName) + // The MAC goes first and the state second: a crash between the two + // leaves a MAC that does not match, which costs a re-parse, never a + // state that is trusted without one. + if len(key) > 0 { + if err := os.WriteFile(macPath, []byte(stateMAC(key, data)+"\n"), 0o600); err != nil { + return err + } + } else if err := os.Remove(macPath); err != nil && !os.IsNotExist(err) { + return err + } + return writeFileAtomic(filepath.Join(dir, stateFileName), data) } // prune deletes segments no artifact in the package refers to any more — diff --git a/internal/progress/model.go b/internal/progress/model.go new file mode 100644 index 0000000..fe772d2 --- /dev/null +++ b/internal/progress/model.go @@ -0,0 +1,146 @@ +package progress + +import ( + "bufio" + "encoding/json" + "os" + "path/filepath" + "time" +) + +// Model predicts how long a step takes on this machine from how long it +// took before. +// +// Each finished step is appended to /perf.jsonl as (step, units, +// seconds). A step's prediction is the exponentially weighted average of +// its seconds-per-unit over recent runs, times this run's units — or, for a +// step that reports no units, of its seconds. With no history it falls +// back to the rate the caller supplies, so the first run is an estimate +// (shown with a "~") and every later run is calibrated on this machine. +type Model struct { + path string + all []sample // file order, oldest first + hist map[string][]sample + new []sample +} + +type sample struct { + Step string `json:"step"` + Key string `json:"key,omitempty"` + Units float64 `json:"units"` + Seconds float64 `json:"seconds"` + TimeUTC string `json:"ts_utc"` +} + +// maxHistory bounds perf.jsonl; older samples say little about today. +const maxHistory = 400 + +// LoadModel reads the timing history at path. A missing or unreadable +// file is an empty history, never an error: timing is advisory. +func LoadModel(path string) *Model { + m := &Model{path: path, hist: map[string][]sample{}} + if path == "" { + return m + } + f, err := os.Open(path) + if err != nil { + return m + } + defer f.Close() + sc := bufio.NewScanner(f) + for sc.Scan() { + var s sample + if json.Unmarshal(sc.Bytes(), &s) == nil && s.Step != "" && s.Seconds >= 0 { + m.all = append(m.all, s) + m.hist[s.Step] = append(m.hist[s.Step], s) + } + } + return m +} + +// Known reports whether the model has history for a step. +func (m *Model) Known(step string) bool { return len(m.hist[step]) > 0 } + +// Predict estimates a step's duration. defaultRate is seconds per unit +// (or plain seconds when units is 0) to use without history. +func (m *Model) Predict(step, key string, units, defaultRate float64) time.Duration { + hs := m.hist[step] + // Prefer this key's history (the same case), else the step's. + var use []sample + for _, s := range hs { + if key != "" && s.Key == key { + use = append(use, s) + } + } + if len(use) == 0 { + use = hs + } + rate := defaultRate + if len(use) > 0 { + const alpha = 0.5 + r := -1.0 + for _, s := range use[max(0, len(use)-8):] { + var x float64 + if units > 0 && s.Units > 0 { + x = s.Seconds / s.Units + } else if units > 0 { + continue + } else { + x = s.Seconds + } + if r < 0 { + r = x + } else { + r = alpha*x + (1-alpha)*r + } + } + if r >= 0 { + rate = r + } + } + if units > 0 { + return time.Duration(rate * units * float64(time.Second)) + } + return time.Duration(rate * float64(time.Second)) +} + +// Record notes a finished step for the next prediction. +func (m *Model) Record(step, key string, units float64, took time.Duration) { + m.new = append(m.new, sample{Step: step, Key: key, Units: units, Seconds: took.Seconds(), + TimeUTC: time.Now().UTC().Format(time.RFC3339)}) +} + +// Save appends this run's samples, trimming the file to its newest +// maxHistory lines. +func (m *Model) Save() error { + if m.path == "" || len(m.new) == 0 { + return nil + } + all := append(append([]sample(nil), m.all...), m.new...) + if len(all) > maxHistory { + all = all[len(all)-maxHistory:] + } + tmp, err := os.CreateTemp(filepath.Dir(m.path), ".perf-*.tmp") + if err != nil { + return err + } + w := bufio.NewWriter(tmp) + enc := json.NewEncoder(w) + for _, s := range all { + if err := enc.Encode(s); err != nil { + tmp.Close() + os.Remove(tmp.Name()) + return err + } + } + if err := w.Flush(); err != nil { + tmp.Close() + os.Remove(tmp.Name()) + return err + } + if err := tmp.Close(); err != nil { + os.Remove(tmp.Name()) + return err + } + return os.Rename(tmp.Name(), m.path) +} diff --git a/internal/progress/tracker.go b/internal/progress/tracker.go new file mode 100644 index 0000000..5599f31 --- /dev/null +++ b/internal/progress/tracker.go @@ -0,0 +1,482 @@ +// Package progress draws the one status display a long command keeps +// redrawing: an overall bar across every step of the run, elapsed time, +// and a time remaining that is predicted from this machine's own history +// and counts down steadily instead of jumping. +// +// AgentDFIR ▕████████████▋ ▏ 46% elapsed 0:41 ETA 0:48 +// analyze · rule packs (stage 4/7) +// +// Every step reports progress in units that mean work (bytes actually +// read, not bytes carried forward from an earlier round). The time +// remaining is the model's prediction for what is left, scaled by how the +// finished steps compared with their predictions today. The displayed +// value then counts down one second per second and is only pulled towards +// the model when the two disagree by more than max(10%, 3 s), so it neither +// flickers nor leaps. It never reads 0:00 while work remains. +// +// On a terminal it redraws in place; otherwise it prints a plain line when +// a step starts and every ten seconds. Log lines written through it print +// above the display without tearing it. +package progress + +import ( + "fmt" + "io" + "math" + "os" + "strings" + "sync" + "time" +) + +// Step is one planned part of a run. +type Step struct { + Name string // stable id, used for the timing history + Label string // what the display calls it + Units float64 // work units expected (0: the step reports none) + // Rate is seconds per unit (or plain seconds when Units is 0) to + // assume when this machine has no history for the step. + Rate float64 +} + +type stepState struct { + Step + predicted time.Duration + done float64 + started time.Time + took time.Duration + finished bool + skipped bool +} + +// Tracker is the display for one run. +type Tracker struct { + mu sync.Mutex + out io.Writer + tty bool + title string + key string + model *Model + now func() time.Time + steps []*stepState + cur int + start time.Time + detail string + + shownETA time.Duration + etaSet bool + lastTick time.Time + lastPlain time.Time + drawn bool + calibrated bool + + stop chan struct{} + done chan struct{} +} + +// Options configure a Tracker. +type Options struct { + Out io.Writer // default os.Stdout + Title string // default "AgentDFIR" + Key string // groups history (e.g. the case), optional + Model *Model // timing history; nil = none + TTY *bool // override terminal detection (tests) + Now func() time.Time +} + +// New plans a run. +func New(steps []Step, o Options) *Tracker { + t := &Tracker{out: o.Out, title: o.Title, key: o.Key, model: o.Model, now: o.Now, cur: -1} + if t.out == nil { + t.out = os.Stdout + } + if t.title == "" { + t.title = "AgentDFIR" + } + if t.model == nil { + t.model = LoadModel("") + } + if t.now == nil { + t.now = time.Now + } + if o.TTY != nil { + t.tty = *o.TTY + } else { + t.tty = IsTerminal(t.out) + } + t.calibrated = true + for _, s := range steps { + st := &stepState{Step: s} + st.predicted = t.model.Predict(s.Name, t.key, s.Units, s.Rate) + if !t.model.Known(s.Name) { + t.calibrated = false + } + t.steps = append(t.steps, st) + } + t.start = t.now() + return t +} + +// IsTerminal reports whether w is an interactive terminal that can take +// in-place redraws. +func IsTerminal(w io.Writer) bool { + f, ok := w.(*os.File) + if !ok { + return false + } + fi, err := f.Stat() + if err != nil || fi.Mode()&os.ModeCharDevice == 0 { + return false + } + return os.Getenv("TERM") != "dumb" +} + +// Run starts the redraw ticker. +func (t *Tracker) Run() { + t.mu.Lock() + if t.stop != nil { + t.mu.Unlock() + return + } + stop, done := make(chan struct{}), make(chan struct{}) + t.stop, t.done = stop, done + t.mu.Unlock() + go func() { + defer close(done) + tk := time.NewTicker(250 * time.Millisecond) + defer tk.Stop() + for { + select { + case <-stop: + return + case <-tk.C: + t.mu.Lock() + t.tick() + t.mu.Unlock() + } + } + }() +} + +func (t *Tracker) find(name string) int { + for i, s := range t.steps { + if s.Name == name { + return i + } + } + return -1 +} + +// Begin starts a step. Steps before it that never began are skipped. +func (t *Tracker) Begin(name string) { + t.mu.Lock() + defer t.mu.Unlock() + i := t.find(name) + if i < 0 { + return + } + t.finishCurrent() + for j := t.cur + 1; j < i; j++ { + if !t.steps[j].finished { + t.steps[j].skipped = true + } + } + t.cur = i + t.steps[i].started = t.now() + t.detail = "" + t.tick() +} + +// Skip marks a planned step as not needed this run (its time leaves the +// estimate). +func (t *Tracker) Skip(name string) { + t.mu.Lock() + defer t.mu.Unlock() + if i := t.find(name); i >= 0 && !t.steps[i].finished && i != t.cur { + t.steps[i].skipped = true + } +} + +// SetUnits replaces a step's expected work once it is known (after a +// survey, say) and re-predicts it. +func (t *Tracker) SetUnits(name string, units float64) { + t.mu.Lock() + defer t.mu.Unlock() + if i := t.find(name); i >= 0 { + s := t.steps[i] + s.Units = units + s.predicted = t.model.Predict(s.Name, t.key, units, s.Rate) + } +} + +// Progress reports how many units of the current step are done. +func (t *Tracker) Progress(done float64) { + t.mu.Lock() + if t.cur >= 0 { + t.steps[t.cur].done = done + } + t.mu.Unlock() +} + +// Detail sets the second line (what the current step is doing). +func (t *Tracker) Detail(format string, a ...any) { + t.mu.Lock() + t.detail = fmt.Sprintf(format, a...) + t.mu.Unlock() +} + +// End finishes the current step. +func (t *Tracker) End() { + t.mu.Lock() + defer t.mu.Unlock() + t.finishCurrent() + t.clear() +} + +func (t *Tracker) finishCurrent() { + if t.cur < 0 { + return + } + s := t.steps[t.cur] + if s.finished || s.started.IsZero() { + return + } + s.finished = true + s.took = t.now().Sub(s.started) + units := s.Units + if units > 0 && s.done > 0 { + units = s.done + } + t.model.Record(s.Name, t.key, units, s.took) +} + +// Stop ends the display and saves the timing history. +func (t *Tracker) Stop() { + t.mu.Lock() + t.finishCurrent() + stop, done := t.stop, t.done + t.stop = nil + t.mu.Unlock() + if stop != nil { + close(stop) + <-done + } + t.mu.Lock() + t.clear() + t.mu.Unlock() + _ = t.model.Save() +} + +// Write prints log lines above the display. +func (t *Tracker) Write(p []byte) (int, error) { + t.mu.Lock() + defer t.mu.Unlock() + t.clear() + n, err := t.out.Write(p) + if t.tty && t.stop != nil { + t.draw() + } + return n, err +} + +// Summary is one line of per-step timings for the end of a run. +func (t *Tracker) Summary() string { + t.mu.Lock() + defer t.mu.Unlock() + var parts []string + for _, s := range t.steps { + if s.finished { + parts = append(parts, fmt.Sprintf("%s %s", s.Name, Clock(s.took))) + } + } + return strings.Join(parts, " · ") +} + +// Elapsed is the time since the run started. +func (t *Tracker) Elapsed() time.Duration { + t.mu.Lock() + defer t.mu.Unlock() + return t.now().Sub(t.start) +} + +// k is how today's finished steps compare with their predictions — a +// mild correction for how busy the machine is today. +func (t *Tracker) k() float64 { + k, n := 1.0, 0 + for _, s := range t.steps { + if !s.finished || s.predicted < 500*time.Millisecond { + continue // tiny steps say nothing about the machine's pace + } + r := s.took.Seconds() / s.predicted.Seconds() + if n == 0 { + k = r + } else { + k = 0.5*r + 0.5*k + } + n++ + } + // Narrow on purpose: steps have different bottlenecks (acquisition is + // I/O, analysis is CPU), so a slow disk today says little about how + // long the analysis will take. Each step switches to its own measured + // pace once it is running. + return math.Min(1.5, math.Max(0.67, k)) +} + +// modelRemaining is the model's current estimate of the time left. +func (t *Tracker) modelRemaining(now time.Time) time.Duration { + k := t.k() + var rem float64 + for i, s := range t.steps { + if s.finished || s.skipped { + continue + } + pred := s.predicted.Seconds() * k + if i != t.cur { + if i > t.cur { + rem += pred + } + continue + } + el := now.Sub(s.started).Seconds() + byTime := math.Max(pred-el, 0) + if s.Units > 0 && s.done > 0 { + f := math.Min(s.done/s.Units, 1) + byRate := el * (1 - f) / f // today's measured pace + byModel := pred * (1 - f) // the work left, at the predicted pace + w := math.Min(1, f/0.2) // the measured pace is the answer once a fifth is done + rem += w*byRate + (1-w)*byModel + } else { + // No units: the prediction, and never less than a moment once + // it has been exceeded — the step is still running. + rem += math.Max(byTime, math.Min(pred*0.1, 5)) + } + } + return time.Duration(rem * float64(time.Second)) +} + +// tick advances the displayed ETA and redraws. +func (t *Tracker) tick() { + now := t.now() + model := t.modelRemaining(now) + if !t.etaSet { + t.shownETA, t.etaSet, t.lastTick = model, true, now + } else { + dt := now.Sub(t.lastTick) + t.lastTick = now + t.shownETA -= dt + if t.shownETA < 0 { + t.shownETA = 0 + } + diff := model - t.shownETA + band := time.Duration(math.Max(0.1*model.Seconds(), 3) * float64(time.Second)) + if diff > band || diff < -band { + // Pull towards the model with a one-second time constant. + f := math.Min(1, dt.Seconds()) + t.shownETA += time.Duration(float64(diff) * f) + } + } + if t.tty { + t.draw() + } else if now.Sub(t.lastPlain) >= 10*time.Second { + fmt.Fprintf(t.out, " %s\n", t.plainStatus(now)) + t.lastPlain = now + } +} + +func (t *Tracker) eta() string { + allDone := true + for _, s := range t.steps { + if !s.finished && !s.skipped { + allDone = false + } + } + if allDone { + return "0:00" + } + if t.shownETA < time.Second { + return "finishing…" + } + s := Clock(t.shownETA) + if !t.calibrated && t.fractionDone() < 0.05 { + s = "~" + s + } + return s +} + +func (t *Tracker) fractionDone() float64 { + el := t.now().Sub(t.start).Seconds() + total := el + t.shownETA.Seconds() + if total <= 0 { + return 0 + } + return math.Min(1, el/total) +} + +func (t *Tracker) plainStatus(now time.Time) string { + label := "" + if t.cur >= 0 { + label = t.steps[t.cur].Label + } + return fmt.Sprintf("%3d%% · %s · elapsed %s · ETA %s", int(100*t.fractionDone()), label, Clock(now.Sub(t.start)), t.eta()) +} + +const barWidth = 28 + +func (t *Tracker) draw() { + now := t.now() + f := t.fractionDone() + cells := f * barWidth + full := int(cells) + partial := []string{"", "▏", "▎", "▍", "▌", "▋", "▊", "▉"}[int((cells-float64(full))*8)] + bar := strings.Repeat("█", full) + partial + bar += strings.Repeat(" ", max(0, barWidth-full-len([]rune(partial)))) + line1 := fmt.Sprintf(" %s ▕%s▏ %3d%% elapsed %s ETA %s", t.title, bar, int(100*f), Clock(now.Sub(t.start)), t.eta()) + line2 := "" + if t.cur >= 0 { + line2 = " " + t.steps[t.cur].Label + if t.detail != "" { + line2 += " · " + t.detail + } + } + if w := width(); w > 0 { + line1, line2 = clip(line1, w), clip(line2, w) + } + t.clear() + fmt.Fprintf(t.out, "%s\n\033[2K%s\033[1A\r", line1, line2) + t.drawn = true +} + +func (t *Tracker) clear() { + if t.tty && t.drawn { + fmt.Fprint(t.out, "\r\033[2K\n\033[2K\033[1A\r") + t.drawn = false + } +} + +func clip(s string, w int) string { + r := []rune(s) + if len(r) >= w { + return string(r[:w-1]) + } + return s +} + +func width() int { + var n int + if _, err := fmt.Sscan(os.Getenv("COLUMNS"), &n); err == nil && n > 20 { + return n + } + return 100 +} + +// Clock renders a duration as m:ss or h:mm:ss. +func Clock(d time.Duration) string { + s := int(d.Round(time.Second).Seconds()) + if s < 0 { + s = 0 + } + if s >= 3600 { + return fmt.Sprintf("%d:%02d:%02d", s/3600, (s/60)%60, s%60) + } + return fmt.Sprintf("%d:%02d", s/60, s%60) +} diff --git a/internal/progress/tracker_test.go b/internal/progress/tracker_test.go new file mode 100644 index 0000000..ac80e34 --- /dev/null +++ b/internal/progress/tracker_test.go @@ -0,0 +1,147 @@ +package progress + +import ( + "bytes" + "path/filepath" + "strings" + "testing" + "time" +) + +type clock struct{ t time.Time } + +func (c *clock) now() time.Time { return c.t } +func (c *clock) advance(d time.Duration) { c.t = c.t.Add(d) } + +func newTest(t *testing.T, steps []Step, m *Model) (*Tracker, *clock, *bytes.Buffer) { + t.Helper() + c := &clock{t: time.Date(2026, 9, 29, 12, 0, 0, 0, time.UTC)} + var out bytes.Buffer + tty := true + tr := New(steps, Options{Out: &out, TTY: &tty, Now: c.now, Model: m}) + return tr, c, &out +} + +// The displayed time remaining counts down one second per second while +// the work goes as predicted — it ticks like the elapsed clock does. +func TestETACountsDownSteadily(t *testing.T) { + tr, c, _ := newTest(t, []Step{{Name: "collect", Units: 100, Rate: 1}}, nil) // 100 s predicted + tr.Begin("collect") + prev := tr.shownETA + for i := 1; i <= 40; i++ { + c.advance(time.Second) + tr.Progress(float64(i)) // exactly on prediction: 1 unit/s + tr.mu.Lock() + tr.tick() + got := tr.shownETA + tr.mu.Unlock() + if d := prev - got; d < 900*time.Millisecond || d > 1100*time.Millisecond { + t.Fatalf("second %d: ETA moved by %v, want ~1s (from %v to %v)", i, d, prev, got) + } + prev = got + } + if prev < 55*time.Second || prev > 65*time.Second { + t.Fatalf("after 40 of 100 units at the predicted pace, ETA = %v, want ~60s", prev) + } +} + +// When the machine is slower than predicted the estimate is pulled +// towards reality instead of running out while work remains. +func TestETAFollowsASlowerMachineWithoutJumping(t *testing.T) { + tr, c, _ := newTest(t, []Step{{Name: "collect", Units: 100, Rate: 1}}, nil) + tr.Begin("collect") + var last time.Duration = tr.shownETA + for i := 1; i <= 100; i++ { + c.advance(time.Second) + tr.Progress(float64(i) / 2) // half the predicted pace + tr.mu.Lock() + tr.tick() + got := tr.shownETA + tr.mu.Unlock() + if got-last > 60*time.Second { + t.Fatalf("second %d: ETA leapt from %v to %v", i, last, got) + } + last = got + } + // 50 of 100 units in 100 s: about 100 s remain. + if last < 80*time.Second || last > 120*time.Second { + t.Fatalf("at half pace, halfway, ETA = %v, want ~100s", last) + } +} + +// The display never claims 0:00 while a step is still running. +func TestETANeverZeroWhileWorkRemains(t *testing.T) { + tr, c, _ := newTest(t, []Step{{Name: "analyze", Rate: 5}}, nil) // no units, 5 s predicted + tr.Begin("analyze") + for i := 0; i < 30; i++ { + c.advance(time.Second) + tr.mu.Lock() + tr.tick() + s := tr.eta() + tr.mu.Unlock() + if s == "0:00" { + t.Fatalf("second %d: ETA shows 0:00 while the step is still running", i+1) + } + } + tr.End() + tr.mu.Lock() + defer tr.mu.Unlock() + if s := tr.eta(); s != "0:00" { + t.Fatalf("after the last step, ETA = %q, want 0:00", s) + } +} + +// The history makes the second run's prediction match what the first run +// took, per unit. +func TestModelLearnsFromHistory(t *testing.T) { + path := filepath.Join(t.TempDir(), "perf.jsonl") + m := LoadModel(path) + m.Record("analyze", "case1", 1000, 50*time.Second) // 0.05 s/unit + if err := m.Save(); err != nil { + t.Fatal(err) + } + m2 := LoadModel(path) + if !m2.Known("analyze") { + t.Fatal("history not read back") + } + if got := m2.Predict("analyze", "case1", 2000, 1); got < 99*time.Second || got > 101*time.Second { + t.Fatalf("predicted %v for twice the units, want ~100s", got) + } + if got := m2.Predict("unknown-step", "", 10, 2); got != 20*time.Second { + t.Fatalf("no history: predicted %v, want the default 20s", got) + } +} + +// The first run on a machine has no history and says its estimate is one. +func TestUncalibratedETAIsMarked(t *testing.T) { + tr, c, out := newTest(t, []Step{{Name: "collect", Units: 100, Rate: 1}}, nil) + tr.Begin("collect") + c.advance(time.Second) + tr.Progress(1) + tr.mu.Lock() + tr.tick() + tr.mu.Unlock() + if !strings.Contains(out.String(), "ETA ~") { + t.Fatalf("uncalibrated estimate not marked with ~:\n%q", out.String()) + } +} + +// Skipped steps leave the estimate. +func TestSkippedStepLeavesTheEstimate(t *testing.T) { + tr, _, _ := newTest(t, []Step{{Name: "collect", Rate: 10}, {Name: "analyze", Rate: 100}}, nil) + tr.Begin("collect") + tr.Skip("analyze") + tr.mu.Lock() + defer tr.mu.Unlock() + if rem := tr.modelRemaining(tr.now()); rem > 11*time.Second { + t.Fatalf("remaining %v still includes the skipped step", rem) + } +} + +func TestClock(t *testing.T) { + for d, want := range map[time.Duration]string{0: "0:00", 59 * time.Second: "0:59", 61 * time.Second: "1:01", 3661 * time.Second: "1:01:01"} { + if got := Clock(d); got != want { + t.Errorf("Clock(%v) = %q, want %q", d, got, want) + } + } +} diff --git a/internal/rulepack/prefilter.go b/internal/rulepack/prefilter.go new file mode 100644 index 0000000..425896d --- /dev/null +++ b/internal/rulepack/prefilter.go @@ -0,0 +1,155 @@ +package rulepack + +import ( + "regexp/syntax" + "strings" + "unicode" + "unicode/utf8" +) + +// The literal prefilter. +// +// Most subjects match no rule, and Go's regexp engine is linear but not +// cheap: on a real case the rule packs spent a minute running 81 regexes +// over 359,000 events. Nearly every one of those regexes cannot match +// without some literal text appearing in the subject — `curl`, `.ssh/`, +// `base64`. requiredLiterals works that set out from the regex's syntax +// tree, and a subject that contains none of them is not handed to the +// regexp engine at all. +// +// It must never skip a subject the regex would match. Two things make that +// hold. The extraction is conservative: any construct it does not fully +// understand (a character class, an optional or repeated-zero-times part, +// an alternation with an un-analyzable branch) makes it give up, and a +// rule with no prefilter runs the regex on everything. And the comparison +// folds case exactly the way regexp's (?i) does — every rune is mapped to +// the smallest member of its Unicode case-folding orbit, so "ſ" (U+017F) +// and "s", or "K" (U+212A, the Kelvin sign) and "k", compare equal — and +// it folds both sides whether or not the regex is case-insensitive. Folding +// is a rune-by-rune function, so a literal that occurs in a subject still +// occurs after both are folded: the filter can only let more through than +// the regex matches, never less. + +// maxLiterals bounds the set: a rule with more alternatives than this is +// cheaper to run directly. +const maxLiterals = 16 + +// minLiteral is the shortest literal worth filtering on. +const minLiteral = 2 + +// requiredLiterals returns folded literals at least one of which occurs in +// every string the regex matches, or nil when no such set can be proven. +func requiredLiterals(expr string) []string { + re, err := syntax.Parse(expr, syntax.Perl) + if err != nil { + return nil + } + set, ok := required(re.Simplify()) + if !ok || len(set) == 0 || len(set) > maxLiterals { + return nil + } + for _, s := range set { + if utf8.RuneCountInString(s) < minLiteral { + return nil + } + } + return set +} + +func required(re *syntax.Regexp) ([]string, bool) { + switch re.Op { + case syntax.OpLiteral: + return []string{fold(string(re.Rune))}, true + case syntax.OpCapture: + return required(re.Sub[0]) + case syntax.OpPlus: + return required(re.Sub[0]) + case syntax.OpRepeat: + if re.Min >= 1 { + return required(re.Sub[0]) + } + return nil, false + case syntax.OpConcat: + // Any one mandatory part is enough; take the most selective. + var best []string + bestLen := -1 + for _, sub := range re.Sub { + set, ok := required(sub) + if !ok { + continue + } + if l := shortest(set); l > bestLen { + best, bestLen = set, l + } + } + return best, best != nil + case syntax.OpAlternate: + var all []string + for _, sub := range re.Sub { + set, ok := required(sub) + if !ok { + return nil, false + } + all = append(all, set...) + } + return all, true + } + return nil, false +} + +func shortest(set []string) int { + n := -1 + for _, s := range set { + if l := utf8.RuneCountInString(s); n < 0 || l < n { + n = l + } + } + return n +} + +// fold maps every rune to the smallest rune of its case-folding orbit — +// the equivalence regexp's (?i) matches under. +func fold(s string) string { + ascii := true + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + ascii = false + break + } + } + if ascii { + return strings.ToUpper(s) // an ASCII letter's orbit minimum is its capital + } + var b strings.Builder + b.Grow(len(s)) + for _, r := range s { + b.WriteRune(foldRune(r)) + } + return b.String() +} + +func foldRune(r rune) rune { + if r < utf8.RuneSelf { + if 'a' <= r && r <= 'z' { + return r - 'a' + 'A' + } + return r + } + lo := r + for f := unicode.SimpleFold(r); f != r; f = unicode.SimpleFold(f) { + if f < lo { + lo = f + } + } + return lo +} + +// mayMatch reports whether a folded subject contains any required literal. +func mayMatch(need []string, folded string) bool { + for _, n := range need { + if strings.Contains(folded, n) { + return true + } + } + return false +} diff --git a/internal/rulepack/prefilter_test.go b/internal/rulepack/prefilter_test.go new file mode 100644 index 0000000..61535eb --- /dev/null +++ b/internal/rulepack/prefilter_test.go @@ -0,0 +1,130 @@ +package rulepack + +import ( + "os" + "path/filepath" + "reflect" + "regexp" + "sort" + "testing" + + "github.com/efij/AgentDFIR/v3/internal/overlay" + "github.com/efij/AgentDFIR/v3/internal/schema" +) + +func TestRequiredLiterals(t *testing.T) { + for _, tc := range []struct { + re string + want []string // nil: no prefilter may be derived + }{ + {`curl\s+http`, []string{"CURL"}}, + {`(?i)base64\s+-d`, []string{"BASE64"}}, + {`(curl|wget)\s+\S+\s*\|\s*(ba)?sh`, []string{"CURL", "WGET"}}, + {`\.ssh/id_[a-z]+`, []string{".SSH/ID_"}}, + {`(?:foo)?bar`, []string{"BAR"}}, + {`(?:foo)?(?:bar)?`, nil}, // nothing is mandatory + {`[a-z]+@[a-z]+`, []string{"@"}[:0:0]}, // one-rune literal: not worth it + {`a|[0-9]+`, nil}, // an alternative with no literal + {`x*`, nil}, + {`(?:ab){2,}`, []string{"AB"}}, + } { + got := requiredLiterals(tc.re) + if len(tc.want) == 0 { + if got != nil { + t.Errorf("%q: got prefilter %q, want none", tc.re, got) + } + continue + } + sort.Strings(got) + sort.Strings(tc.want) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("%q: got %q, want %q", tc.re, got, tc.want) + } + } +} + +// Case-insensitive matching in Go folds whole Unicode orbits: the long s +// matches "s" and the Kelvin sign matches "k". A prefilter that only +// lowercased would skip these and miss a match an attacker can craft. +func TestPrefilterFoldsLikeRegexp(t *testing.T) { + for _, tc := range []struct{ re, subject string }{ + {`(?i)secret`, "ſecret"}, + {`(?i)kubectl`, "Kubectl"}, + {`(?i)SSH`, "ſſh"}, + {`(?i)ÅNGSTRÖM`, "Ångström"}, // Angstrom sign + } { + re := regexp.MustCompile(tc.re) + if !re.MatchString(tc.subject) { + t.Fatalf("setup: %q does not match %q", tc.re, tc.subject) + } + need := requiredLiterals(tc.re) + if need == nil { + t.Fatalf("%q: no prefilter derived", tc.re) + } + if !mayMatch(need, fold(tc.subject)) { + t.Errorf("%q matches %q but the prefilter would skip it", tc.re, tc.subject) + } + } +} + +// The prefilter may only ever let more through than the regex matches. +func FuzzPrefilterNeverSkipsAMatch(f *testing.F) { + packs, _, err := Embedded() + if err != nil { + f.Fatal(err) + } + var rules []*Rule + for _, p := range packs { + for i := range p.Rules { + if p.Rules[i].re != nil && p.Rules[i].need != nil { + rules = append(rules, &p.Rules[i]) + } + } + } + for _, seed := range []string{"curl http://x | sh", "cat ~/.ssh/id_rsa", "ſudo rm -rf", "base64 -d <<< x", "Kubectl get secrets"} { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, s string) { + folded := fold(s) + for _, r := range rules { + if r.re.MatchString(s) && !mayMatch(r.need, folded) { + t.Fatalf("rule %s matches %q but its prefilter %q skips it", r.ID, s, r.need) + } + } + }) +} + +// Every shipped regex that gets a prefilter keeps matching what it +// matched: the embedded packs' rules are run with and without it over +// every event of a real case, when one is supplied. +// +// AGENTDFIR_PREFILTER_CASE=/path/to/case.adfir go test -run RealCase ./internal/rulepack/ +func TestPrefilterMatchesRealCase(t *testing.T) { + pkg := os.Getenv("AGENTDFIR_PREFILTER_CASE") + if pkg == "" { + t.Skip("AGENTDFIR_PREFILTER_CASE not set") + } + events := overlay.ReadJSONL[schema.Event](filepath.Join(pkg, "normalized", "events.jsonl")) + packs, _, err := Embedded() + if err != nil { + t.Fatal(err) + } + packs, _ = Dedupe(packs) + with, err := Apply(packs, &schema.Normalized{Events: events}, pkg) + if err != nil { + t.Fatal(err) + } + for _, p := range packs { + for i := range p.Rules { + p.Rules[i].need = nil + } + } + without, err := Apply(packs, &schema.Normalized{Events: events}, pkg) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(with, without) { + t.Fatalf("prefilter changed the findings: %d with, %d without", len(with), len(without)) + } + t.Logf("%d events, %d findings, identical with and without the prefilter", len(events), len(with)) +} diff --git a/internal/rulepack/rulepack.go b/internal/rulepack/rulepack.go index 2faca66..1fd4947 100644 --- a/internal/rulepack/rulepack.go +++ b/internal/rulepack/rulepack.go @@ -10,6 +10,9 @@ package rulepack import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" "fmt" "github.com/efij/AgentDFIR/v3/internal/shellshape" "os" @@ -20,6 +23,7 @@ import ( "sync" "sync/atomic" + "github.com/efij/AgentDFIR/v3/internal/artmemo" "github.com/efij/AgentDFIR/v3/internal/casepkg" "github.com/efij/AgentDFIR/v3/internal/schema" ) @@ -40,6 +44,9 @@ type Rule struct { re *regexp.Regexp targetRe *regexp.Regexp excludeRe *regexp.Regexp + // need is the regex's literal prefilter (see prefilter.go): nil when + // no set of required literals could be proven. + need []string } // Match declares what a rule inspects. @@ -149,6 +156,7 @@ func validatePack(p *Pack) error { return fmt.Errorf("rule %s: bad regex: %w", r.ID, err) } r.re = re + r.need = requiredLiterals(r.Match.Regex) } if r.Match.TargetRegex != "" { tre, err := regexp.Compile(r.Match.TargetRegex) @@ -197,12 +205,13 @@ func Apply(packs []Pack, res *schema.Normalized, pkgDir string) ([]schema.Findin var out []schema.Finding var artRules []*Rule dec := decodeEvents(res.Events) + subj := newSubjects(res.Events) for _, p := range packs { for i := range p.Rules { r := &p.Rules[i] switch r.Match.Type { case "command", "summary": - out = append(out, matchEvents(r, res, dec)...) + out = append(out, matchEvents(r, res, dec, subj)...) case "config", "instructions", "transcript": artRules = append(artRules, r) } @@ -214,38 +223,25 @@ func Apply(packs []Pack, res *schema.Normalized, pkgDir string) ([]schema.Findin if err != nil { return out, err } - out = append(out, matchArtifacts(artRules, man, casepkg.NewStore(pkgDir, man))...) + memo := artmemo.Open[[]schema.Finding](pkgDir, "rulepack-artifacts", rulesDigest(artRules)) + out = append(out, matchArtifacts(artRules, man, casepkg.NewStore(pkgDir, man), memo)...) + _ = memo.Save() // advisory: a memo that cannot be written costs the next run a re-scan + ArtifactMemoHits = memo.Hits() } return out, nil } -func matchEvents(r *Rule, res *schema.Normalized, dec *decoded) []schema.Finding { +func matchEvents(r *Rule, res *schema.Normalized, dec *decoded, subj *subjects) []schema.Finding { var out []schema.Finding + kind := subjectKind(r) for i := range res.Events { ev := &res.Events[i] - var subject string - switch r.Match.Type { - case "command": - // The full command, not the 300-character display copy: the - // flag or payload that matters is often past the cut. - subject = ev.FullCommand() - switch r.Match.Scope { - case "shell": - subject = shellshape.Strip(shellshape.ExpandVars(subject)) - default: - // Quoted arguments stay (SQL in `psql -c '…'` is the - // command); heredoc bodies — files being written, edit - // scripts full of string literals — go. - subject = shellshape.StripAllHeredocs(subject) - } - case "summary": - subject = ev.Summary - } + subject := subj.text(kind, i) if subject == "" { continue } evidence := fmt.Sprintf("%s:%d (artifact %.12s)", ev.SourcePath, ev.SourceLine, ev.SourceArtifact) - if matches(r, subject) { + if matchesPrepared(r, subject, subj.lower(kind, i), subj.folded(kind, i)) && !excluded(r, subject) { if r.Match.SkipScratchTarget && r.targetRe != nil && onlyScratchTargets(r.targetRe, shellshape.ExpandVars(ev.FullCommand())) { continue } @@ -311,7 +307,7 @@ func artifactClass(a casepkg.ArtifactRecord) string { // per artifact too, not once per rule. Binaries are skipped: a content // rule's phrase or regex inside a .pptx or a node_modules blob is noise, // and the other content rules already stay off them. -func matchArtifacts(rules []*Rule, man *casepkg.Manifest, store *casepkg.Store) []schema.Finding { +func matchArtifacts(rules []*Rule, man *casepkg.Manifest, store *casepkg.Store, memo *artmemo.Memo[[]schema.Finding]) []schema.Finding { cur := man.Current() results := make([][]schema.Finding, len(cur)) var wg sync.WaitGroup @@ -321,7 +317,15 @@ func matchArtifacts(rules []*Rule, man *casepkg.Manifest, store *casepkg.Store) go func() { defer wg.Done() for i := range next { + // An artifact's matches depend only on its bytes, its + // record and the rules, so an unchanged artifact is not + // read again (see internal/artmemo). + if f, ok := memo.Get(cur[i]); ok { + results[i] = f + continue + } results[i] = matchOneArtifact(rules, cur[i], store) + memo.Put(cur[i], results[i]) } }() } @@ -363,13 +367,16 @@ func matchOneArtifact(rules []*Rule, a casepkg.ArtifactRecord, store *casepkg.St } artifactReads.Add(1) s := string(data) - low := "" + low, folded := "", "" var out []schema.Finding for _, r := range applicable { if len(r.Match.Contains) > 0 && low == "" { low = strings.ToLower(s) } - if !matchesPrepared(r, s, low) { + if r.need != nil && folded == "" { + folded = fold(s) + } + if !matchesPrepared(r, s, low, folded) { continue } out = append(out, finding(r, "", "", schema.StateObserved, @@ -379,20 +386,28 @@ func matchOneArtifact(rules []*Rule, a casepkg.ArtifactRecord, store *casepkg.St } func matches(r *Rule, s string) bool { - if r.excludeRe != nil && r.excludeRe.MatchString(s) { + if excluded(r, s) { return false } - low := "" + low, folded := "", "" if len(r.Match.Contains) > 0 { low = strings.ToLower(s) } - return matchesPrepared(r, s, low) + if r.need != nil { + folded = fold(s) + } + return matchesPrepared(r, s, low, folded) +} + +func excluded(r *Rule, s string) bool { + return r.excludeRe != nil && r.excludeRe.MatchString(s) } -// matchesPrepared is matches with the lowercased subject supplied by the -// caller, so one artifact is lowercased once for all its rules. -func matchesPrepared(r *Rule, s, low string) bool { - if r.re != nil && r.re.MatchString(s) { +// matchesPrepared is matches with the lowercased and case-folded subject +// supplied by the caller, so a subject is prepared once for all rules. +// folded may be "" only when the rule has no prefilter. +func matchesPrepared(r *Rule, s, low, folded string) bool { + if r.re != nil && (r.need == nil || mayMatch(r.need, folded)) && r.re.MatchString(s) { return true } for _, c := range r.Match.Contains { @@ -403,6 +418,22 @@ func matchesPrepared(r *Rule, s, low string) bool { return false } +// ArtifactMemoHits is how many artifacts the last Apply served from the +// memo instead of scanning (for tests and progress reporting). +var ArtifactMemoHits int + +// rulesDigest identifies a rule set by its full definitions, so remembered +// results are never reused under a rule that changed. +func rulesDigest(rules []*Rule) string { + h := sha256.New() + for _, r := range rules { + data, _ := json.Marshal(r) + h.Write(data) + h.Write([]byte{'\n'}) + } + return hex.EncodeToString(h.Sum(nil)) +} + func finding(r *Rule, session, agent, status, evidence string) schema.Finding { return schema.Finding{ RuleID: r.ID, diff --git a/internal/rulepack/subjects.go b/internal/rulepack/subjects.go new file mode 100644 index 0000000..4572dd1 --- /dev/null +++ b/internal/rulepack/subjects.go @@ -0,0 +1,88 @@ +package rulepack + +import ( + "strings" + + "github.com/efij/AgentDFIR/v3/internal/schema" + "github.com/efij/AgentDFIR/v3/internal/shellshape" +) + +// subjects prepares each event's match subject once for every rule that +// reads it, instead of once per rule. Deriving a command's subject strips +// heredocs or quoting and expands variables, and the rules also want it +// lowercased (contains) and case-folded (the regex prefilter). Doing that +// for each of 81 rules over 359,000 events was most of the rule packs' +// minute on a real case; the results are identical because the subject is +// a function of the event and the rule's scope alone. +type subjects struct { + events []schema.Event + text_ [kinds][]string + lower_ [kinds][]string + fold_ [kinds][]string +} + +// Subject kinds: what a rule's match type and scope make it read. +const ( + kindCommand = iota // full command, heredoc bodies removed + kindCommandShell // full command as the shell runs it + kindSummary + kinds +) + +func newSubjects(events []schema.Event) *subjects { return &subjects{events: events} } + +func subjectKind(r *Rule) int { + switch { + case r.Match.Type == "summary": + return kindSummary + case r.Match.Scope == "shell": + return kindCommandShell + } + return kindCommand +} + +func (s *subjects) text(kind, i int) string { + if s.text_[kind] == nil { + out := make([]string, len(s.events)) + for j := range s.events { + ev := &s.events[j] + switch kind { + case kindSummary: + out[j] = ev.Summary + case kindCommandShell: + // The full command, not the 300-character display copy: + // the flag or payload that matters is often past the cut. + out[j] = shellshape.Strip(shellshape.ExpandVars(ev.FullCommand())) + default: + // Quoted arguments stay (SQL in `psql -c '…'` is the + // command); heredoc bodies — files being written, edit + // scripts full of string literals — go. + out[j] = shellshape.StripAllHeredocs(ev.FullCommand()) + } + } + s.text_[kind] = out + } + return s.text_[kind][i] +} + +func (s *subjects) lower(kind, i int) string { + if s.lower_[kind] == nil { + out := make([]string, len(s.events)) + for j := range s.events { + out[j] = strings.ToLower(s.text(kind, j)) + } + s.lower_[kind] = out + } + return s.lower_[kind][i] +} + +func (s *subjects) folded(kind, i int) string { + if s.fold_[kind] == nil { + out := make([]string, len(s.events)) + for j := range s.events { + out[j] = fold(s.text(kind, j)) + } + s.fold_[kind] = out + } + return s.fold_[kind][i] +} diff --git a/internal/seal/seal.go b/internal/seal/seal.go index 49383f0..bc86af4 100644 --- a/internal/seal/seal.go +++ b/internal/seal/seal.go @@ -220,3 +220,45 @@ func VerifyFileSig(dataPath, sigPath, trustedPubPath string) error { } return nil } + +// SigFile is the signature file's name inside a package. +const SigFile = sigFile + +// LoadPrivateKey reads an ed25519 private key written by GenerateKey. +func LoadPrivateKey(path string) (ed25519.PrivateKey, error) { + priv, err := readKey(path, privBlock) + if err != nil { + return nil, err + } + if len(priv) != ed25519.PrivateKeySize { + return nil, errors.New("invalid private key size") + } + return ed25519.PrivateKey(priv), nil +} + +// PublicKeyHex is the hex public key of a private key file, the form +// SEAL.sig records and Verify pins. +func PublicKeyHex(keyPath string) (string, error) { + priv, err := LoadPrivateKey(keyPath) + if err != nil { + return "", err + } + return hex.EncodeToString(priv.Public().(ed25519.PublicKey)), nil +} + +// Fingerprint shortens a hex public key for display: the first 16 hex +// digits of its sha256. +func Fingerprint(pubHex string) string { + sum := sha256.Sum256([]byte(pubHex)) + return hex.EncodeToString(sum[:])[:16] +} + +// Digest is the sha256 of the package's SHA256SUMS — the value a signature +// covers and an anchor records. +func Digest(pkgDir string) (string, error) { + d, err := sumsDigest(pkgDir) + if err != nil { + return "", err + } + return hex.EncodeToString(d), nil +} diff --git a/internal/serve/serve_test.go b/internal/serve/serve_test.go index 56d35a9..4d9b75d 100644 --- a/internal/serve/serve_test.go +++ b/internal/serve/serve_test.go @@ -15,6 +15,7 @@ import ( "github.com/efij/AgentDFIR/v3/internal/casepkg" "github.com/efij/AgentDFIR/v3/internal/collector" + "github.com/efij/AgentDFIR/v3/internal/normalize" "github.com/efij/AgentDFIR/v3/internal/overlay" "github.com/efij/AgentDFIR/v3/internal/products" ) @@ -251,6 +252,13 @@ func TestServesEveryEventPastTheOldCap(t *testing.T) { t.Fatal(err) } f.Close() + // The padding stands in for a large case's own overlay, so record it + // the way a stage that annotates events does. Without this the overlay + // is (correctly) treated as changed outside AgentDFIR and rebuilt from + // the sealed evidence, which drops the padding. + if err := normalize.RecordEvents(pkg); err != nil { + t.Fatal(err) + } // MaxEvents is what the old cap came in through: it must no longer be // able to hide anything. diff --git a/internal/store/keys.go b/internal/store/keys.go new file mode 100644 index 0000000..e517595 --- /dev/null +++ b/internal/store/keys.go @@ -0,0 +1,208 @@ +package store + +import ( + "bufio" + "crypto/hmac" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/efij/AgentDFIR/v3/internal/hashchain" + "github.com/efij/AgentDFIR/v3/internal/seal" +) + +// The machine sealing key. +// +// Every round a machine seals is signed with a key that lives in the +// AgentDFIR home, not in the case. That is what lets the next round prove +// the earlier ones were not rewritten before it builds on them: without a +// signature, anyone who can write the case can regenerate SHA256SUMS and +// both hash chains from scratch and every check passes. +// +// Its limit is stated in SECURITY.md: an attacker who controls this +// account also controls this key. The anchor log and the seal digest +// printed at the end of every run are what survive that. + +const ( + keysDir = "keys" + sealKey = "seal.ed25519" + sealPub = "seal.ed25519.pub" + anchorFile = "anchors.jsonl" +) + +// MachineKey returns this machine's sealing key, creating it on first use. +func MachineKey() (string, error) { + home, err := Home() + if err != nil { + return "", err + } + dir := filepath.Join(home, keysDir) + if err := ensureSecureDir(dir); err != nil { + return "", err + } + priv := filepath.Join(dir, sealKey) + if _, err := os.Lstat(priv); errors.Is(err, os.ErrNotExist) { + if err := seal.GenerateKey(priv, filepath.Join(dir, sealPub)); err != nil { + return "", fmt.Errorf("create machine key: %w", err) + } + } + if err := checkKeyFile(priv); err != nil { + return "", err + } + return priv, nil +} + +// MachinePublicKey returns the machine key's public half (hex), or "" when +// this machine has none yet. It never creates anything. +func MachinePublicKey() string { + priv, ok := existingKey() + if !ok { + return "" + } + pub, err := seal.PublicKeyHex(priv) + if err != nil { + return "" + } + return pub +} + +// CacheKey is the key that authenticates derived caches (the normalized +// overlay's state), derived from the machine key. It is nil when the +// machine has no key; it never creates one, so opening a case somewhere +// else leaves no trace in that machine's home. +func CacheKey() []byte { + priv, ok := existingKey() + if !ok { + return nil + } + k, err := seal.LoadPrivateKey(priv) + if err != nil { + return nil + } + m := hmac.New(sha256.New, k.Seed()) + m.Write([]byte("agentdfir derived-cache authentication v1")) + return m.Sum(nil) +} + +func existingKey() (string, bool) { + home := os.Getenv(EnvHome) + if home == "" { + b, err := defaultBase() + if err != nil { + return "", false + } + home = b + } + priv := filepath.Join(home, keysDir, sealKey) + if checkKeyFile(priv) != nil { + return "", false + } + return priv, true +} + +// checkKeyFile refuses a key that is a symlink, not a regular file, owned +// by someone else, or readable by anyone but its owner. +func checkKeyFile(path string) error { + fi, err := os.Lstat(path) + if err != nil { + return err + } + if !fi.Mode().IsRegular() { + return fmt.Errorf("%s is not a regular file; refusing to use it as a signing key", path) + } + if err := checkOwner(path, fi); err != nil { + return err + } + if PermissionsAreReal && fi.Mode().Perm()&0o077 != 0 { + return fmt.Errorf("%s is accessible to other users (%o); refusing to sign with it", path, fi.Mode().Perm()) + } + return nil +} + +// Anchor is one sealed round, recorded outside the case. +type Anchor struct { + CaseID string `json:"case_id"` + Round int `json:"round"` + SealsDigest string `json:"seals_digest"` + Signer string `json:"signer,omitempty"` + Package string `json:"package,omitempty"` + TimeUTC string `json:"ts_utc"` +} + +// RecordAnchor appends a sealed round to the anchor log, a hash chain of +// its own in the AgentDFIR home. Rewriting a case then means rewriting +// this file too, consistently — and the digest printed at the end of the +// run, pasted into a ticket, is out of reach altogether. +func RecordAnchor(a Anchor) error { + home, err := Home() + if err != nil { + return err + } + path := filepath.Join(home, anchorFile) + var w *hashchain.Writer + if _, err := os.Lstat(path); errors.Is(err, os.ErrNotExist) { + w, err = hashchain.NewWriter(path) + if err != nil { + return err + } + } else if w, err = hashchain.NewAppender(path); err != nil { + return fmt.Errorf("anchor log: %w", err) + } + if a.TimeUTC == "" { + a.TimeUTC = time.Now().UTC().Format(time.RFC3339) + } + rec := map[string]any{ + "event": "round_sealed", "case_id": a.CaseID, "round": a.Round, + "seals_digest": a.SealsDigest, "signer": a.Signer, "package": a.Package, "ts_utc": a.TimeUTC, + } + if err := w.Append(rec); err != nil { + w.Close() + return err + } + return w.Close() +} + +// LatestAnchor returns the newest anchor recorded for a case at a given +// package path, or nil when there is none (or no anchor log). The path is +// part of the key because a copied case legitimately diverges from the +// original. An anchor log whose own chain is broken is an error: it is +// exactly the file an attacker would edit. +func LatestAnchor(caseID, pkgPath string) (*Anchor, error) { + home := os.Getenv(EnvHome) + if home == "" { + b, err := defaultBase() + if err != nil { + return nil, nil + } + home = b + } + path := filepath.Join(home, anchorFile) + f, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + defer f.Close() + if _, err := hashchain.Verify(f); err != nil { + return nil, fmt.Errorf("anchor log %s: %w", path, err) + } + if _, err := f.Seek(0, 0); err != nil { + return nil, err + } + var latest *Anchor + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 0, 64<<10), 1<<20) + for sc.Scan() { + var a Anchor + if json.Unmarshal(sc.Bytes(), &a) == nil && a.CaseID == caseID && a.Package == pkgPath { + latest = &a + } + } + return latest, sc.Err() +}