From d0bcb389b9bcee55c49daa3ab963a06d81942b7d Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Sun, 30 Aug 2026 10:01:38 +0100 Subject: [PATCH 01/14] Add Lean disaster recovery model Add an executable Lean port of the Stateright model, a C++-aligned canonical model with refinement and temporal proofs, and a versioned implementation trace validator.\n\nAdd exhaustive bounded Rust/Lean graph comparison and CI coverage while retaining Stateright as the migration oracle.\n\nCo-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>\nCopilot-Session: 46d56f8a-935f-4551-abe3-84bd4f951865 --- .github/workflows/README.md | 9 + .github/workflows/ci-verification.yml | 35 + .github/workflows/lean-shallow.yml | 78 ++ lean/disaster-recovery/.gitignore | 1 + lean/disaster-recovery/CanonicalTests.lean | 147 ++++ lean/disaster-recovery/DisasterRecovery.lean | 6 + .../DisasterRecovery/Checker.lean | 152 ++++ .../DisasterRecovery/Model.lean | 392 +++++++++ .../DisasterRecovery/Protocol/Model.lean | 286 +++++++ .../DisasterRecovery/Protocol/Refinement.lean | 333 ++++++++ .../DisasterRecovery/Protocol/Temporal.lean | 231 ++++++ .../DisasterRecovery/Protocol/Trace.lean | 772 ++++++++++++++++++ lean/disaster-recovery/Main.lean | 36 + lean/disaster-recovery/README.md | 396 +++++++++ lean/disaster-recovery/TRACE_FORMAT_V1.md | 145 ++++ lean/disaster-recovery/Tests.lean | 72 ++ lean/disaster-recovery/TraceMain.lean | 23 + lean/disaster-recovery/TraceTests.lean | 394 +++++++++ lean/disaster-recovery/compare.py | 343 ++++++++ .../fixtures/accepted-failover.ndjson | 8 + .../fixtures/accepted-multinode.ndjson | 5 + .../fixtures/accepted.ndjson | 10 + .../fixtures/rejected-cause.ndjson | 4 + .../fixtures/rejected.ndjson | 2 + lean/disaster-recovery/lake-manifest.json | 116 +++ lean/disaster-recovery/lakefile.toml | 38 + lean/disaster-recovery/lean-toolchain | 1 + tla/disaster-recovery/Readme.md | 48 ++ tla/disaster-recovery/src/export.rs | 370 +++++++++ tla/disaster-recovery/src/main.rs | 32 +- 30 files changed, 4484 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/lean-shallow.yml create mode 100644 lean/disaster-recovery/.gitignore create mode 100644 lean/disaster-recovery/CanonicalTests.lean create mode 100644 lean/disaster-recovery/DisasterRecovery.lean create mode 100644 lean/disaster-recovery/DisasterRecovery/Checker.lean create mode 100644 lean/disaster-recovery/DisasterRecovery/Model.lean create mode 100644 lean/disaster-recovery/DisasterRecovery/Protocol/Model.lean create mode 100644 lean/disaster-recovery/DisasterRecovery/Protocol/Refinement.lean create mode 100644 lean/disaster-recovery/DisasterRecovery/Protocol/Temporal.lean create mode 100644 lean/disaster-recovery/DisasterRecovery/Protocol/Trace.lean create mode 100644 lean/disaster-recovery/Main.lean create mode 100644 lean/disaster-recovery/README.md create mode 100644 lean/disaster-recovery/TRACE_FORMAT_V1.md create mode 100644 lean/disaster-recovery/Tests.lean create mode 100644 lean/disaster-recovery/TraceMain.lean create mode 100644 lean/disaster-recovery/TraceTests.lean create mode 100644 lean/disaster-recovery/compare.py create mode 100644 lean/disaster-recovery/fixtures/accepted-failover.ndjson create mode 100644 lean/disaster-recovery/fixtures/accepted-multinode.ndjson create mode 100644 lean/disaster-recovery/fixtures/accepted.ndjson create mode 100644 lean/disaster-recovery/fixtures/rejected-cause.ndjson create mode 100644 lean/disaster-recovery/fixtures/rejected.ndjson create mode 100644 lean/disaster-recovery/lake-manifest.json create mode 100644 lean/disaster-recovery/lakefile.toml create mode 100644 lean/disaster-recovery/lean-toolchain create mode 100644 tla/disaster-recovery/src/export.rs diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 4eb030772236..8d0da5157804 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -101,6 +101,15 @@ Runs on pull requests that change `tla/` or `src/consensus/aft/raft.h`. File: `tla-shallow.yml` 3rd party dependencies: None +# Lean Shallow Verification + +Builds and checks the Lean disaster-recovery models, validates trace fixtures, +and compares the bounded Lean legacy model with Stateright on relevant pull +requests. + +File: `lean-shallow.yml` +3rd party dependencies: None + # Vendored Dependency Verification Verifies that files under `3rdparty/` match the Git commits or release artifacts diff --git a/.github/workflows/ci-verification.yml b/.github/workflows/ci-verification.yml index 1a72ca4feb95..3cda3abeb8a2 100644 --- a/.github/workflows/ci-verification.yml +++ b/.github/workflows/ci-verification.yml @@ -264,3 +264,38 @@ jobs: tdnf install -y cargo - run: cd tla/disaster-recovery && cargo run check + + model-checking-self-healing-open-lean: + name: Model Checking - Self-Healing Open (Lean) + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install Lean + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y elan + elan toolchain install "$(cat lean/disaster-recovery/lean-toolchain)" + + - name: Restore Mathlib cache + working-directory: lean/disaster-recovery + shell: bash + run: | + set -euo pipefail + lake exe cache get + + - name: Check Lean model and full legacy equivalence + working-directory: lean/disaster-recovery + shell: bash + run: | + set -euo pipefail + lake build + lake exe semantic-checks + lake exe canonical-checks + lake exe trace-checks + lake exe disaster-recovery check --nodes 3 + python3 compare.py --nodes 1 2 3 diff --git a/.github/workflows/lean-shallow.yml b/.github/workflows/lean-shallow.yml new file mode 100644 index 000000000000..3e2e72593c52 --- /dev/null +++ b/.github/workflows/lean-shallow.yml @@ -0,0 +1,78 @@ +name: "Lean Shallow Verification" + +on: + pull_request: + paths: + - "lean/**" + - "tla/disaster-recovery/**" + - "include/ccf/node/startup_config.h" + - "include/ccf/service/tables/self_healing_open.h" + - "src/node/recovery_decision_protocol.cpp" + - "src/node/recovery_decision_protocol.h" + - "src/node/rpc/self_healing_open_handlers.h" + - ".github/workflows/lean-shallow.yml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: read-all + +jobs: + disaster-recovery: + name: Disaster Recovery + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install Lean + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y elan + elan toolchain install "$(cat lean/disaster-recovery/lean-toolchain)" + + - name: Restore Mathlib cache + working-directory: lean/disaster-recovery + shell: bash + run: | + set -euo pipefail + lake exe cache get + + - name: Build and check Lean models + working-directory: lean/disaster-recovery + shell: bash + run: | + set -euo pipefail + lake build + lake exe semantic-checks + lake exe canonical-checks + lake exe trace-checks + lake exe disaster-recovery check --nodes 3 + + - name: Check trace validation + working-directory: lean/disaster-recovery + shell: bash + run: | + set -euo pipefail + lake exe trace-validator fixtures/accepted.ndjson + lake exe trace-validator fixtures/accepted-failover.ndjson + lake exe trace-validator fixtures/accepted-multinode.ndjson + if lake exe trace-validator fixtures/rejected.ndjson; then + echo "Rejected state trace was accepted" + exit 1 + fi + if lake exe trace-validator fixtures/rejected-cause.ndjson; then + echo "Rejected causal trace was accepted" + exit 1 + fi + + - name: Compare Lean and Stateright + working-directory: lean/disaster-recovery + shell: bash + run: | + set -euo pipefail + python3 compare.py --nodes 1 2 diff --git a/lean/disaster-recovery/.gitignore b/lean/disaster-recovery/.gitignore new file mode 100644 index 000000000000..4080d07dfc31 --- /dev/null +++ b/lean/disaster-recovery/.gitignore @@ -0,0 +1 @@ +/.lake/ diff --git a/lean/disaster-recovery/CanonicalTests.lean b/lean/disaster-recovery/CanonicalTests.lean new file mode 100644 index 000000000000..8d7fc46722c6 --- /dev/null +++ b/lean/disaster-recovery/CanonicalTests.lean @@ -0,0 +1,147 @@ +import DisasterRecovery.Protocol.Refinement +import DisasterRecovery.Protocol.Temporal +import DisasterRecovery.Protocol.Trace + +open DisasterRecovery.Protocol + +private def expect (condition : Bool) (message : String) : IO Unit := + unless condition do throw (IO.userError message) + +private def eventsFor (config : Config) : List Event := + let messages := config.expectedLocations.flatMap fun source => + [ + .receiveGossip source { view := 0, seqno := source.length } .accepted, + .receiveGossip source { view := 0, seqno := source.length } .rejected, + .receiveVote source .accepted, + .receiveVote source .rejected, + .receiveIAmOpen source .accepted, + .receiveIAmOpen source .rejected + ] + messages ++ [.timeout, .retry] + +private def invariant (state : NodeState) : Bool := + let chosenReady := + if state.phase == .voting then state.chosen.isSome else true + let openingKind := + if state.phase == .opening || state.phase == .open then + state.openKind.isSome + else + true + let restartOnlyJoining := + if state.restartRequested then state.phase == .joining else true + chosenReady && openingKind && restartOnlyJoining + +private def enumerate (config : Config) (location : Location) : IO (Prod Nat Nat) := do + let initial := initialNode location + let mut states := #[initial] + let mut seen : Std.HashMap String Nat := {} + seen := seen.insert (stateKey initial) 0 + let mut cursor := 0 + let mut edges := 0 + while cursor < states.size do + let state := states[cursor]! + expect (invariant state) s!"canonical invariant failed: {stateKey state}" + for event in eventsFor config do + let next := (step config state event).state + edges := edges + 1 + let key := stateKey next + if !seen.contains key then + seen := seen.insert key states.size + states := states.push next + cursor := cursor + 1 + pure (states.size, edges) + +def main : IO UInt32 := do + let config : Config := { + instanceId := "canonical-tests" + expectedLocations := ["A", "B"] + } + expect config.isValid "canonical test configuration is invalid" + expect + (!({ instanceId := "invalid", expectedLocations := ["A", "A"] } : + Config).isValid) + "duplicate expected locations were accepted" + expect (voteQuorum config == 2) "two-node strict majority must be two" + + let initial := initialNode "A" + expect initial.gossips.isEmpty "canonical C++ state must start without gossip" + + let first := step config initial + (.receiveGossip "A" { view := 1, seqno := 10 } .accepted) + expect (first.state.phase == .gossiping) "one of two gossips advanced early" + let duplicate := step config first.state + (.receiveGossip "A" { view := 99, seqno := 99 } .accepted) + expect (duplicate.state == first.state) + "duplicate gossip source changed its recorded TxID" + let second := step config first.state + (.receiveGossip "B" { view := 2, seqno := 1 } .accepted) + expect (second.state.phase == .voting) "all expected gossips did not advance" + expect (second.state.chosen == some "B") "full TxID maximum was not chosen" + + let tiedA := step config initial + (.receiveGossip "A" { view := 2, seqno := 1 } .accepted) + let tiedB := step config tiedA.state + (.receiveGossip "B" { view := 2, seqno := 1 } .accepted) + expect (tiedB.state.chosen == some "B") + "location name did not break an equal TxID tie lexicographically" + + let frozen := step config second.state + (.receiveGossip "C" { view := 9, seqno := 9 } .accepted) + expect (!frozen.accepted && frozen.state == second.state) + "gossip did not freeze after choosing a node" + + let oneVote := step config second.state (.receiveVote "A" .accepted) + expect (oneVote.state.phase == .voting) "even-node quorum used legacy threshold" + let twoVotes := step config oneVote.state (.receiveVote "B" .accepted) + expect (twoVotes.state.phase == .opening) "strict voting quorum did not open" + expect (twoVotes.state.openKind == some .quorum) "quorum path mislabeled" + + let emptyVoting := { + initial with + phase := .voting + timeoutState := .voting + chosen := some "A" + } + let noVotes := step config emptyVoting .timeout + expect (noVotes.state == emptyVoting) + "aligned voting timeout with zero votes advanced" + + let oneVoteWaiting := { emptyVoting with votes := ["A"] } + let failover := step config oneVoteWaiting .timeout + expect (failover.state.phase == .opening) "failover vote did not open" + expect (failover.state.openKind == some .failover) "failover path mislabeled" + + let opening := { + twoVotes.state with + timeoutState := .opening + } + let complete := step config opening .timeout + expect (complete.state.phase == .open) "Opening timeout did not reach Open" + + let joining := step config initial + (.receiveIAmOpen "B" .accepted) + expect (joining.state.phase == .joining && joining.state.restartRequested) + "IAmOpen did not request joining restart" + + let retry := step config second.state .retry + expect + (retry.effects == + [.sendVote "B", .sendGossip "A", .sendGossip "B"]) + "Voting retry did not send vote before continuing gossip" + + let unexpectedConfig : Config := { + instanceId := "unexpected" + expectedLocations := ["A"] + } + let unexpected := step unexpectedConfig (initialNode "A") + (.receiveGossip "OUTSIDE" { view := 1, seqno := 1 } .accepted) + expect (unexpected.state.phase == .voting) + "model no longer exposes C++ acceptance of unexpected validated locations" + + let (oneStates, oneEdges) <- enumerate + { instanceId := "n1", expectedLocations := ["A"] } "A" + let (twoStates, twoEdges) <- enumerate config "A" + IO.println s!"canonical n=1: {oneStates} states, {oneEdges} event edges" + IO.println s!"canonical n=2: {twoStates} states, {twoEdges} event edges" + IO.println "all canonical semantic and proof checks passed" + pure 0 diff --git a/lean/disaster-recovery/DisasterRecovery.lean b/lean/disaster-recovery/DisasterRecovery.lean new file mode 100644 index 000000000000..a414ad18aebe --- /dev/null +++ b/lean/disaster-recovery/DisasterRecovery.lean @@ -0,0 +1,6 @@ +import DisasterRecovery.Model +import DisasterRecovery.Checker +import DisasterRecovery.Protocol.Model +import DisasterRecovery.Protocol.Temporal +import DisasterRecovery.Protocol.Refinement +import DisasterRecovery.Protocol.Trace diff --git a/lean/disaster-recovery/DisasterRecovery/Checker.lean b/lean/disaster-recovery/DisasterRecovery/Checker.lean new file mode 100644 index 000000000000..58705a610279 --- /dev/null +++ b/lean/disaster-recovery/DisasterRecovery/Checker.lean @@ -0,0 +1,152 @@ +import DisasterRecovery.Model + +namespace DisasterRecovery + +structure Edge where + src : Nat + action : Action + dst : Nat +deriving Repr, BEq + +structure Graph where + states : Array GlobalState + edges : Array Edge + parents : Array (Option (Prod Nat Action)) + +def enumerate (n : Nat) : IO Graph := do + let initial := initialState n + let mut states := #[initial] + let mut edges := #[] + let mut parents : Array (Option (Prod Nat Action)) := #[none] + let mut seen : Std.HashMap String Nat := {} + seen := seen.insert (stateKey initial) 0 + let mut cursor := 0 + while cursor < states.size do + let state := states[cursor]! + for action in actions state do + match nextState n state action with + | none => pure () + | some next => + let key := stateKey next + let (dst, discovered) := + match seen[key]? with + | some index => (index, false) + | none => (states.size, true) + if discovered then + seen := seen.insert key dst + states := states.push next + parents := parents.push (some (cursor, action)) + edges := edges.push { src := cursor, action, dst } + cursor := cursor + 1 + pure { states, edges, parents } + +def valuationBits (values : Array Bool) : String := + String.ofList (values.toList.map fun value => if value then '1' else '0') + +private structure ExportEdge where + src : Nat + action : String + dst : Nat + +private def exportEdgeLE (left right : ExportEdge) : Bool := + left.src < right.src || + (left.src == right.src && + (left.action < right.action || + (left.action == right.action && left.dst <= right.dst))) + +private def traceTo (graph : Graph) (target : Nat) : List Action := + let rec collect (index : Nat) (fuel : Nat) (suffix : List Action) : List Action := + match fuel with + | 0 => suffix + | fuel + 1 => + match graph.parents[index]? |>.bind id with + | none => suffix + | some (parent, action) => collect parent fuel (action :: suffix) + collect target graph.states.size [] + +private def printTrace (graph : Graph) (target : Nat) : IO Unit := do + let trace := traceTo graph target + if trace.isEmpty then + IO.eprintln " trace: " + else + for (action, step) in trace.zipIdx do + IO.eprintln s!" {step + 1}. {actionKey action}" + +private def eventuallyGood (graph : Graph) (property : Nat) : Array Bool := + Id.run do + let mut good := + graph.states.map fun state => (legacyValuations state.actors.size state)[property]! + let mut remaining := Array.replicate graph.states.size 0 + let mut predecessors : Array (List Nat) := Array.replicate graph.states.size [] + for edge in graph.edges do + remaining := remaining.modify edge.src (fun count => count + 1) + predecessors := predecessors.modify edge.dst (fun values => edge.src :: values) + let mut queue := #[] + for index in List.range good.size do + if good[index]! then queue := queue.push index + let mut cursor := 0 + while cursor < queue.size do + let resolved := queue[cursor]! + for predecessor in predecessors[resolved]! do + if !good[predecessor]! then + remaining := remaining.modify predecessor (fun count => count - 1) + if remaining[predecessor]! == 0 then + good := good.set! predecessor true + queue := queue.push predecessor + cursor := cursor + 1 + return good + +def checkGraph (n : Nat) (graph : Graph) : IO Bool := do + IO.eprintln s!"reachable states: {graph.states.size}, transitions: {graph.edges.size}" + let mut passed := true + for property in List.range legacyPropertyNames.size do + let name := legacyPropertyNames[property]! + let expectation := legacyExpectations[property]! + let values := graph.states.map fun state => (legacyValuations n state)[property]! + let eventual := if expectation == "eventually" then eventuallyGood graph property else #[] + let result := + if expectation == "always" then values.all id + else if expectation == "sometimes" then values.any id + else eventual[0]! + IO.eprintln s!"{if result then "PASS" else "FAIL"} [{expectation}] {name}" + if result && expectation == "sometimes" then + match (List.range values.size).find? (fun index => values[index]!) with + | none => pure () + | some index => + IO.eprintln " shortest example:" + printTrace graph index + else if !result then + passed := false + let witness := + if expectation == "always" then + (List.range values.size).find? fun index => !values[index]! + else if expectation == "sometimes" then + some 0 + else + (List.range values.size).find? fun index => + !eventual[index]! + match witness with + | none => IO.eprintln " no reachable example" + | some index => printTrace graph index + pure passed + +def exportGraph (n : Nat) (graph : Graph) : IO Unit := do + let canonical := (graph.states.toList.zipIdx.map fun (state, bfsId) => + (stateKey state, bfsId)).mergeSort (fun left right => left.1 <= right.1) + let mut ids := Array.replicate graph.states.size 0 + for ((_, bfsId), canonicalId) in canonical.zipIdx do + ids := ids.set! bfsId canonicalId + IO.println "format\tccf-legacy-dr-graph-v1" + IO.println s!"nodes\t{n}" + IO.println s!"init\t{ids[0]!}" + for ((key, bfsId), canonicalId) in canonical.zipIdx do + IO.println s!"state\t{canonicalId}\t{key}\t{valuationBits (legacyValuations n graph.states[bfsId]!)}" + let canonicalEdges := (graph.edges.toList.map fun edge => { + src := ids[edge.src]! + action := actionKey edge.action + dst := ids[edge.dst]! + }).mergeSort exportEdgeLE + for edge in canonicalEdges do + IO.println s!"edge\t{edge.src}\t{edge.action}\t{edge.dst}" + +end DisasterRecovery diff --git a/lean/disaster-recovery/DisasterRecovery/Model.lean b/lean/disaster-recovery/DisasterRecovery/Model.lean new file mode 100644 index 000000000000..8dcd3e44c71a --- /dev/null +++ b/lean/disaster-recovery/DisasterRecovery/Model.lean @@ -0,0 +1,392 @@ +import Std + +namespace DisasterRecovery + +abbrev Id := Nat +abbrev Txid := Nat + +structure Gossip where + src : Id + txid : Txid +deriving Repr, BEq, Hashable + +structure Vote where + src : Id + recv : List Gossip +deriving Repr, BEq, Hashable + +inductive Msg where + | gossip (value : Gossip) + | vote (value : Vote) + | iAmOpen (src : Id) +deriving Repr, BEq, Hashable + +inductive Phase where + | vote + | openJoin + | open (timeout : Bool) + | join +deriving Repr, BEq, Hashable, Inhabited + +structure ActorState where + nextStep : Phase + gossips : List Gossip + votes : List Vote + submittedVote : Option (Prod Id Vote) + txid : Txid +deriving Repr, BEq, Hashable, Inhabited + +structure Envelope where + src : Id + dst : Id + msg : Msg +deriving Repr, BEq, Hashable + +structure GlobalState where + actors : Array ActorState + timers : Array Bool + network : List Envelope +deriving Repr, BEq, Hashable, Inhabited + +inductive Action where + | deliver (envelope : Envelope) + | timeout (id : Id) +deriving Repr, BEq, Hashable + +structure Output where + sent : List (Prod Id Msg) := [] + setTimer : Bool := false +deriving Repr, BEq + +private def comma (values : List String) : String := + String.intercalate "," values + +def gossipKey (gossip : Gossip) : String := + s!"g({gossip.src},{gossip.txid})" + +def voteKey (vote : Vote) : String := + s!"v({vote.src},[{comma (vote.recv.map gossipKey)}])" + +def msgKey : Msg -> String + | .gossip gossip => gossipKey gossip + | .vote vote => voteKey vote + | .iAmOpen src => s!"o({src})" + +def envelopeKey (envelope : Envelope) : String := + s!"e({envelope.src},{envelope.dst},{msgKey envelope.msg})" + +def phaseKey : Phase -> String + | .vote => "vote" + | .openJoin => "openjoin" + | .open false => "open0" + | .open true => "open1" + | .join => "join" + +def submittedKey : Option (Prod Id Vote) -> String + | none => "none" + | some (dst, vote) => s!"some({dst},{voteKey vote})" + +def actorKey (actor : ActorState) : String := + s!"s({phaseKey actor.nextStep},[{comma (actor.gossips.map gossipKey)}],[{comma (actor.votes.map voteKey)}],{submittedKey actor.submittedVote},{actor.txid})" + +private def networkRunsFrom (current : Envelope) (count : Nat) : + List Envelope -> List (Prod Envelope Nat) + | [] => [(current, count)] + | head :: tail => + if head == current then + networkRunsFrom current (count + 1) tail + else + (current, count) :: networkRunsFrom head 1 tail + +private def networkRuns : List Envelope -> List (Prod Envelope Nat) + | [] => [] + | head :: tail => networkRunsFrom head 1 tail + +def stateKey (state : GlobalState) : String := + let actors := String.intercalate ";" (state.actors.toList.map actorKey) + let timers := comma (((List.range state.timers.size).filter + (fun id => state.timers[id]!)).map toString) + let network := comma ((networkRuns state.network).map fun (env, count) => + s!"{envelopeKey env}#{count}") + s!"S([{actors}],[{timers}],[{network}])" + +def actionKey : Action -> String + | .deliver env => s!"deliver({env.src},{env.dst},{msgKey env.msg})" + | .timeout id => s!"timeout({id},election)" + +private def insertSorted (before : a -> a -> Bool) (value : a) : List a -> List a + | [] => [value] + | head :: tail => + if before value head then + value :: head :: tail + else + head :: insertSorted before value tail + +private def insertUniqueSorted [BEq a] (before : a -> a -> Bool) (value : a) (values : List a) : + List a := + if values.contains value then values else insertSorted before value values + +private def removeOne [BEq a] (value : a) : List a -> List a + | [] => [] + | head :: tail => if head == value then tail else head :: removeOne value tail + +private def gossipGreater (left right : Gossip) : Bool := + right.txid < left.txid || (right.txid == left.txid && right.src < left.src) + +private def gossipBefore (left right : Gossip) : Bool := + left.src < right.src || (left.src == right.src && left.txid < right.txid) + +private def gossipListBefore : List Gossip -> List Gossip -> Bool + | [], [] => false + | [], _ :: _ => true + | _ :: _, [] => false + | left :: leftTail, right :: rightTail => + if left == right then gossipListBefore leftTail rightTail + else gossipBefore left right + +private def voteBefore (left right : Vote) : Bool := + left.src < right.src || (left.src == right.src && gossipListBefore left.recv right.recv) + +private def msgBefore : Msg -> Msg -> Bool + | .gossip left, .gossip right => gossipBefore left right + | .gossip _, _ => true + | .vote _, .gossip _ => false + | .vote left, .vote right => voteBefore left right + | .vote _, .iAmOpen _ => true + | .iAmOpen _, .gossip _ => false + | .iAmOpen _, .vote _ => false + | .iAmOpen left, .iAmOpen right => left < right + +private def envelopeBefore (left right : Envelope) : Bool := + left.src < right.src || + (left.src == right.src && + (left.dst < right.dst || (left.dst == right.dst && msgBefore left.msg right.msg))) + +private def maximumGossip : List Gossip -> Option Gossip + | [] => none + | head :: tail => + some (tail.foldl (fun current candidate => + if gossipGreater candidate current then candidate else current) head) + +private def voteForMax (gossips : List Gossip) (id : Id) : Option (Prod Id Vote) := do + let maximum <- maximumGossip gossips + pure (maximum.src, { src := id, recv := gossips }) + +private def otherPeers (n id : Nat) : List Id := + (List.range n).filter (fun peer => peer != id) + +private def advanceStep (n id : Nat) (timeout : Bool) (state : ActorState) : + Prod ActorState (Prod Output Bool) := + match state.nextStep with + | .vote => + if state.gossips.length == n || timeout then + match voteForMax state.gossips id with + | none => (state, {}, false) + | some (dst, vote) => + let next := { + state with + nextStep := .openJoin + submittedVote := some (dst, vote) + votes := if dst == id then insertUniqueSorted voteBefore vote state.votes else state.votes + } + let sent := if dst == id then [] else [(dst, Msg.vote vote)] + (next, { sent }, true) + else + (state, {}, false) + | .openJoin => + if state.votes.length >= (n + 1) / 2 || timeout then + let sent := (otherPeers n id).map (fun peer => (peer, Msg.iAmOpen id)) + ({ state with nextStep := .open timeout }, { sent }, true) + else + (state, {}, false) + | _ => (state, {}, false) + +def advanceSeveral (n id : Nat) (timeout : Bool) (state : ActorState) : + Prod ActorState Output := + let (state1, output1, advanced1) := advanceStep n id timeout state + if advanced1 then + let (state2, output2, _) := advanceStep n id timeout state1 + (state2, { sent := output1.sent ++ output2.sent }) + else + (state, {}) + +def onMessage (n id : Nat) (state : ActorState) (msg : Msg) : + Option (Prod ActorState Output) := + let received := + match msg with + | .gossip gossip => + if !state.gossips.contains gossip && state.submittedVote.isNone then + { state with gossips := insertUniqueSorted gossipBefore gossip state.gossips } + else + state + | .vote vote => + { state with votes := insertUniqueSorted voteBefore vote state.votes } + | .iAmOpen _ => + match state.nextStep with + | .open _ => state + | _ => { state with nextStep := .join } + let (next, output) := advanceSeveral n id false received + some (next, output) + +def onTimeout (n id : Nat) (state : ActorState) : Option (Prod ActorState Output) := + match state.nextStep with + | .vote => + if state.gossips.isEmpty then none + else + let (next, output) := advanceSeveral n id true state + some (next, { output with setTimer := true }) + | .openJoin => + if state.votes.isEmpty then none + else some (advanceSeveral n id true state) + | _ => none + +private def applyOutput (src : Id) (output : Output) (state : GlobalState) : GlobalState := + let network := output.sent.foldl + (fun current (dst, msg) => insertSorted envelopeBefore { src, dst, msg } current) + state.network + let timers := if output.setTimer then state.timers.set! src true else state.timers + { state with network, timers } + +private def startActor (n id : Nat) : Prod ActorState Output := + let gossip := { src := id, txid := id } + let initial : ActorState := { + nextStep := .vote + gossips := [gossip] + votes := [] + submittedVote := none + txid := id + } + let output : Output := { + sent := (otherPeers n id).map (fun peer => (peer, Msg.gossip gossip)) + setTimer := true + } + let (state, advanced) := advanceSeveral n id false initial + (state, { sent := output.sent ++ advanced.sent, setTimer := true }) + +def initialState (n : Nat) : GlobalState := + (List.range n).foldl (fun global id => + let (actor, output) := startActor n id + let withActor := { + global with + actors := global.actors.push actor + timers := global.timers.push false + } + applyOutput id output withActor) + { actors := #[], timers := #[], network := [] } + +private def distinctNetworkFrom (previous : Envelope) : List Envelope -> List Envelope + | [] => [] + | head :: tail => + if head == previous then + distinctNetworkFrom previous tail + else + head :: distinctNetworkFrom head tail + +private def distinctNetwork : List Envelope -> List Envelope + | [] => [] + | head :: tail => head :: distinctNetworkFrom head tail + +def actions (state : GlobalState) : List Action := + (distinctNetwork state.network).map Action.deliver ++ + ((List.range state.timers.size).filter + (fun id => state.timers[id]!)).map Action.timeout + +def nextState (n : Nat) (state : GlobalState) : Action -> Option GlobalState + | .deliver envelope => do + let actor <- state.actors[envelope.dst]? + let (nextActor, output) <- onMessage n envelope.dst actor envelope.msg + let delivered := { + state with + actors := state.actors.set! envelope.dst nextActor + network := removeOne envelope state.network + } + pure (applyOutput envelope.dst output delivered) + | .timeout id => do + guard (state.timers[id]?.getD false) + let actor <- state.actors[id]? + let (nextActor, output) <- onTimeout n id actor + let expired := { + state with + actors := state.actors.set! id nextActor + timers := state.timers.set! id false + } + pure (applyOutput id output expired) + +def reachedOpen (state : GlobalState) : Bool := + state.actors.any fun actor => + match actor.nextStep with + | .open _ => true + | _ => false + +def reachedOpenTimeout (state : GlobalState) (expected : Bool) : Bool := + state.actors.any fun actor => actor.nextStep == .open expected + +def unanimousVotes (n : Nat) (state : GlobalState) : Bool := + state.actors.all fun actor => + match actor.submittedVote with + | none => false + | some (_, vote) => + (List.range n).all fun peer => vote.recv.any (fun gossip => gossip.src == peer) + +def majorityHaveSameMaximum (state : GlobalState) : Bool := + let chosen := state.actors.toList.filterMap fun actor => do + let (_, vote) <- actor.submittedVote + let maximum <- maximumGossip vote.recv + pure maximum.src + let chosen := chosen.foldl (fun values id => + insertSorted (fun left right => left < right) id values) [] + let majorityIndex := state.actors.size / 2 + match chosen[majorityIndex]? with + | none => false + | some majority => (chosen.take majorityIndex).all (fun chosen => chosen == majority) + +private def implies (left right : Bool) : Bool := + !left || right + +def legacyValuations (n : Nat) (state : GlobalState) : Array Bool := + let openCount := state.actors.countP fun actor => + match actor.nextStep with + | .open _ => true + | _ => false + let allOpenJoin := state.actors.all (fun actor => actor.nextStep == .openJoin) + let allVotesDelivered := !state.network.any fun envelope => + match envelope.msg with + | .vote _ => true + | _ => false + let majorityIndex := state.actors.size / 2 + let commitTxid := (state.actors[majorityIndex]!).txid + let persisted := state.actors.all fun actor => + match actor.nextStep with + | .open _ => actor.txid >= commitTxid + | _ => true + #[ + implies (unanimousVotes n state) (reachedOpenTimeout state false), + reachedOpen state, + implies (majorityHaveSameMaximum state) (reachedOpenTimeout state false), + implies (!reachedOpenTimeout state true) (openCount <= 1), + !(allOpenJoin && allVotesDelivered), + implies (!reachedOpenTimeout state true) persisted, + implies (state.actors.size > 1) (reachedOpen state), + reachedOpenTimeout state true, + majorityHaveSameMaximum state && reachedOpenTimeout state false + ] + +def legacyPropertyNames : Array String := #[ + "Unanimous votes => no chance of a fork", + "Open", + "Majority votes => no fork", + "No open with timeout, no fork", + "Deadlock", + "Persist committed txs", + "Open is possible", + "Unsafe open with timeout", + "Majority vote still opens without timeout" +] + +def legacyExpectations : Array String := #[ + "eventually", "eventually", "eventually", + "always", "always", "always", + "sometimes", "sometimes", "sometimes" +] + +end DisasterRecovery diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/Model.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/Model.lean new file mode 100644 index 000000000000..a5dd71dce88c --- /dev/null +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/Model.lean @@ -0,0 +1,286 @@ +import Std + +namespace DisasterRecovery.Protocol + +abbrev Location := String + +structure TxID where + view : Nat + seqno : Nat +deriving Repr, BEq, Hashable, Inhabited, DecidableEq + +inductive Phase where + | gossiping + | voting + | opening + | joining + | open +deriving Repr, BEq, Hashable, Inhabited, DecidableEq + +inductive OpenKind where + | quorum + | failover +deriving Repr, BEq, Hashable, Inhabited, DecidableEq + +inductive Validation where + | accepted + | rejected +deriving Repr, BEq, Hashable, Inhabited, DecidableEq + +structure Config where + instanceId : String + expectedLocations : List Location +deriving Repr, BEq, Hashable, Inhabited + +def Config.isValid (config : Config) : Bool := + !config.instanceId.isEmpty && + !config.expectedLocations.isEmpty && + !config.expectedLocations.any String.isEmpty && + config.expectedLocations.eraseDups.length = + config.expectedLocations.length + +structure NodeState where + location : Location + phase : Phase := .gossiping + timeoutState : Phase := .gossiping + gossips : List (Prod Location TxID) := [] + votes : List Location := [] + chosen : Option Location := none + openKind : Option OpenKind := none + restartRequested : Bool := false +deriving Repr, BEq, Hashable, Inhabited + +inductive Event where + | receiveGossip (source : Location) (txid : TxID) (validation : Validation) + | receiveVote (source : Location) (validation : Validation) + | receiveIAmOpen (source : Location) (validation : Validation) + | timeout + | retry +deriving Repr, BEq, Hashable + +inductive Effect where + | sendGossip (destination : Location) + | sendVote (destination : Location) + | sendIAmOpen (destination : Location) + | opening (kind : OpenKind) + | restart (chosen : Location) + | completed + | rejected (reason : String) +deriving Repr, BEq, Hashable + +structure StepOutput where + state : NodeState + effects : List Effect := [] + accepted : Bool := true +deriving Repr, BEq, Inhabited + +structure SystemState where + nodes : List (Prod Location NodeState) +deriving Repr, BEq, Hashable, Inhabited + +def phaseName : Phase -> String + | .gossiping => "GOSSIPING" + | .voting => "VOTING" + | .opening => "OPENING" + | .joining => "JOINING" + | .open => "OPEN" + +def openKindName : OpenKind -> String + | .quorum => "QUORUM" + | .failover => "FAILOVER" + +def initialNode (location : Location) : NodeState := + { location } + +def initialSystem (config : Config) : SystemState := + { nodes := config.expectedLocations.map fun location => + (location, initialNode location) } + +def voteQuorum (config : Config) : Nat := + config.expectedLocations.length / 2 + 1 + +def validTimeout (state : NodeState) (timeout : Bool) : Bool := + timeout && decide (state.phase = state.timeoutState) + +private def txScoreGreater + (leftName : Location) + (left : TxID) + (rightName : Location) + (right : TxID) : Bool := + right.view < left.view || + (right.view == left.view && + (right.seqno < left.seqno || + (right.seqno == left.seqno && rightName < leftName))) + +def maximumGossip : List (Prod Location TxID) -> Option (Prod Location TxID) + | [] => none + | head :: tail => + some (tail.foldl (fun current candidate => + if txScoreGreater candidate.1 candidate.2 current.1 current.2 then + candidate + else + current) head) + +def insertGossip + (source : Location) + (txid : TxID) + (gossips : List (Prod Location TxID)) : + List (Prod Location TxID) := + if gossips.any (fun entry => entry.1 == source) then + gossips + else + ((source, txid) :: gossips).mergeSort (fun left right => left.1 <= right.1) + +def insertVote (source : Location) (votes : List Location) : List Location := + if votes.contains source then votes + else (source :: votes).mergeSort (fun left right => left <= right) + +def advanceTimeoutState : Phase -> Phase + | .gossiping => .voting + | .voting => .opening + | state => state + +def advanceTimeoutLane (state : NodeState) (timeout : Bool) : NodeState := + if timeout then + { state with timeoutState := advanceTimeoutState state.timeoutState } + else + state + +def advance (config : Config) (state : NodeState) (timeout : Bool) : + Option StepOutput := + let aligned := validTimeout state timeout + match state.phase with + | .gossiping => + if decide (state.gossips.length >= config.expectedLocations.length) || aligned then + match maximumGossip state.gossips with + | none => none + | some (chosen, _) => + let next := { state with phase := .voting, chosen := some chosen } + some { state := advanceTimeoutLane next timeout } + else + some { state := advanceTimeoutLane state timeout } + | .voting => + let sufficient := decide (state.votes.length >= voteQuorum config) + if sufficient || aligned then + if aligned && state.votes.isEmpty then + some { state } + else + let kind := if aligned && !sufficient then .failover else .quorum + let next := { + state with + phase := .opening + openKind := some kind + } + some { + state := advanceTimeoutLane next timeout + effects := [.opening kind] + } + else + some { state := advanceTimeoutLane state timeout } + | .joining => + match state.chosen with + | none => none + | some chosen => + some { + state := advanceTimeoutLane + { state with restartRequested := true } timeout + effects := [.restart chosen] + } + | .opening => + if aligned then + some { + state := advanceTimeoutLane { state with phase := .open } timeout + effects := [.completed] + } + else + some { state := advanceTimeoutLane state timeout } + | .open => + some { state := advanceTimeoutLane state timeout } + +def rejected (state : NodeState) (reason : String) : StepOutput := + { state, effects := [.rejected reason], accepted := false } + +def step (config : Config) (state : NodeState) : Event -> StepOutput + | .receiveGossip source txid validation => + match validation with + | .rejected => rejected state "quote-or-certificate" + | .accepted => + if state.chosen != none then + rejected state "gossip-frozen" + else + let received := { state with + gossips := insertGossip source txid state.gossips } + (advance config received false).getD + (rejected state "empty-gossip-advance") + | .receiveVote source validation => + match validation with + | .rejected => rejected state "quote-or-certificate" + | .accepted => + let received := { state with votes := insertVote source state.votes } + (advance config received false).getD + (rejected state "vote-advance") + | .receiveIAmOpen source validation => + match validation with + | .rejected => rejected state "quote-or-certificate" + | .accepted => + match state.phase with + | .opening | .open => + rejected state "already-opening-or-open" + | _ => + let received := { + state with + phase := .joining + chosen := some source + } + (advance config received false).getD + (rejected state "join-without-chosen") + | .timeout => + (advance config state true).getD + (rejected state "empty-gossip-timeout-aborts") + | .retry => + let effects := + match state.phase with + | .gossiping => + config.expectedLocations.map .sendGossip + | .voting => + match state.chosen with + | none => config.expectedLocations.map .sendGossip + | some chosen => + .sendVote chosen :: config.expectedLocations.map .sendGossip + | .opening => + (config.expectedLocations.filter + (fun location => location != state.location)).map .sendIAmOpen + | .joining | .open => [] + { state, effects } + +private def replaceNode + (target : Location) + (next : NodeState) + (nodes : List (Prod Location NodeState)) : + List (Prod Location NodeState) := + nodes.map fun entry => if entry.1 == target then (target, next) else entry + +def systemStep + (config : Config) + (state : SystemState) + (target : Location) + (event : Event) : + Option (Prod SystemState StepOutput) := do + let node <- (state.nodes.find? fun entry => entry.1 == target).map Prod.snd + let output := step config node event + pure ({ + nodes := replaceNode target output.state state.nodes + }, output) + +def expectedSource (config : Config) (source : Location) : Bool := + config.expectedLocations.contains source + +def stateKey (state : NodeState) : String := + let gossips := String.intercalate "," (state.gossips.map fun entry => + s!"{entry.1}@{entry.2.view}.{entry.2.seqno}") + let votes := String.intercalate "," state.votes + let chosen := state.chosen.getD "-" + let kind := state.openKind.map openKindName |>.getD "-" + s!"{state.location}|{phaseName state.phase}|{phaseName state.timeoutState}|g={gossips}|v={votes}|c={chosen}|k={kind}|r={state.restartRequested}" + +end DisasterRecovery.Protocol diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/Refinement.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/Refinement.lean new file mode 100644 index 000000000000..0159c6cb9029 --- /dev/null +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/Refinement.lean @@ -0,0 +1,333 @@ +import DisasterRecovery.Model +import DisasterRecovery.Protocol.Model +import Mathlib.Logic.Relation + +namespace DisasterRecovery.Protocol.Refinement + +def projectPhase (state : NodeState) : DisasterRecovery.Phase := + match state.phase with + | .gossiping => .vote + | .voting => .openJoin + | .opening | .open => + match state.openKind with + | some .failover => .open true + | _ => .open false + | .joining => .join + +inductive LegacyAtomic : + DisasterRecovery.Phase -> DisasterRecovery.Phase -> Prop where + | gossipToVoting : LegacyAtomic .vote .openJoin + | quorumOpen : LegacyAtomic .openJoin (.open false) + | failoverOpen : LegacyAtomic .openJoin (.open true) + | gossipToJoin : LegacyAtomic .vote .join + | votingToJoin : LegacyAtomic .openJoin .join + +abbrev LegacyWeakStep := + Relation.ReflTransGen LegacyAtomic + +def embeddedTxID + (config : Config) + (source : Location) + (txid : TxID) : Prop := + txid.view = 0 /\ config.expectedLocations[txid.seqno]? = some source + +structure LegacyDataAssumptions + (config : Config) + (event : Event) + (after : NodeState) : Prop where + /-- Recorded for a future data refinement; phase simulation does not assume it. -/ + oddNodeCount : + exists half, config.expectedLocations.length = 2 * half + 1 + acceptedExpectedInput : + match event with + | .receiveGossip source txid validation => + validation = .accepted /\ + expectedSource config source = true /\ + embeddedTxID config source txid + | .receiveVote source validation => + validation = .accepted /\ expectedSource config source = true + | .receiveIAmOpen source validation => + validation = .accepted /\ expectedSource config source = true + | .timeout | .retry => True + quorumOnly : + after.openKind != some .failover + +structure CompatibilityStep + (config : Config) + (before : NodeState) + (event : Event) + (after : NodeState) : Prop where + canonical : + after = (step config before event).state + +private theorem advance_simulates + (config : Config) + (state : NodeState) + (timeout : Bool) + (output : StepOutput) + (advanced : advance config state timeout = some output) : + LegacyWeakStep (projectPhase state) (projectPhase output.state) := by + cases timeout <;> cases phase : state.phase <;> + simp [advance, phase] at advanced <;> + repeat' split at advanced <;> + simp_all [projectPhase, advanceTimeoutLane, advanceTimeoutState] + all_goals subst output + all_goals simp_all [projectPhase, advanceTimeoutLane] + all_goals + first + | (split <;> simp_all) + | skip + all_goals + first + | exact .refl + | exact .single .gossipToVoting + | exact .single .quorumOpen + | exact .single .failoverOpen + +private theorem receive_gossip_simulates + (config : Config) + (before : NodeState) + (source : Location) + (txid : TxID) + (validation : Validation) : + LegacyWeakStep + (projectPhase before) + (projectPhase + (step config before (.receiveGossip source txid validation)).state) := by + cases validation with + | rejected => + simp [step, rejected] + exact .refl + | accepted => + by_cases frozen : before.chosen != none + case pos => + simp [step, frozen, rejected] + exact .refl + case neg => + let received := { + before with gossips := insertGossip source txid before.gossips } + have same : projectPhase received = projectPhase before := by + simp [received, projectPhase] + cases advanced : advance config received false with + | none => + simp [step, frozen, received, advanced, rejected] + exact .refl + | some output => + simp [step, frozen, received, advanced] + have simulation := + advance_simulates config received false output advanced + rw [same] at simulation + exact simulation + +private theorem receive_vote_simulates + (config : Config) + (before : NodeState) + (source : Location) + (validation : Validation) : + LegacyWeakStep + (projectPhase before) + (projectPhase + (step config before (.receiveVote source validation)).state) := by + cases validation with + | rejected => + simp [step, rejected] + exact .refl + | accepted => + let received := { before with votes := insertVote source before.votes } + have same : projectPhase received = projectPhase before := by + simp [received, projectPhase] + cases advanced : advance config received false with + | none => + simp [step, received, advanced, rejected] + exact .refl + | some output => + simp [step, received, advanced] + have simulation := + advance_simulates config received false output advanced + rw [same] at simulation + exact simulation + +private theorem receive_iamopen_simulates + (config : Config) + (before : NodeState) + (source : Location) + (validation : Validation) : + LegacyWeakStep + (projectPhase before) + (projectPhase + (step config before (.receiveIAmOpen source validation)).state) := by + cases validation with + | rejected => + simp [step, rejected] + exact .refl + | accepted => + cases phase : before.phase <;> + simp [step, phase, advance, rejected, projectPhase, + advanceTimeoutLane] + all_goals + first + | exact .single .gossipToJoin + | exact .single .votingToJoin + | exact .refl + +theorem canonical_step_simulates + (config : Config) + (before : NodeState) + (event : Event) : + LegacyWeakStep + (projectPhase before) + (projectPhase (step config before event).state) := by + cases event with + | receiveGossip source txid validation => + exact receive_gossip_simulates config before source txid validation + | receiveVote source validation => + exact receive_vote_simulates config before source validation + | receiveIAmOpen source validation => + exact receive_iamopen_simulates config before source validation + | timeout => + cases advanced : advance config before true with + | none => + simp [step, advanced, rejected] + exact .refl + | some output => + simp [step, advanced] + exact advance_simulates config before true output advanced + | retry => + exact .refl + +theorem compatibility_step_simulates + {config : Config} + {before after : NodeState} + {event : Event} + (compatible : CompatibilityStep config before event after) : + LegacyWeakStep (projectPhase before) (projectPhase after) := by + rw [compatible.canonical] + exact canonical_step_simulates config before event + +def retryCompatibility + (config : Config) + (state : NodeState) : + CompatibilityStep config state .retry state := { + canonical := rfl +} + +def voteQuorumCompatibility + (config : Config) + (before : NodeState) + (source : Location) : + CompatibilityStep config before + (.receiveVote source .accepted) + (step config before (.receiveVote source .accepted)).state := { + canonical := rfl +} + +theorem quorum_phase_step_is_weak + (before after : NodeState) + (beforePhase : before.phase = .voting) + (afterPhase : after.phase = .opening) + (kind : after.openKind = some .quorum) : + LegacyWeakStep (projectPhase before) (projectPhase after) := by + simp [projectPhase, beforePhase, afterPhase, kind] + exact .single .quorumOpen + +theorem opening_to_open_is_stuttering + (before after : NodeState) + (beforePhase : before.phase = .opening) + (afterPhase : after.phase = .open) + (kind : after.openKind = before.openKind) : + LegacyWeakStep (projectPhase before) (projectPhase after) := by + simp [projectPhase, beforePhase, afterPhase, kind] + exact .refl + +inductive CompatibilityTrace + (config : Config) : + NodeState -> + List Event -> + NodeState -> + Prop where + | nil (state) : CompatibilityTrace config state [] state + | cons + (first middle last event rest) + (head : CompatibilityStep config first event middle) + (tail : CompatibilityTrace config middle rest last) : + CompatibilityTrace config first (event :: rest) last + +theorem compatibility_trace_simulates + {config : Config} + {first last : NodeState} + {events : List Event} + (compatible : CompatibilityTrace config first events last) : + LegacyWeakStep (projectPhase first) (projectPhase last) := by + induction compatible with + | nil state => exact .refl + | cons first middle last event rest head tail induction => + exact Relation.ReflTransGen.trans + (compatibility_step_simulates head) induction + +theorem initial_phase_correspondence : + projectPhase (initialNode "node0") = DisasterRecovery.Phase.vote := by + rfl + +theorem three_node_initial_correspondence : + let config : Config := { + instanceId := "compat" + expectedLocations := ["0", "1", "2"] + } + ((initialSystem config).nodes.map + (fun entry => projectPhase entry.2) == + (DisasterRecovery.initialState 3).actors.toList.map + (fun actor => actor.nextStep)) = true := by + rfl + +theorem odd_quorum_matches_legacy + (nodes half : Nat) + (odd : nodes = 2 * half + 1) : + nodes / 2 + 1 = (nodes + 1) / 2 := by + subst nodes + simp [Nat.add_div] + +theorem even_quorum_exceeds_legacy_by_one + (nodes half : Nat) + (even : nodes = 2 * half) : + nodes / 2 + 1 = (nodes + 1) / 2 + 1 := by + subst nodes + simp [Nat.add_div] + +def canonicalReachedOpen (state : NodeState) : Prop := + state.phase = .opening \/ state.phase = .open + +def projectedReachedOpen (state : NodeState) : Prop := + match projectPhase state with + | .open _ => True + | _ => False + +theorem reached_open_is_preserved + (state : NodeState) : + canonicalReachedOpen state <-> projectedReachedOpen state := by + cases phase : state.phase <;> + cases kind : state.openKind <;> + simp [canonicalReachedOpen, projectedReachedOpen, projectPhase, phase, kind] + all_goals + rename_i value + cases value <;> + simp + +theorem quorum_kind_projects_to_non_timeout_open + (state : NodeState) + (phase : state.phase = .opening \/ state.phase = .open) + (kind : state.openKind = some .quorum) : + projectPhase state = .open false := by + cases phase with + | inl opening => + cases state + simp_all [projectPhase] + | inr opened => + cases state + simp_all [projectPhase] + +theorem single_node_full_initial_models_differ : + projectPhase (initialNode "0") != + (DisasterRecovery.initialState 1).actors[0]!.nextStep := by + decide + +end DisasterRecovery.Protocol.Refinement \ No newline at end of file diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/Temporal.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/Temporal.lean new file mode 100644 index 000000000000..72db88bcbc0b --- /dev/null +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/Temporal.lean @@ -0,0 +1,231 @@ +import DisasterRecovery.Protocol.Model + +namespace DisasterRecovery.Protocol + +def EventuallyFrom (start : Nat) (predicate : Nat -> Prop) : Prop := + exists n, start <= n /\ predicate n + +def AlwaysFrom (start : Nat) (predicate : Nat -> Prop) : Prop := + forall n, start <= n -> predicate n + +def InfinitelyOften (predicate : Nat -> Prop) : Prop := + forall start, EventuallyFrom start predicate + +def EventuallyAlways (predicate : Nat -> Prop) : Prop := + exists start, AlwaysFrom start predicate + +structure Execution (config : Config) where + states : Nat -> NodeState + events : Nat -> Event + step_succ : forall n, + states (n + 1) = (step config (states n) (events n)).state + +def WeakFairness + {config : Config} + (execution : Execution config) + (enabled : NodeState -> Prop) + (fired : NodeState -> Event -> Prop) : Prop := + forall start, + AlwaysFrom start (fun n => enabled (execution.states n)) -> + EventuallyFrom start + (fun n => fired (execution.states n) (execution.events n)) + +def StrongFairness + {config : Config} + (execution : Execution config) + (enabled : NodeState -> Prop) + (fired : NodeState -> Event -> Prop) : Prop := + InfinitelyOften (fun n => enabled (execution.states n)) -> + InfinitelyOften + (fun n => fired (execution.states n) (execution.events n)) + +def AlignedOpening (state : NodeState) : Prop := + state.phase = .opening /\ state.timeoutState = .opening + +theorem valid_timeout_requires_alignment + (state : NodeState) + (h : validTimeout state true = true) : + state.phase = state.timeoutState := by + simpa [validTimeout] using h + +theorem gossip_freezes_after_choice + (config : Config) + (state : NodeState) + (source : Location) + (txid : TxID) + (h : state.chosen.isSome = true) : + let output := step config state (.receiveGossip source txid .accepted) + output.state = state /\ output.accepted = false := by + cases chosen : state.chosen <;> simp_all [step, rejected] + +theorem rejected_gossip_stutters + (config : Config) + (state : NodeState) + (source : Location) + (txid : TxID) : + let output := step config state (.receiveGossip source txid .rejected) + output.state = state /\ output.accepted = false := by + simp [step, rejected] + +theorem duplicate_vote_is_idempotent + (source : Location) + (votes : List Location) + (h : votes.contains source = true) : + insertVote source votes = votes := by + unfold insertVote + rw [h] + simp + +theorem opening_rejects_iamopen + (config : Config) + (state : NodeState) + (source : Location) : + let opening := { state with phase := .opening } + let output := step config opening (.receiveIAmOpen source .accepted) + output.state = opening /\ output.accepted = false := by + simp [step, rejected] + +theorem open_rejects_iamopen + (config : Config) + (state : NodeState) + (source : Location) : + let opened := { state with phase := .open } + let output := step config opened (.receiveIAmOpen source .accepted) + output.state = opened /\ output.accepted = false := by + simp [step, rejected] + +theorem aligned_voting_timeout_without_votes_stutters + (config : Config) + (state : NodeState) : + let waiting := { + state with + phase := .voting + timeoutState := .voting + votes := [] + } + step config waiting .timeout = { state := waiting } := by + simp [step, advance, validTimeout, voteQuorum] + +theorem aligned_opening_timeout_completes + (config : Config) + (state : NodeState) : + let opening := { + state with + phase := .opening + timeoutState := .opening + } + let output := step config opening .timeout + output.state.phase = .open /\ + output.state.timeoutState = .opening /\ + output.effects = [.completed] := by + simp [step, advance, validTimeout, advanceTimeoutLane, advanceTimeoutState] + +theorem quorum_advance_opens + (config : Config) + (state : NodeState) + (phase : state.phase = .voting) + (quorum : state.votes.length >= voteQuorum config) : + let output := (advance config state false).get! + output.state.phase = .opening /\ + output.state.openKind = some .quorum /\ + output.effects = [.opening .quorum] := by + simp [advance, phase, quorum, validTimeout, advanceTimeoutLane] + +theorem aligned_empty_gossip_timeout_aborts + (config : Config) + (state : NodeState) : + let waiting := { + state with + phase := .gossiping + timeoutState := .gossiping + gossips := [] + } + let output := step config waiting .timeout + output.state = waiting /\ output.accepted = false := by + simp [step, advance, validTimeout, rejected, maximumGossip] + +theorem non_timeout_step_preserves_aligned_opening + (config : Config) + (state : NodeState) + (event : Event) + (aligned : AlignedOpening state) + (notTimeout : Not (event = .timeout)) : + AlignedOpening (step config state event).state := by + have phase := aligned.1 + have timeoutState := aligned.2 + cases event with + | receiveGossip source txid validation => + cases validation <;> + simp_all [AlignedOpening, step, rejected, advance, validTimeout, + advanceTimeoutLane] + split <;> simp_all + | receiveVote source validation => + cases validation <;> + simp_all [AlignedOpening, step, rejected, advance, validTimeout, + advanceTimeoutLane] + | receiveIAmOpen source validation => + cases validation <;> + simp_all [AlignedOpening, step, rejected] + | timeout => + exact (notTimeout rfl).elim + | retry => + simp [AlignedOpening, step, phase, timeoutState] + +theorem aligned_timeout_transitions_to_open + (config : Config) + (state : NodeState) + (aligned : AlignedOpening state) : + (step config state .timeout).state.phase = .open := by + have phase := aligned.1 + have timeoutState := aligned.2 + simp [step, advance, validTimeout, phase, timeoutState, + advanceTimeoutLane, advanceTimeoutState] + +theorem fairness_supplies_firing + {config : Config} + (execution : Execution config) + (enabled : NodeState -> Prop) + (fired : NodeState -> Event -> Prop) + (fair : WeakFairness execution enabled fired) + (alwaysEnabled : forall n, enabled (execution.states n)) : + InfinitelyOften + (fun n => fired (execution.states n) (execution.events n)) := by + intro start + exact fair start (fun n _ => alwaysEnabled n) + +theorem fair_aligned_opening_progress + {config : Config} + (execution : Execution config) + (initial : AlignedOpening (execution.states 0)) + (fair : WeakFairness execution AlignedOpening + (fun _ event => event = .timeout)) : + EventuallyFrom 0 + (fun n => (execution.states n).phase = .open) := by + apply Classical.byContradiction + intro noOpen + have neverOpen : + forall n, Not ((execution.states n).phase = .open) := by + intro n opened + apply noOpen + exact Exists.intro n (And.intro (Nat.zero_le n) opened) + have alignedAlways : forall n, AlignedOpening (execution.states n) := by + intro n + induction n with + | zero => exact initial + | succ n aligned => + have notTimeout : Not (execution.events n = .timeout) := by + intro timeout + apply neverOpen (n + 1) + rw [execution.step_succ n, timeout] + exact aligned_timeout_transitions_to_open config _ aligned + rw [execution.step_succ n] + exact non_timeout_step_preserves_aligned_opening + config _ _ aligned notTimeout + have firing := fair 0 (fun n _ => alignedAlways n) + let n := firing.choose + have timeout := firing.choose_spec.2 + apply neverOpen (n + 1) + rw [execution.step_succ n, timeout] + exact aligned_timeout_transitions_to_open config _ (alignedAlways n) + +end DisasterRecovery.Protocol \ No newline at end of file diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/Trace.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/Trace.lean new file mode 100644 index 000000000000..af486ee32185 --- /dev/null +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/Trace.lean @@ -0,0 +1,772 @@ +import DisasterRecovery.Protocol.Model +import Lean.Data.Json +import Lean.Data.Json.FromToJson + +namespace DisasterRecovery.Protocol.Trace + +open Lean + +def contractVersion : String := + "ccf.recovery_decision_protocol.trace/1" + +inductive Kind where + | start + | gossipAccepted + | gossipRejected + | voteAccepted + | voteRejected + | iAmOpenAccepted + | iAmOpenRejected + | timeout + | retry + | send + | open + | joinRestart + | complete +deriving Repr, BEq, Inhabited + +structure TraceEvent where + version : String + instanceId : String + expectedLocations : List Location + node : Location + sequence : Nat + kind : Kind + messageId : Option String + causedBy : Option String + source : Option Location + txid : Option TxID + pre : Option Phase + post : Option Phase + openKind : Option OpenKind + send : Option String +deriving Repr, BEq, Inhabited + +structure Failure where + prefixLength : Nat + message : String + expected : List String +deriving Repr, BEq + +structure ObservedSend where + messageId : String + source : Location + description : String +deriving Repr, BEq + +structure HiddenSend where + source : Location + description : String +deriving Repr, BEq + +structure PendingEffect where + node : Location + effect : Effect +deriving Repr, BEq + +structure Candidate where + system : SystemState + hiddenSends : List HiddenSend := [] + pendingEffects : List PendingEffect := [] +deriving Repr, BEq + +private def parseKind : String -> Except String Kind + | "start" => pure .start + | "gossip_accepted" => pure .gossipAccepted + | "gossip_rejected" => pure .gossipRejected + | "vote_accepted" => pure .voteAccepted + | "vote_rejected" => pure .voteRejected + | "iamopen_accepted" => pure .iAmOpenAccepted + | "iamopen_rejected" => pure .iAmOpenRejected + | "timeout" => pure .timeout + | "retry" => pure .retry + | "send" => pure .send + | "open" => pure .open + | "join_restart" => pure .joinRestart + | "complete" => pure .complete + | value => throw s!"unknown kind '{value}'" + +private def parsePhase : String -> Except String Phase + | "GOSSIPING" => pure .gossiping + | "VOTING" => pure .voting + | "OPENING" => pure .opening + | "JOINING" => pure .joining + | "OPEN" => pure .open + | value => throw s!"unknown phase '{value}'" + +private def parseOpenKind : String -> Except String OpenKind + | "QUORUM" => pure .quorum + | "FAILOVER" => pure .failover + | value => throw s!"unknown open kind '{value}'" + +private def optionalString (json : Json) (key : String) : + Except String (Option String) := + match json.getObjVal? key with + | .error _ => pure none + | .ok .null => pure none + | .ok value => do + let parsed <- value.getStr? + pure (some parsed) + +private def optionalNat (json : Json) (key : String) : + Except String (Option Nat) := + match json.getObjVal? key with + | .error _ => pure none + | .ok .null => pure none + | .ok value => do + let parsed <- value.getNat? + pure (some parsed) + +private def optionalPhase (json : Json) (key : String) : + Except String (Option Phase) := do + let value <- optionalString json key + match value with + | none => pure none + | some name => do + let phase <- parsePhase name + pure (some phase) + +private def optionalOpenKind (json : Json) (key : String) : + Except String (Option OpenKind) := do + let value <- optionalString json key + match value with + | none => pure none + | some name => do + let kind <- parseOpenKind name + pure (some kind) + +def parseEvent (line : String) : Except String TraceEvent := do + let json <- Json.parse line + let version <- json.getObjValAs? String "version" + let instanceId <- json.getObjValAs? String "instance" + let expectedLocations <- json.getObjValAs? (List String) "expected_locations" + let node <- json.getObjValAs? String "node" + let sequence <- json.getObjValAs? Nat "sequence" + let kindName <- json.getObjValAs? String "kind" + let kind <- parseKind kindName + let messageId <- optionalString json "message_id" + let causedBy <- optionalString json "caused_by" + let source <- optionalString json "source" + let view <- optionalNat json "view" + let seqno <- optionalNat json "seqno" + let txid := + match view, seqno with + | some view, some seqno => some { view, seqno } + | none, none => none + | _, _ => none + let pre <- optionalPhase json "pre" + let post <- optionalPhase json "post" + let openKind <- optionalOpenKind json "open_kind" + let send <- optionalString json "send" + if version != contractVersion then + throw s!"unsupported version '{version}'" + if (view.isSome != seqno.isSome) then + throw "view and seqno must appear together" + pure { + version + instanceId + expectedLocations + node + sequence + kind + messageId + causedBy + source + txid + pre + post + openKind + send + } + +def parseNDJSON (input : String) : Except String (List TraceEvent) := do + let lines := (input.splitOn "\n").filter + (fun line => !line.trimAscii.isEmpty) + let mut events := [] + for (line, index) in lines.zipIdx do + match parseEvent line with + | .ok event => events := event :: events + | .error message => throw s!"line {index + 1}: {message}" + pure events.reverse + +private def nodeState (system : SystemState) (node : Location) : Option NodeState := + (system.nodes.find? fun entry => entry.1 == node).map Prod.snd + +private def phaseMatches (expected : Option Phase) (actual : Phase) : Bool := + match expected with + | none => true + | some phase => phase == actual + +private def requiresMessageId : Kind -> Bool + | .gossipAccepted | .gossipRejected + | .voteAccepted | .voteRejected + | .iAmOpenAccepted | .iAmOpenRejected + | .send => true + | _ => false + +private def requiresSource : Kind -> Bool + | .gossipAccepted | .gossipRejected + | .voteAccepted | .voteRejected + | .iAmOpenAccepted | .iAmOpenRejected => true + | _ => false + +private def missingRequiredFields (event : TraceEvent) : List String := + let phases := + (if event.pre.isNone then ["pre"] else []) ++ + (if event.post.isNone then ["post"] else []) + let message := + if requiresMessageId event.kind && event.messageId.isNone then + ["message_id"] + else + [] + let source := + if requiresSource event.kind && event.source.isNone then ["source"] else [] + let txid := + match event.kind with + | .gossipAccepted | .gossipRejected => + if event.txid.isNone then ["view", "seqno"] else [] + | _ => [] + let send := + match event.kind with + | .send => if event.send.isNone then ["send"] else [] + | _ => [] + let openKind := + match event.kind with + | .open => if event.openKind.isNone then ["open_kind"] else [] + | _ => [] + phases ++ message ++ source ++ txid ++ send ++ openKind + +private def configError (config : Config) : Option String := + if config.instanceId.isEmpty then + some "instance must not be empty" + else if config.expectedLocations.isEmpty then + some "expected_locations must not be empty" + else if config.expectedLocations.any String.isEmpty then + some "expected_locations must not contain an empty name" + else if config.expectedLocations.eraseDups.length != + config.expectedLocations.length then + some "expected_locations must not contain duplicates" + else + none + +private def eventInputs (event : TraceEvent) : List Event := + let validations : List Validation := + match event.kind with + | .gossipRejected | .voteRejected | .iAmOpenRejected => + [Validation.rejected, Validation.accepted] + | _ => [Validation.accepted] + match event.kind, event.source, event.txid with + | .gossipAccepted, some source, some txid => + [.receiveGossip source txid .accepted] + | .gossipRejected, some source, some txid => + validations.map fun validation => .receiveGossip source txid validation + | .voteAccepted, some source, _ => + [.receiveVote source .accepted] + | .voteRejected, some source, _ => + validations.map fun validation => .receiveVote source validation + | .iAmOpenAccepted, some source, _ => + [.receiveIAmOpen source .accepted] + | .iAmOpenRejected, some source, _ => + validations.map fun validation => .receiveIAmOpen source validation + | .timeout, _, _ => [.timeout] + | .retry, _, _ => [.retry] + | _, _, _ => [] + +private def expectsAcceptance : Kind -> Option Bool + | .gossipAccepted | .voteAccepted | .iAmOpenAccepted => some true + | .gossipRejected | .voteRejected | .iAmOpenRejected => some false + | .timeout | .retry => some true + | _ => none + +private def effectName : Effect -> Option String + | .sendGossip destination => some s!"gossip:{destination}" + | .sendVote destination => some s!"vote:{destination}" + | .sendIAmOpen destination => some s!"iamopen:{destination}" + | _ => none + +private def hiddenSendBefore (left right : HiddenSend) : Bool := + left.source < right.source || + (left.source == right.source && left.description <= right.description) + +private def uniqueHiddenSends (sends : List HiddenSend) : List HiddenSend := + (sends.foldl (fun result send => + if result.contains send then result else send :: result) []).mergeSort + hiddenSendBefore + +private def uniqueCandidates (candidates : List Candidate) : List Candidate := + candidates.foldl (fun result candidate => + if result.contains candidate then result else candidate :: result) [] + +private def hiddenSendsAt + (config : Config) + (startedNodes : List Location) + (system : SystemState) : List HiddenSend := + startedNodes.flatMap fun source => + match nodeState system source with + | none => [] + | some sender => + (step config sender .retry).effects.filterMap fun effect => do + let description <- effectName effect + pure { source, description } + +def hiddenClosure + (config : Config) + (startedNodes : List Location) + (candidates : List Candidate) : List Candidate := + uniqueCandidates (candidates.map fun candidate => { + candidate with + hiddenSends := uniqueHiddenSends + (candidate.hiddenSends ++ + hiddenSendsAt config startedNodes candidate.system) + }) + +private def expectedReceiveSend (event : TraceEvent) : Option String := + match event.kind with + | .gossipAccepted | .gossipRejected => + some s!"gossip:{event.node}" + | .voteAccepted | .voteRejected => + some s!"vote:{event.node}" + | .iAmOpenAccepted | .iAmOpenRejected => + some s!"iamopen:{event.node}" + | _ => none + +private def observedSendCompatible + (send : ObservedSend) + (event : TraceEvent) : Bool := + match event.source, expectedReceiveSend event with + | some source, some expected => + send.source == source && send.description == expected + | _, _ => false + +private def hiddenSendCompatible + (candidate : Candidate) + (event : TraceEvent) : Bool := + match event.source, expectedReceiveSend event with + | some source, some expected => + candidate.hiddenSends.contains { + source + description := expected + } + | _, _ => false + +private def observationCompatible + (config : Config) + (system : SystemState) + (event : TraceEvent) : Bool := + match nodeState system event.node with + | none => false + | some state => + if !phaseMatches event.pre state.phase || + !phaseMatches event.post state.phase then + false + else + match event.kind with + | .send => + match event.send with + | none => false + | some expected => + (step config state .retry).effects.any + (fun effect => effectName effect == some expected) + | .open => + state.phase == .opening && event.openKind == state.openKind + | .joinRestart => + state.phase == .joining && state.restartRequested + | .complete => + state.phase == .open + | _ => false + +private def removePendingEffect (node : Location) (target : Effect) : + List PendingEffect -> Option (List PendingEffect) + | [] => none + | pending :: rest => + if pending.node == node && pending.effect == target then + some rest + else + (removePendingEffect node target rest).map (fun remaining => + pending :: remaining) + +private def removePendingRestart (node : Location) : + List PendingEffect -> Option (List PendingEffect) + | [] => none + | pending :: rest => + match pending.effect with + | .restart _ => + if pending.node == node then + some rest + else + (removePendingRestart node rest).map (fun remaining => + pending :: remaining) + | _ => + (removePendingRestart node rest).map (fun remaining => + pending :: remaining) + +private def consumeObservation + (config : Config) + (candidate : Candidate) + (event : TraceEvent) : Option Candidate := do + guard (observationCompatible config candidate.system event) + match event.kind with + | .send => some candidate + | .open => do + let kind <- event.openKind + let pendingEffects <- removePendingEffect event.node (.opening kind) + candidate.pendingEffects + some { candidate with pendingEffects } + | .joinRestart => do + let pendingEffects <- removePendingRestart event.node + candidate.pendingEffects + some { candidate with pendingEffects } + | .complete => do + let pendingEffects <- removePendingEffect event.node .completed + candidate.pendingEffects + some { candidate with pendingEffects } + | _ => none + +private def isOneShotEffect : Effect -> Bool + | .opening _ | .restart _ | .completed => true + | _ => false + +private def transitionCandidates + (config : Config) + (system : SystemState) + (event : TraceEvent) : List (Prod SystemState StepOutput) := + match nodeState system event.node with + | none => [] + | some before => + if !phaseMatches event.pre before.phase then [] + else + (eventInputs event).filterMap fun input => do + let (nextSystem, output) <- systemStep config system event.node input + let acceptanceOk := + match expectsAcceptance event.kind with + | none => true + | some expected => output.accepted == expected + if acceptanceOk && phaseMatches event.post output.state.phase then + some (nextSystem, output) + else + none + +private def expectedEvents (candidates : List Candidate) (node : Location) : + List String := + let phases := + candidates.filterMap (fun candidate => nodeState candidate.system node) |>.map + (fun state => phaseName state.phase) + let phaseText := String.intercalate "/" phases.eraseDups + let common := [ + "timeout", "retry", "gossip_accepted|gossip_rejected", + "vote_accepted|vote_rejected", "iamopen_accepted|iamopen_rejected" + ] + let canOpen := candidates.any fun candidate => + candidate.pendingEffects.any fun pending => + pending.node == node && + match pending.effect with + | .opening _ => true + | _ => false + let canRestart := candidates.any fun candidate => + candidate.pendingEffects.any fun pending => + pending.node == node && + match pending.effect with + | .restart _ => true + | _ => false + let canComplete := candidates.any fun candidate => + candidate.pendingEffects.any fun pending => + pending.node == node && pending.effect == .completed + let observations := + (if canOpen then ["open(open_kind=QUORUM|FAILOVER)"] else []) ++ + (if phases.contains "OPENING" then ["send(iamopen:DEST)"] else []) ++ + (if canRestart then ["join_restart"] else []) ++ + (if canComplete then ["complete"] else []) + s!"state={phaseText}" :: common ++ observations + +structure ValidatorState where + config : Option Config := none + candidates : List Candidate := [] + nextSequence : List (Prod Location Nat) := [] + startedNodes : List Location := [] + seenMessageIds : List String := [] + observedSends : List ObservedSend := [] + consumedSendIds : List String := [] +deriving Inhabited + +private def expectedSequence (state : ValidatorState) (node : Location) : Nat := + (state.nextSequence.find? fun entry => entry.1 == node).map Prod.snd |>.getD 0 + +private def setSequence + (sequences : List (Prod Location Nat)) + (node : Location) + (next : Nat) : + List (Prod Location Nat) := + if sequences.any (fun entry => entry.1 == node) then + sequences.map fun entry => if entry.1 == node then (node, next) else entry + else + (node, next) :: sequences + +private def process + (index : Nat) + (state : ValidatorState) + (event : TraceEvent) : + Except Failure ValidatorState := do + let missing := missingRequiredFields event + if !missing.isEmpty then + throw { + prefixLength := index + 1 + message := s!"missing required fields: {String.intercalate ", " missing}" + expected := [] + } + if requiresSource event.kind && (event.source.getD "").isEmpty then + throw { + prefixLength := index + 1 + message := "source must not be empty" + expected := [] + } + if event.messageId.map String.isEmpty |>.getD false then + throw { + prefixLength := index + 1 + message := "message_id must not be empty" + expected := [] + } + if event.causedBy.map String.isEmpty |>.getD false then + throw { + prefixLength := index + 1 + message := "caused_by must not be empty" + expected := [] + } + if !requiresSource event.kind && event.causedBy.isSome then + throw { + prefixLength := index + 1 + message := "caused_by is only valid on receive events" + expected := [] + } + let expectedSeq := expectedSequence state event.node + if event.sequence != expectedSeq then + throw { + prefixLength := index + 1 + message := s!"node {event.node} sequence {event.sequence}, expected {expectedSeq}" + expected := [] + } + let config : Config := { + instanceId := event.instanceId + expectedLocations := event.expectedLocations + } + match configError config with + | some message => + throw { + prefixLength := index + 1 + message + expected := [] + } + | none => pure () + match event.messageId with + | some messageId => + if state.seenMessageIds.contains messageId || + state.consumedSendIds.contains messageId then + throw { + prefixLength := index + 1 + message := s!"message_id '{messageId}' was already used" + expected := [] + } + | none => pure () + if event.messageId.isSome && event.messageId == event.causedBy then + throw { + prefixLength := index + 1 + message := "message_id and caused_by must identify distinct observations" + expected := [] + } + match event.kind, state.config with + | .start, none => + if !config.expectedLocations.contains event.node then + throw { + prefixLength := index + 1 + message := s!"start node {event.node} is not expected" + expected := config.expectedLocations + } + let system := initialSystem config + let node := (nodeState system event.node).get! + if !phaseMatches event.pre node.phase || !phaseMatches event.post node.phase then + throw { + prefixLength := index + 1 + message := "start pre/post phase does not match GOSSIPING" + expected := ["pre=GOSSIPING", "post=GOSSIPING"] + } + pure { + config := some config + candidates := [{ system }] + nextSequence := setSequence state.nextSequence event.node (expectedSeq + 1) + startedNodes := [event.node] + seenMessageIds := event.messageId.toList + } + | .start, some established => + if established != config then + throw { + prefixLength := index + 1 + message := "instance or expected_locations changed" + expected := [] + } + if !config.expectedLocations.contains event.node then + throw { + prefixLength := index + 1 + message := s!"start node {event.node} is not expected" + expected := config.expectedLocations + } + if state.startedNodes.contains event.node then + throw { + prefixLength := index + 1 + message := s!"duplicate start event for node {event.node}" + expected := [] + } + let closure := + hiddenClosure config state.startedNodes state.candidates + let next := closure.filter fun candidate => + match nodeState candidate.system event.node with + | none => false + | some node => + phaseMatches event.pre node.phase && + phaseMatches event.post node.phase + if next.isEmpty then + throw { + prefixLength := index + 1 + message := "start pre/post phase does not match GOSSIPING" + expected := ["pre=GOSSIPING", "post=GOSSIPING"] + } + let startedNodes := event.node :: state.startedNodes + pure { + state with + candidates := hiddenClosure config startedNodes next + nextSequence := + setSequence state.nextSequence event.node (expectedSeq + 1) + startedNodes + seenMessageIds := event.messageId.toList ++ state.seenMessageIds + } + | _, none => + throw { + prefixLength := index + 1 + message := "trace must begin with start" + expected := ["start"] + } + | _, some established => + if established != config then + throw { + prefixLength := index + 1 + message := "instance or expected_locations changed" + expected := [] + } + if !state.startedNodes.contains event.node then + throw { + prefixLength := index + 1 + message := s!"node {event.node} has no start event" + expected := ["start"] + } + let closure := + hiddenClosure config state.startedNodes state.candidates + let mustMatchSend := + match event.kind with + | .gossipAccepted | .voteAccepted | .iAmOpenAccepted => true + | .gossipRejected | .voteRejected | .iAmOpenRejected => + event.causedBy.isSome + | _ => false + let sourceCandidates := + if mustMatchSend && + config.expectedLocations.contains (event.source.getD "") then + closure.filter + (fun (candidate : Candidate) => + hiddenSendCompatible candidate event) + else + closure + let causalCandidates := + match event.causedBy with + | none => sourceCandidates + | some cause => + if state.consumedSendIds.contains cause then + [] + else + match state.observedSends.find? + (fun (send : ObservedSend) => send.messageId == cause) with + | some send => + if observedSendCompatible send event then + sourceCandidates + else + [] + | none => + if state.seenMessageIds.contains cause then + [] + else + sourceCandidates + if event.causedBy.isSome && causalCandidates.isEmpty then + throw { + prefixLength := index + 1 + message := s!"caused_by '{event.causedBy.getD ""}' has no prior or hidden compatible send" + expected := expectedEvents closure event.node + } + let next := + match event.kind with + | .send | .open | .joinRestart | .complete => + causalCandidates.filterMap + (fun (candidate : Candidate) => + consumeObservation config candidate event) + | _ => + causalCandidates.flatMap fun (candidate : Candidate) => + (transitionCandidates config candidate.system event).map + (fun (system, output) => { + candidate with + system + pendingEffects := candidate.pendingEffects ++ + (output.effects.filter isOneShotEffect).map + (fun effect => { + node := event.node + effect + }) + }) + let next := hiddenClosure config state.startedNodes + (uniqueCandidates next) + if next.isEmpty then + throw { + prefixLength := index + 1 + message := s!"event {repr event.kind} is incompatible" + expected := expectedEvents closure event.node + } + pure { + config := some config + candidates := next + nextSequence := + setSequence state.nextSequence event.node (expectedSeq + 1) + startedNodes := state.startedNodes + seenMessageIds := event.messageId.toList ++ state.seenMessageIds + observedSends := + match event.kind, event.messageId, event.send with + | .send, some messageId, some description => + { + messageId + source := event.node + description + } :: state.observedSends + | _, _, _ => state.observedSends + consumedSendIds := event.causedBy.toList ++ state.consumedSendIds + } + +def validate (events : List TraceEvent) : Except Failure Nat := do + let mut state : ValidatorState := {} + for (event, index) in events.zipIdx do + state <- process index state event + if events.isEmpty then + throw { + prefixLength := 0 + message := "empty trace" + expected := ["start"] + } + match state.config with + | none => + throw { + prefixLength := events.length + message := "trace has no configuration" + expected := ["start"] + } + | some _ => pure () + pure state.candidates.length + +def renderFailure (failure : Failure) : String := + let expected := + if failure.expected.isEmpty then "" + else s!"\nexpected compatible events:\n {String.intercalate "\n " failure.expected}" + s!"shortest failing prefix: {failure.prefixLength}\n{failure.message}{expected}" + +end DisasterRecovery.Protocol.Trace diff --git a/lean/disaster-recovery/Main.lean b/lean/disaster-recovery/Main.lean new file mode 100644 index 000000000000..b1fc46eceefe --- /dev/null +++ b/lean/disaster-recovery/Main.lean @@ -0,0 +1,36 @@ +import DisasterRecovery.Checker + +open DisasterRecovery + +private def usage : String := + "usage: disaster-recovery (check|export) [--nodes N]" + +private def parseNodes : List String -> Except String Nat + | [] => pure 3 + | ["--nodes", value] => + match value.toNat? with + | some n => if n > 0 then pure n else throw "--nodes must be positive" + | none => throw s!"invalid node count: {value}" + | _ => throw usage + +def main (args : List String) : IO UInt32 := do + match args with + | command :: rest => + match parseNodes rest with + | .error message => + IO.eprintln message + pure 2 + | .ok n => + let graph <- enumerate n + match command with + | "export" => + exportGraph n graph + pure 0 + | "check" => + if <- checkGraph n graph then pure 0 else pure 1 + | _ => + IO.eprintln usage + pure 2 + | [] => + IO.eprintln usage + pure 2 diff --git a/lean/disaster-recovery/README.md b/lean/disaster-recovery/README.md new file mode 100644 index 000000000000..9cd56d1b2fb7 --- /dev/null +++ b/lean/disaster-recovery/README.md @@ -0,0 +1,396 @@ +# Disaster recovery models in Lean + +This project contains two deliberately separate models: + +- `DisasterRecovery.Model` is an executable reimplementation of the legacy + model in `tla/disaster-recovery/`. +- `DisasterRecovery.Protocol.Model` is a production-oriented model of the C++ + recovery decision protocol. + +The legacy model remains the comparison oracle; differences in the canonical +model are not silently backported. The project is pinned to Lean 4.28.0 and +Mathlib `v4.28.0` (resolved revision +`8f9d9cff6bd728b17a24e163c9402775d9e6a365`). + +## Status and scope + +The legacy Lean model and the Rust/Stateright model have identical canonical +graphs for one, two, and three nodes. This includes the initial state, every +reachable state, every labeled transition in both directions, and all nine +legacy predicate valuations. + +The canonical Lean model is separate because the production C++ protocol +intentionally differs from the legacy model in several places. It has formal +phase-refinement and fairness-aware progress results, plus a versioned trace +validator designed for future committed C++ instrumentation. + +The initial migration is approximately 4,300 lines across 26 new files: + +| Area | Files | Approximate lines | Purpose | +| ----------------------------------------- | ----: | ----------------: | --------------------------------------------------------------- | +| Exact legacy Lean model | 4 | 650 | Stateright state, actions, timers, network, predicates, and BFS | +| Canonical protocol and proofs | 4 | 1,000 | C++ behavior, phase refinement, quorum, and temporal proofs | +| Trace validation | 9 | 1,360 | NDJSON contract, validator, tests, and quorum/failover fixtures | +| Equivalence tooling | 2 | 710 | Rust graph exporter and bidirectional Rust/Lean comparison | +| Project, documentation, and configuration | 6 | 530 | Lake/Mathlib setup, entry points, and documentation | +| Pull-request CI | 1 | 80 | Lean model and bounded equivalence checks | + +### Principal files + +| File | Lines | Role | +| ---------------------------------------------------------------------------------------- | ----: | ------------------------------------------------------------------------ | +| [`DisasterRecovery/Model.lean`](DisasterRecovery/Model.lean) | 392 | Exact executable legacy semantics | +| [`DisasterRecovery/Checker.lean`](DisasterRecovery/Checker.lean) | 152 | BFS enumeration, property checking, and canonical graph export | +| [`DisasterRecovery/Protocol/Model.lean`](DisasterRecovery/Protocol/Model.lean) | 286 | Production-oriented C++ protocol model | +| [`DisasterRecovery/Protocol/Refinement.lean`](DisasterRecovery/Protocol/Refinement.lean) | 332 | Canonical-to-legacy formal phase refinement | +| [`DisasterRecovery/Protocol/Temporal.lean`](DisasterRecovery/Protocol/Temporal.lean) | 230 | Execution streams, fairness, safety, and progress proofs | +| [`DisasterRecovery/Protocol/Trace.lean`](DisasterRecovery/Protocol/Trace.lean) | 772 | Versioned implementation-trace parser and validator | +| [`TraceTests.lean`](TraceTests.lean) | 394 | Configuration, causality, delayed-send, and committed-effect regressions | +| [`compare.py`](compare.py) | 343 | Exhaustive canonical graph comparison | +| [`../../tla/disaster-recovery/src/export.rs`](../../tla/disaster-recovery/src/export.rs) | 370 | Canonical export of the actual Stateright model | +| [`TRACE_FORMAT_V1.md`](TRACE_FORMAT_V1.md) | 145 | Contract for future committed C++ trace events | + +The migration also updates the existing Stateright CLI and documentation, adds +weekly exhaustive verification, and adds +`.github/workflows/lean-shallow.yml` for relevant Lean, Rust, and C++ pull +requests. + +## How equivalence is established + +There are two distinct arguments: executable bounded equivalence with the Rust +model, and formal refinement between the two Lean models. + +### Executable Rust/Lean equivalence + +1. [`tla/disaster-recovery/src/export.rs`](../../tla/disaster-recovery/src/export.rs) + traverses the actual Stateright model through the public + `Model::next_steps` interface. It emits canonical full-state keys, explicit + action labels, and the values returned by the nine registered Rust property + functions. +2. [`DisasterRecovery/Checker.lean`](DisasterRecovery/Checker.lean) performs + independent BFS enumeration of the Lean legacy model and emits the same + `ccf-legacy-dr-graph-v1` representation. +3. [`compare.py`](compare.py) compares: + - the initial state; + - the complete normalized reachable-state sets; + - labeled edge sets in both directions; + - all nine property valuations for every state; and + - the final canonical files byte for byte. + +Run the complete comparison from this directory: + +```console +python3 compare.py --nodes 1 2 3 +``` + +The checked graph sizes are: + +| Nodes | Reachable states | Labeled transitions | +| ----: | ---------------: | ------------------: | +| 1 | 1 | 0 | +| 2 | 54 | 95 | +| 3 | 105,558 | 552,282 | + +This is exhaustive evidence for the checked finite configurations. It is not an +unbounded theorem about the Rust executable: such a theorem would require a +formal semantics for Rust and Stateright. + +### Formal Lean refinement + +[`DisasterRecovery/Protocol/Refinement.lean`](DisasterRecovery/Protocol/Refinement.lean) +contains the unbounded Lean proofs: + +- `canonical_step_simulates` proves that every canonical protocol step projects + to a reflexive-transitive legacy phase step. +- `compatibility_trace_simulates` composes that result across finite traces. +- `odd_quorum_matches_legacy` proves threshold equality for odd node counts. +- `even_quorum_exceeds_legacy_by_one` records the intentional even-node + discrepancy. +- `reached_open_is_preserved` proves preservation of the collapsed Open + predicate. + +[`DisasterRecovery/Protocol/Temporal.lean`](DisasterRecovery/Protocol/Temporal.lean) +defines evolving executions and suffix-based weak fairness. +`fair_aligned_opening_progress` proves that an execution beginning in aligned +Opening eventually reaches Open when timeout firing is weakly fair. + +These theorems relate the canonical and legacy Lean abstractions. They do not +claim that the canonical C++-aligned model is identical to the Rust model. + +## Legacy source mapping + +| Rust/Stateright source | Lean definition | +| ---------------------------------------- | ------------------------------------- | +| `ModelCfg`, `Id`, and `Txid` | node-count argument, `Id`, and `Txid` | +| `GossipStruct`, `VoteStruct`, and `Msg` | `Gossip`, `Vote`, and `Msg` | +| `NextStep` and actor `State` | `Phase` and `ActorState` | +| `Timer::ElectionTimeout` | one Boolean timer-set entry per actor | +| `Network::UnorderedNonDuplicating` | sorted `List Envelope` multiset | +| `ActorModelState` | `GlobalState` | +| `on_start` | `startActor` and `initialState` | +| `advance_step` and `advance_several` | `advanceStep` and `advanceSeveral` | +| `on_msg` and `on_timeout` | `onMessage` and `onTimeout` | +| `Model::actions` and `Model::next_state` | `actions` and `nextState` | +| predicates registered in `main.rs` | `legacyValuations` | + +The following legacy details are intentional: + +- A node's transaction ID is its actor ID. +- Gossip and vote collections are sets with canonical sorted representations. +- The network is an unordered multiset. Equal envelopes retain their + multiplicity, while only one delivery action is offered for that envelope. + Each delivery removes one occurrence. +- The network is reliable: there is no message-loss action. +- Every node has one election timer. Resetting an unchanged timer is a + Stateright no-op and does not create a transition. +- Rust `on_msg` calls `Cow::to_mut` before inspecting the message. Consequently + even an otherwise inert message delivery is a transition that removes one + envelope. +- Gossip collection freezes after `submitted_vote` becomes nonempty. +- `advance_several` takes the Vote to OpenJoin step and then immediately takes + the OpenJoin to Open step when the same event enables both. +- `IAmOpen` changes every non-Open phase, including Join, to Join. +- Rust phases map only abstractly to the implementation: Vote is gossiping, + OpenJoin is voting, `Open { timeout }` collapses opening and open, and Join is + joining. + +Constant-empty Stateright fields (`history`, random choices, actor storage) and +the all-false crash vector are omitted from the graph key. They cannot change +in this model. + +## Legacy predicates + +The nine valuations preserve the code in `tla/disaster-recovery/src/main.rs`, +including names that overstate or obscure what the Boolean predicate checks. + +| Expectation | Legacy name | State predicate | +| ----------- | ----------------------------------------- | ----------------------------------------------------------------------------------------- | +| Eventually | Unanimous votes => no chance of a fork | Unanimous submitted vote receive-sets imply a non-timeout Open exists | +| Eventually | Open | Some actor is Open | +| Eventually | Majority votes => no fork | The legacy sorted-prefix majority test implies a non-timeout Open exists | +| Always | No open with timeout, no fork | If no timeout Open exists, at most one actor is Open | +| Always | Deadlock | Not(all actors are OpenJoin and no Vote envelope exists) | +| Always | Persist committed txs | If no timeout Open exists, every Open actor's ID/txid is at least the middle actor's txid | +| Sometimes | Open is possible | Actor count greater than one implies some actor is Open | +| Sometimes | Unsafe open with timeout | Some timeout Open exists | +| Sometimes | Majority vote still opens without timeout | The legacy majority test and a non-timeout Open both hold | + +`Eventually` is checked on the finite graph as the least fixed point containing +states where the predicate is true and states whose nonempty successor set is +entirely in that fixed point. This is cycle-complete and stronger than +Stateright 0.31's terminal-path `Eventually` implementation, which can report +false negatives on cycles. Compatibility valuations and this Lean temporal +interpretation are therefore kept separate. `Always` and `Sometimes` quantify +over reachable states. These are executable bounded checks, not theorems about +the Rust program, and they add no fairness assumption. + +## Canonical C++ model + +The canonical model keeps one `NodeState` per fixed location and separates the +main phase from the timeout phase. Its executable `step` relation covers: + +- `GOSSIPING`, `VOTING`, `OPENING`, `JOINING`, and `OPEN`; +- lexicographic gossip selection by `(view, seqno, location-name)`; +- gossip completion after all expected locations or an aligned timeout, with + nonempty gossip required; +- quorum size `n / 2 + 1`, failover after a valid timeout, and no open after an + aligned voting timeout with zero votes; +- timeout-lane advancement after the main state machine except on the same + early-return/error paths as C++; +- retries, duplicate/idempotent receives, IAmOpen rejection in Opening/Open, + join/restart, and the Opening-to-Open timeout; +- an explicit `Validation.accepted`/`rejected` boundary. It assumes the C++ + quote and certificate checks have returned a result; it proves nothing about + cryptography. + +`Location` is the configured location name, the protocol's semantic identity +and final TxID tie-break. Network addresses and certificates are opaque +metadata: they affect validation and transport, not this state relation. + +### C++ source mapping + +| C++ source or branch | Canonical definition | +| ---------------------------------------------------------------------------- | ---------------------------------------------------------- | +| `self_healing_open.h`: `Location`, `StateMachine`, `OpenKinds`, service maps | `Location`, `Phase`, `OpenKind`, and fields of `NodeState` | +| `try_start()` writes both states to `GOSSIPING` | `initialNode` | +| `wrap_recovery_decision_protocol()` quote/certificate checks | `Validation` input boundary | +| gossip callback freezes after chosen node and inserts once | `receiveGossip` branch of `step` | +| vote callback inserts into a service set | `receiveVote` branch of `step` | +| IAmOpen callback rejects Opening/Open, otherwise chooses and joins | `receiveIAmOpen` branch of `step` | +| `advance()`: `valid_timeout` and Gossiping branch | `validTimeout` and gossip branch of `advance` | +| `advance()`: Voting quorum/failover/zero-vote return | voting branch of `advance` | +| `advance()`: Joining restart ringbuffer write | `Effect.restart` | +| `advance()`: inert Joining fallthrough under the timeout-lane invariant | Joining branch of `advance` | +| `advance()`: Opening aligned timeout | Opening branch and `Effect.complete` | +| final timeout-state switch in `advance()` | `advanceTimeoutState`, sequenced after main advancement | +| retry timer's Gossiping/Voting/Opening switch | `.retry` and `Effect.send` | +| timeout handler calls `advance(tx, true)` | `.timeout` | + +The handler wrapper validates and records messages before dispatch, then calls +`advance(tx, false)`. The model represents those committed semantic boundaries +as one receive event. + +### Confirmed implementation/model discrepancy + +The C++ handlers do not reject a successfully validated location name merely +because it is absent from `expected_locations`. Sends use the configured list, +but gossip/vote maps and their size thresholds can include unexpected names. +`CanonicalTests.lean` demonstrates that one unexpected accepted gossip can +satisfy a one-location threshold. The model preserves this behavior. The phase +refinement theorem covers this step; expected-source restrictions appear only +in `LegacyDataAssumptions` for richer data comparisons. + +The trace validator treats a nonempty accepted source outside +`expected_locations` as an external input. It applies the C++-aligned receive +transition but cannot validate the unmodeled sender's behavior. + +## Temporal results + +`DisasterRecovery.Protocol.Temporal` defines suffix predicates directly over +natural-numbered streams and bundles an evolving state/event stream in +`Execution`. Its `step_succ` field requires each next state to be the result of +`Protocol.step` on the current state and event. Weak fairness requires that, on +every suffix where an action is continuously enabled, that action fires on the +same suffix. Strong fairness requires infinitely-often enablement to imply +infinitely-often firing. + +Safety lemmas retain timeout alignment, gossip freeze, rejection stuttering, +duplicate-vote idempotence, Opening/Open IAmOpen rejection, zero-vote timeout +stuttering, empty-gossip timeout abort behavior, quorum opening, and +Opening-to-Open completion. `non_timeout_step_preserves_aligned_opening` proves +that every non-timeout event preserves `AlignedOpening`, while +`aligned_timeout_transitions_to_open` proves that an aligned timeout changes the +phase to Open. `fair_aligned_opening_progress` proves that an execution whose +initial state is `AlignedOpening` eventually reaches phase Open, assuming weak +fairness for timeout firing while `AlignedOpening` is enabled. This is not a +global termination theorem; reaching aligned Opening still requires separate +message-delivery and timeout progress assumptions. + +## Legacy compatibility + +`DisasterRecovery.Protocol.Refinement` intentionally does not claim +bisimulation or unrestricted data refinement. `canonical_step_simulates` proves +that every canonical single-node `Protocol.step`, for every event and state, +projects to a reflexive-transitive legacy phase step. No odd-node, +expected-source, embedded-TxID, or quorum-only premise is used for this phase +theorem. `CompatibilityStep` records only that the post-state is the executable +canonical result, and `compatibility_step_simulates` derives its legacy phase +step from `Protocol.step` semantics. + +The phase projection drops the timeout lane, gossips, votes, chosen node, and +restart state, makes retries and data-only updates stutter, maps Gossiping to +legacy Vote and Voting to legacy OpenJoin, and collapses canonical Opening/Open +to legacy Open with its quorum/failover kind. `CompatibilityTrace` contains +canonical steps, and `compatibility_trace_simulates` composes their derived +legacy weak steps. `retryCompatibility` and `voteQuorumCompatibility` are +canonical-step constructors, not phase simulation assumptions. + +`LegacyDataAssumptions` separately records the odd-node, expected-source, +embedded-TxID, and quorum-only restrictions relevant to a richer data/property +comparison; the current file does not claim such a data refinement. Initial +phase correspondence, the full three-node legacy-shaped initial phase +correspondence, collapsed-Open property preservation, quorum-kind projection, +and the single-node full-initial-state mismatch are proved. + +The threshold proofs state exactly: + +- for `n = 2*k + 1`, canonical `n / 2 + 1` equals legacy `(n + 1) / 2`; +- for `n = 2*k`, canonical quorum is legacy quorum plus one. + +Only predicates that depend on reaching collapsed Open and choosing the quorum +path are preserved by the current projection. Timeout-open, majority, deadlock, +and transaction-persistence legacy predicates are not claimed preserved: +timeout semantics, thresholds, TxIDs, initial gossip, retries, and phase +splitting differ. `single_node_full_initial_models_differ` is a proved +counterexample to full initial-state equality. + +## Trace validation + +The versioned NDJSON contract is documented in +[`TRACE_FORMAT_V1.md`](TRACE_FORMAT_V1.md). `trace-validator` parses it without +another dependency, tracks a set of compatible canonical states, and closes +that set over hidden retry/send stuttering. Each candidate retains the send +classes that could have been emitted by earlier hidden retries, so delayed +messages remain matchable after a sender changes phase without combining +incompatible histories. Candidates also retain one-shot committed effects; +`open`, `join_restart`, and `complete` observations consume these exactly once +for the node that produced them. On failure the validator reports the shortest +failing prefix and expected compatible events. + +The canonical v1 relation is deterministic, so a successful v1 prefix currently +retains one candidate. The candidate list and separate histories are explicit +for future under-observed or nondeterministic refinements. + +The validator also enforces configuration validity, one ordered event sequence +per node, one start per participating node, required event fields, globally +unique message IDs, feasible accepted receives from configured sources, and one +receive per supplied causal send ID. Observable `pre`, `post`, and open-kind +values are checked against every remaining candidate state. + +## Commands + +From this directory: + +```console +lake build +lake exe semantic-checks +lake exe canonical-checks +lake exe trace-checks +lake exe disaster-recovery check --nodes 3 +lake exe trace-validator fixtures/accepted.ndjson +lake exe trace-validator fixtures/accepted-failover.ndjson +lake exe trace-validator fixtures/accepted-multinode.ndjson +! lake exe trace-validator fixtures/rejected.ndjson +! lake exe trace-validator fixtures/rejected-cause.ndjson +python3 compare.py +``` + +The checker prints shortest BFS traces for unmet expectations, representative +`Sometimes` examples, and exits nonzero on failure. The semantic checks cover +immediate single-node open, frozen gossip, unordered delivery, timeout open, +`IAmOpen` joining, and duplicate envelope multiplicity. + +The Rust and Lean exporters emit tab-separated `ccf-legacy-dr-graph-v1`: + +```text +format ccf-legacy-dr-graph-v1 +nodes N +init CANONICAL_ID +state CANONICAL_ID STATE_KEY NINE_PROPERTY_BITS +edge SOURCE_ID NORMALIZED_ACTION DESTINATION_ID +``` + +Canonical IDs are assigned by lexicographically sorting state keys. State keys +encode actor states, set timers, and envelope multiplicities. Actions are +normalized as `deliver(src,dst,msg)` or `timeout(id,election)` rather than Rust +debug text. + +`compare.py` exhaustively compares the initial state, normalized reachable +state set, bidirectional labeled edge set, and all nine valuations for node +counts 1, 2, and 3. This executable comparison is the evidence for parity in +this slice. A Lean-only theorem could not establish behavior of the Rust and +Stateright executables without formal semantics for them. + +The Rust comparison exporter is checked separately from +`tla/disaster-recovery/` with `cargo check` and `cargo build`. + +`.github/workflows/lean-shallow.yml` runs the Lean build, semantic checks, +trace checks, three-node legacy property check, and the one- and two-node +Rust/Lean comparison on relevant pull requests. The weekly continuous +verification workflow additionally runs the exhaustive three-node comparison. +The Rust job remains in place until the replacement criteria below are met. + +## Replacement criteria + +The Stateright model can be removed only after: + +1. bounded bidirectional graph comparison remains green for all configurations + previously checked with Stateright; +2. the canonical Lean model and its proof targets cover every retained legacy + property, or an intentional semantic change is reviewed explicitly; and +3. committed traces from the quorum, failover, and multiple-timeout C++ e2e + scenarios validate against the canonical Lean model. + +This change establishes the first two migration mechanisms and freezes the +trace contract needed by the third. It does not claim implementation +conformance before C++ emits committed semantic events. diff --git a/lean/disaster-recovery/TRACE_FORMAT_V1.md b/lean/disaster-recovery/TRACE_FORMAT_V1.md new file mode 100644 index 000000000000..a274264b0194 --- /dev/null +++ b/lean/disaster-recovery/TRACE_FORMAT_V1.md @@ -0,0 +1,145 @@ +# Recovery decision protocol trace format, version 1 + +The media type is newline-delimited JSON. Each nonempty line is one committed +semantic observation. The version string is: + +```text +ccf.recovery_decision_protocol.trace/1 +``` + +## Record + +Every record is a JSON object with these required fields: + +| Field | Type | Meaning | +| -------------------- | ---------------- | ----------------------------------- | +| `version` | string | Exactly the version above | +| `instance` | string | Stable recovery instance identifier | +| `expected_locations` | array of strings | Stable configured location names | +| `node` | string | Observed node/location name | +| `sequence` | natural number | Per-node sequence, starting at zero | +| `kind` | string | Event kind from the table below | + +These fields are optional unless the event requires them: + +| Field | Type | Meaning | +| ------------ | ---------------- | ------------------------------------------------------------------------- | +| `message_id` | string | Globally unique ID for an observed send or receive | +| `caused_by` | string | `message_id` of the send that caused a receive | +| `source` | string | Sender location name | +| `view` | natural number | Gossip TxID view | +| `seqno` | natural number | Gossip TxID sequence number | +| `pre` | phase string | Observable phase before the event | +| `post` | phase string | Observable phase after the event | +| `open_kind` | open-kind string | `QUORUM` or `FAILOVER` for an `open` observation | +| `send` | string | Send class and destination: `gossip:NAME`, `vote:NAME`, or `iamopen:NAME` | + +Phase strings are `GOSSIPING`, `VOTING`, `OPENING`, `JOINING`, and `OPEN`. +Unknown fields are ignored for forward-compatible instrumentation metadata. +All integers must be nonnegative Lean `Nat` values. + +## Event kinds + +| Kind | Required event fields | Canonical boundary | +| ------------------ | --------------------------------------------------------------- | ------------------------------------------- | +| `start` | `pre`, `post` | Protocol state initialized | +| `gossip_accepted` | `message_id`, `source`, `view`, `seqno`, `pre`, `post` | Validated gossip callback committed | +| `gossip_rejected` | `message_id`, `source`, `view`, `seqno`, `pre`, `post` | Validation or protocol rejection | +| `vote_accepted` | `message_id`, `source`, `pre`, `post` | Validated vote callback committed | +| `vote_rejected` | `message_id`, `source`, `pre`, `post` | Validation rejection | +| `iamopen_accepted` | `message_id`, `source`, `pre`, `post` | IAmOpen selected peer and Joining committed | +| `iamopen_rejected` | `message_id`, `source`, `pre`, `post` | Validation or Opening/Open rejection | +| `timeout` | `pre`, `post` | Timeout transaction committed | +| `retry` | `pre`, `post` | Retry task observed | +| `send` | `send` in `class:destination` form, `message_id`, `pre`, `post` | Transport send observed | +| `open` | `open_kind`, `pre`, `post` | Service-open transition committed | +| `join_restart` | `pre`, `post` | Joining/restart side effect committed | +| `complete` | `pre`, `post` | Opening-to-Open completion committed | + +Receive events may use `caused_by` to identify the observed send. A referenced +send is checked for the expected sender, destination, and message class. If the +send was not logged, the validator may match it against a compatible hidden +send from a node that has already started. Message IDs, causal IDs, and source +names must be nonempty. Message IDs cannot be reused. Rejected events branch +over rejection at the explicit validation boundary and rejection by the +protocol state. A send ID may cause at most one receive event. + +Even without `caused_by`, an accepted receive from a configured source must +match a send that source could have emitted earlier. A rejected receive without +`caused_by` may represent rejection at the untrusted validation boundary. A +nonempty accepted source outside `expected_locations` is treated as an external +input, matching the current C++ behavior; the validator checks the local receive +transition but cannot constrain that external sender. + +Non-receive events must omit `caused_by`. + +Each participating configured node has one `start` event at sequence zero. The +first creates the initial candidate system; later starts activate other +configured nodes without resetting candidates. Non-start events for a node +before its start are rejected. Configured but unavailable nodes may have no +start event. Subsequent records must preserve `instance` and +`expected_locations`, refer to a configured node, and increment that node's +sequence exactly. Empty instance IDs, empty configurations, empty location +names, and duplicate configured names are rejected. + +The NDJSON record order must be a topological linearization of the distributed +trace. It must order a node's start and any observed transition enabling an +omitted send before the receive caused by that send. The collector must retain +this hidden happens-before edge while merging, even if it omits the send record +from the final trace. Per-node `sequence`, message causality, and these hidden +edges define the ordering; wall-clock timestamps do not. Unknown fields may +carry collector-specific merge metadata. + +## Under-observation + +An implementation trace need not expose every retry or network send. The +validator maintains compatible candidates containing a `SystemState` and the +send classes that could have been emitted by earlier hidden retries, plus +unobserved one-shot effects produced by committed transitions. This history +allows delayed delivery after a sender changes phase without merging +incompatible executions. `open`, `join_restart`, and `complete` each consume +one matching pending effect from the observed node, so one node cannot replay +its transition or consume another node's effect. +The validator computes a finite hidden closure over retry/send stuttering +before and after each observation. It does not guess protocol-state changes. +`pre` and `post` filter all candidates. Message IDs constrain causal matching +but are not protocol state. + +The canonical v1 transition relation is deterministic, so a successful v1 +prefix currently retains one candidate. Candidate sets and separate histories +remain explicit so later under-observed refinements can introduce genuine +alternatives without changing the validation architecture. + +If no candidate remains, validation stops at the shortest failing prefix and +prints the observed incompatible kind plus expected compatible events from the +last nonempty candidate set. A successful parse with no compatible execution is +never reported as success. + +## Instrumentation transaction rule + +Future C++ instrumentation must emit only after the transaction containing the +modeled state change commits. It must not log a handler mutation, open +transition, timeout-lane advance, or restart side effect from a transaction +that later aborts. Validation/HTTP rejection events that perform no state +mutation may be emitted at their final rejection boundary. This slice defines +the contract and Lean validator only; it does not instrument C++. + +## Example + +```json +{ + "version": "ccf.recovery_decision_protocol.trace/1", + "instance": "example", + "expected_locations": ["node0"], + "node": "node0", + "sequence": 0, + "kind": "start", + "pre": "GOSSIPING", + "post": "GOSSIPING" +} +``` + +Accepted quorum, failover-with-unavailable-locations, and multi-node examples +are `fixtures/accepted.ndjson`, `fixtures/accepted-failover.ndjson`, and +`fixtures/accepted-multinode.ndjson`. Deliberately rejected state and causal +examples are `fixtures/rejected.ndjson` and `fixtures/rejected-cause.ndjson`. diff --git a/lean/disaster-recovery/Tests.lean b/lean/disaster-recovery/Tests.lean new file mode 100644 index 000000000000..ebdb20c961e8 --- /dev/null +++ b/lean/disaster-recovery/Tests.lean @@ -0,0 +1,72 @@ +import DisasterRecovery.Model + +open DisasterRecovery + +private def expect (condition : Bool) (message : String) : IO Unit := + unless condition do throw (IO.userError message) + +private def deliverByKey (n : Nat) (state : GlobalState) (key : String) : Option GlobalState := do + let action <- (actions state).find? (fun action => actionKey action == key) + nextState n state action + +def main : IO UInt32 := do + let single := initialState 1 + expect (single.actors[0]!.nextStep == .open false) + "single node did not open immediately without timeout" + + let initial3 := initialState 3 + let timed <- match nextState 3 initial3 (.timeout 0) with + | some state => pure state + | none => throw (IO.userError "node 0 timeout was suppressed") + expect (timed.actors[0]!.nextStep == .open true) + "timeout did not drive vote and open-join closure to timeout-open" + + let opened := timed.actors[0]! + let lateGossip : Gossip := { src := 2, txid := 2 } + let frozen <- match onMessage 3 0 opened (.gossip lateGossip) with + | some result => pure result + | none => throw (IO.userError "message callback was unexpectedly suppressed") + expect (frozen.1.gossips == opened.gossips) + "gossip collection changed after the vote was submitted" + + let joinActor : ActorState := { + nextStep := .openJoin + gossips := [{ src := 1, txid := 1 }] + votes := [] + submittedVote := none + txid := 1 + } + let joined <- match onMessage 3 1 joinActor (.iAmOpen 0) with + | some result => pure result + | none => throw (IO.userError "IAmOpen was suppressed") + expect (joined.1.nextStep == .join) "IAmOpen did not cause Join" + + let firstOrder <- match deliverByKey 3 initial3 "deliver(1,0,g(1,1))" with + | some state => pure state + | none => throw (IO.userError "first unordered delivery failed") + let firstOrder <- match deliverByKey 3 firstOrder "deliver(2,0,g(2,2))" with + | some state => pure state + | none => throw (IO.userError "second unordered delivery failed") + let secondOrder <- match deliverByKey 3 initial3 "deliver(2,0,g(2,2))" with + | some state => pure state + | none => throw (IO.userError "reverse first unordered delivery failed") + let secondOrder <- match deliverByKey 3 secondOrder "deliver(1,0,g(1,1))" with + | some state => pure state + | none => throw (IO.userError "reverse second unordered delivery failed") + expect (firstOrder == secondOrder) "unordered deliveries produced different states" + + let duplicate : Envelope := { src := 1, dst := 0, msg := .gossip { src := 1, txid := 1 } } + let duplicated := { initial3 with network := duplicate :: duplicate :: initial3.network } + let once <- match nextState 3 duplicated (.deliver duplicate) with + | some state => pure state + | none => throw (IO.userError "first duplicate delivery was suppressed") + expect (once.network.count duplicate + 1 == duplicated.network.count duplicate) + "delivery did not remove exactly one duplicate" + let twice <- match nextState 3 once (.deliver duplicate) with + | some state => pure state + | none => throw (IO.userError "second duplicate delivery was suppressed") + expect (twice.network.count duplicate + 1 == once.network.count duplicate) + "second delivery did not remove exactly one duplicate" + + IO.println "all Lean semantic checks passed" + pure 0 diff --git a/lean/disaster-recovery/TraceMain.lean b/lean/disaster-recovery/TraceMain.lean new file mode 100644 index 000000000000..b00316203cbd --- /dev/null +++ b/lean/disaster-recovery/TraceMain.lean @@ -0,0 +1,23 @@ +import DisasterRecovery.Protocol.Trace + +open DisasterRecovery.Protocol.Trace + +def main (args : List String) : IO UInt32 := do + match args with + | [path] => + let input <- IO.FS.readFile path + match parseNDJSON input with + | .error message => + IO.eprintln message + pure 1 + | .ok events => + match validate events with + | .error failure => + IO.eprintln (renderFailure failure) + pure 1 + | .ok candidates => + IO.println s!"trace accepted: {events.length} events, {candidates} compatible final state(s)" + pure 0 + | _ => + IO.eprintln "usage: trace-validator TRACE.ndjson" + pure 2 diff --git a/lean/disaster-recovery/TraceTests.lean b/lean/disaster-recovery/TraceTests.lean new file mode 100644 index 000000000000..457cf33a6f39 --- /dev/null +++ b/lean/disaster-recovery/TraceTests.lean @@ -0,0 +1,394 @@ +import DisasterRecovery.Protocol.Trace + +open DisasterRecovery.Protocol +open DisasterRecovery.Protocol.Trace + +private def expect (condition : Bool) (message : String) : IO Unit := + unless condition do throw (IO.userError message) + +private def baseEvent + (locations : List Location) + (node : Location) + (sequence : Nat) + (kind : DisasterRecovery.Protocol.Trace.Kind) : + DisasterRecovery.Protocol.Trace.TraceEvent := { + version := contractVersion + instanceId := "trace-tests" + expectedLocations := locations + node + sequence + kind + messageId := none + causedBy := none + source := none + txid := none + pre := none + post := none + openKind := none + send := none +} + +private def startEvent + (locations : List Location) + (node : Location) : DisasterRecovery.Protocol.Trace.TraceEvent := { + baseEvent locations node 0 .start with + pre := some .gossiping + post := some .gossiping +} + +private def validationFailedAt + (events : List DisasterRecovery.Protocol.Trace.TraceEvent) + (expectedPrefix : Nat) : Bool := + match validate events with + | .error failure => failure.prefixLength == expectedPrefix + | .ok _ => false + +def main : IO UInt32 := do + let locations := ["A", "B"] + match validate [startEvent locations "A", startEvent locations "B"] with + | .ok 1 => pure () + | result => + throw (IO.userError s!"multi-node starts were not accepted: {repr result}") + + match validate [startEvent locations "A"] with + | .ok 1 => pure () + | result => + throw (IO.userError s!"unavailable configured node was required to start: {repr result}") + + expect + (validationFailedAt + [startEvent locations "A", startEvent locations "A"] 2) + "duplicate start was not rejected at the second event" + + expect + (validationFailedAt + [startEvent ["A", "A"] "A"] 1) + "duplicate expected_locations were not rejected" + + expect + (validationFailedAt + [{ startEvent ["A"] "A" with instanceId := "" }] 1) + "empty recovery instance was not rejected" + + expect + (validationFailedAt + [{ baseEvent ["A"] "A" 0 .start with post := some .gossiping }] 1) + "missing required start pre-state was not rejected" + + let single := ["A"] + let start := startEvent single "A" + let retry := { + baseEvent single "A" 1 .retry with + pre := some .gossiping + post := some .gossiping + } + let send := { + baseEvent single "A" 2 .send with + messageId := some "send-1" + pre := some .gossiping + post := some .gossiping + send := some "gossip:A" + } + let wrongCause := { + baseEvent single "A" 3 .voteAccepted with + messageId := some "receive-1" + causedBy := some "send-1" + source := some "A" + pre := some .gossiping + post := some .gossiping + } + expect + (validationFailedAt [start, retry, send, wrongCause] 4) + "caused_by accepted a send with the wrong message class" + + let duplicateId := { + baseEvent single "A" 3 .gossipAccepted with + messageId := some "send-1" + source := some "A" + txid := some { view := 1, seqno := 1 } + pre := some .gossiping + post := some .voting + } + expect + (validationFailedAt [start, retry, send, duplicateId] 4) + "duplicate message_id was not rejected" + + let received := { + baseEvent single "A" 3 .gossipAccepted with + messageId := some "receive-1" + causedBy := some "send-1" + source := some "A" + txid := some { view := 1, seqno := 1 } + pre := some .gossiping + post := some .voting + } + let reusedCause := { + baseEvent single "A" 4 .gossipRejected with + messageId := some "receive-2" + causedBy := some "send-1" + source := some "A" + txid := some { view := 1, seqno := 1 } + pre := some .voting + post := some .voting + } + expect + (validationFailedAt [start, retry, send, received, reusedCause] 5) + "a send was accepted as the cause of multiple receives" + + let openingVote := { + baseEvent single "A" 4 .voteAccepted with + messageId := some "opening-vote" + source := some "A" + pre := some .voting + post := some .opening + } + let openedOnce := { + baseEvent single "A" 5 .open with + pre := some .opening + post := some .opening + openKind := some .quorum + } + let openedTwice := { + openedOnce with sequence := 6 + } + expect + (validationFailedAt + [start, retry, send, received, openingVote, openedOnce, openedTwice] 7) + "one opening transition produced multiple committed open observations" + + let hiddenReceive := { + baseEvent single "A" 1 .gossipAccepted with + messageId := some "receive-hidden" + causedBy := some "hidden-send" + source := some "A" + txid := some { view := 1, seqno := 1 } + pre := some .gossiping + post := some .voting + } + let lateHiddenSend := { + baseEvent single "A" 2 .send with + messageId := some "hidden-send" + pre := some .voting + post := some .voting + send := some "gossip:A" + } + expect + (validationFailedAt [start, hiddenReceive, lateHiddenSend] 3) + "a hidden causal send ID was accepted later in the trace" + + let selfCaused := { + hiddenReceive with + messageId := some "same-id" + causedBy := some "same-id" + } + expect + (validationFailedAt [start, selfCaused] 2) + "one observation was accepted as both a send and its receive" + + let abortedTimeout := { + baseEvent single "A" 1 .timeout with + pre := some .gossiping + post := some .gossiping + } + expect + (validationFailedAt [start, abortedTimeout] 2) + "an aborted empty-gossip timeout was accepted as committed" + + let beforeSourceStart := { + baseEvent locations "A" 1 .gossipAccepted with + messageId := some "receive-before-start" + causedBy := some "hidden-before-start" + source := some "B" + txid := some { view := 1, seqno := 1 } + pre := some .gossiping + post := some .gossiping + } + expect + (validationFailedAt + [startEvent locations "A", beforeSourceStart, startEvent locations "B"] 2) + "a hidden send originated before its source node started" + + let impossibleVote := { + baseEvent locations "A" 1 .voteAccepted with + messageId := some "impossible-vote" + source := some "B" + pre := some .gossiping + post := some .gossiping + } + expect + (validationFailedAt + [startEvent locations "A", startEvent locations "B", impossibleVote] 3) + "an accepted receive bypassed hidden-send feasibility" + + let externalVote := { + baseEvent single "A" 1 .voteAccepted with + messageId := some "external-vote" + source := some "OUTSIDE" + pre := some .gossiping + post := some .gossiping + } + match validate [start, externalVote] with + | .ok 1 => pure () + | result => + throw (IO.userError s!"external accepted input was rejected: {repr result}") + + let emptySource := { + baseEvent single "A" 1 .gossipAccepted with + messageId := some "empty-source" + source := some "" + txid := some { view := 1, seqno := 1 } + pre := some .gossiping + post := some .voting + } + expect + (validationFailedAt [start, emptySource] 2) + "an empty receive source was accepted" + + let aGossipA := { + baseEvent locations "A" 1 .gossipAccepted with + messageId := some "a-gossip-a" + source := some "A" + txid := some { view := 2, seqno := 1 } + pre := some .gossiping + post := some .gossiping + } + let aGossipB := { + baseEvent locations "A" 2 .gossipAccepted with + messageId := some "a-gossip-b" + source := some "B" + txid := some { view := 1, seqno := 1 } + pre := some .gossiping + post := some .voting + } + let bGossipA := { + baseEvent locations "B" 1 .gossipAccepted with + messageId := some "b-gossip-a" + source := some "A" + txid := some { view := 2, seqno := 1 } + pre := some .gossiping + post := some .gossiping + } + let bGossipB := { + baseEvent locations "B" 2 .gossipAccepted with + messageId := some "b-gossip-b" + source := some "B" + txid := some { view := 1, seqno := 1 } + pre := some .gossiping + post := some .voting + } + let aVoteA := { + baseEvent locations "A" 3 .voteAccepted with + messageId := some "a-vote-a" + source := some "A" + pre := some .voting + post := some .voting + } + let aVoteB := { + baseEvent locations "A" 4 .voteAccepted with + messageId := some "a-vote-b" + source := some "B" + pre := some .voting + post := some .opening + } + let delayedGossip := { + baseEvent locations "B" 3 .gossipRejected with + messageId := some "b-delayed-gossip" + causedBy := some "hidden-delayed-gossip" + source := some "A" + txid := some { view := 2, seqno := 1 } + pre := some .voting + post := some .voting + } + let delayedTrace := [ + startEvent locations "A", + startEvent locations "B", + aGossipA, + aGossipB, + bGossipA, + bGossipB, + aVoteA, + aVoteB, + delayedGossip + ] + match validate delayedTrace with + | .ok 1 => pure () + | result => + throw (IO.userError s!"delayed hidden send was rejected: {repr result}") + + let failoverGossipA := { + baseEvent locations "A" 1 .gossipAccepted with + messageId := some "failover-gossip-a" + source := some "A" + txid := some { view := 1, seqno := 1 } + pre := some .gossiping + post := some .gossiping + } + let failoverTimeoutA1 := { + baseEvent locations "A" 2 .timeout with + pre := some .gossiping + post := some .voting + } + let failoverVoteA := { + baseEvent locations "A" 3 .voteAccepted with + messageId := some "failover-vote-a" + source := some "A" + pre := some .voting + post := some .voting + } + let failoverTimeoutA2 := { + baseEvent locations "A" 4 .timeout with + pre := some .voting + post := some .opening + } + let failoverGossipB := { + baseEvent locations "B" 1 .gossipAccepted with + messageId := some "failover-gossip-b" + source := some "B" + txid := some { view := 1, seqno := 1 } + pre := some .gossiping + post := some .gossiping + } + let failoverTimeoutB1 := { + baseEvent locations "B" 2 .timeout with + pre := some .gossiping + post := some .voting + } + let failoverVoteB := { + baseEvent locations "B" 3 .voteAccepted with + messageId := some "failover-vote-b" + source := some "B" + pre := some .voting + post := some .voting + } + let failoverTimeoutB2 := { + baseEvent locations "B" 4 .timeout with + pre := some .voting + post := some .opening + } + let openA := { + baseEvent locations "A" 5 .open with + pre := some .opening + post := some .opening + openKind := some .failover + } + let duplicateOpenA := { openA with sequence := 6 } + let twoFailovers := [ + startEvent locations "A", + startEvent locations "B", + failoverGossipA, + failoverTimeoutA1, + failoverVoteA, + failoverTimeoutA2, + failoverGossipB, + failoverTimeoutB1, + failoverVoteB, + failoverTimeoutB2, + openA, + duplicateOpenA + ] + expect + (validationFailedAt twoFailovers 12) + "node A consumed node B's pending open effect" + + IO.println "all trace contract checks passed" + pure 0 diff --git a/lean/disaster-recovery/compare.py b/lean/disaster-recovery/compare.py new file mode 100644 index 000000000000..b9ac6822e7f7 --- /dev/null +++ b/lean/disaster-recovery/compare.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +import argparse +import filecmp +import subprocess +import sys +import tempfile +from collections import defaultdict, deque +from dataclasses import dataclass +from pathlib import Path + +FORMAT = "ccf-legacy-dr-graph-v1" +PROPERTY_NAMES = ( + "Unanimous votes => no chance of a fork", + "Open", + "Majority votes => no fork", + "No open with timeout, no fork", + "Deadlock", + "Persist committed txs", + "Open is possible", + "Unsafe open with timeout", + "Majority vote still opens without timeout", +) + + +@dataclass(frozen=True) +class Summary: + initial_key: str + states: int + edges: int + + +@dataclass +class Graph: + initial: str + valuations: dict[str, str] + edges: set[tuple[str, str, str]] + + +def run(command: list[str], cwd: Path, output: Path | None = None) -> None: + print(f"+ (cd {cwd} && {' '.join(command)})", flush=True) + if output is None: + result = subprocess.run( + command, cwd=cwd, text=True, capture_output=True, check=False + ) + else: + with output.open("w", encoding="ascii", newline="") as stream: + result = subprocess.run( + command, + cwd=cwd, + text=True, + stdout=stream, + stderr=subprocess.PIPE, + check=False, + ) + if result.returncode != 0: + if result.stderr: + print(result.stderr, file=sys.stderr, end="") + raise RuntimeError(f"command exited with status {result.returncode}") + + +def validate(path: Path, expected_nodes: int) -> Summary: + ids_to_keys: list[str] = [] + initial_id: int | None = None + edge_count = 0 + previous_edge: tuple[int, str, int] | None = None + section = "header" + + with path.open(encoding="ascii") as stream: + for line_number, raw_line in enumerate(stream, 1): + fields = raw_line.rstrip("\n").split("\t") + if fields == ["format", FORMAT] and line_number == 1: + continue + if fields == ["nodes", str(expected_nodes)] and line_number == 2: + continue + if len(fields) == 2 and fields[0] == "init" and line_number == 3: + initial_id = int(fields[1]) + section = "states" + continue + if len(fields) == 4 and fields[0] == "state" and section == "states": + state_id = int(fields[1]) + if state_id != len(ids_to_keys): + raise ValueError( + f"{path}:{line_number}: expected dense state id " + f"{len(ids_to_keys)}, found {state_id}" + ) + if ids_to_keys and fields[2] <= ids_to_keys[-1]: + raise ValueError( + f"{path}:{line_number}: state keys are unsorted or duplicated" + ) + bits = fields[3] + if len(bits) != len(PROPERTY_NAMES) or set(bits) - {"0", "1"}: + raise ValueError( + f"{path}:{line_number}: invalid property bitstring" + ) + ids_to_keys.append(fields[2]) + continue + if len(fields) == 4 and fields[0] == "edge": + section = "edges" + edge = (int(fields[1]), fields[2], int(fields[3])) + if edge[0] >= len(ids_to_keys) or edge[2] >= len(ids_to_keys): + raise ValueError( + f"{path}:{line_number}: edge references unknown state" + ) + if previous_edge is not None and edge <= previous_edge: + raise ValueError( + f"{path}:{line_number}: edges are unsorted or duplicated" + ) + previous_edge = edge + edge_count += 1 + continue + raise ValueError( + f"{path}:{line_number}: invalid record: {raw_line.rstrip()}" + ) + + if initial_id is None or initial_id >= len(ids_to_keys): + raise ValueError(f"{path}: invalid or missing initial state") + return Summary(ids_to_keys[initial_id], len(ids_to_keys), edge_count) + + +def load(path: Path) -> Graph: + ids_to_keys: list[str] = [] + valuations: dict[str, str] = {} + raw_edges: list[tuple[int, str, int]] = [] + initial_id = -1 + with path.open(encoding="ascii") as stream: + for raw_line in stream: + fields = raw_line.rstrip("\n").split("\t") + if fields[0] == "init": + initial_id = int(fields[1]) + elif fields[0] == "state": + state_id = int(fields[1]) + key = fields[2] + if state_id != len(ids_to_keys): + raise ValueError(f"{path}: non-dense state IDs") + ids_to_keys.append(key) + valuations[key] = fields[3] + elif fields[0] == "edge": + raw_edges.append((int(fields[1]), fields[2], int(fields[3]))) + edges = { + (ids_to_keys[src], action, ids_to_keys[dst]) for src, action, dst in raw_edges + } + return Graph(ids_to_keys[initial_id], valuations, edges) + + +def shortest_paths(graph: Graph) -> tuple[dict[str, int], dict[str, tuple[str, str]]]: + adjacency: dict[str, list[tuple[str, str]]] = defaultdict(list) + for src, action, dst in graph.edges: + adjacency[src].append((action, dst)) + for outgoing in adjacency.values(): + outgoing.sort() + + distance = {graph.initial: 0} + parent: dict[str, tuple[str, str]] = {} + pending = deque([graph.initial]) + while pending: + src = pending.popleft() + for action, dst in adjacency[src]: + if dst not in distance: + distance[dst] = distance[src] + 1 + parent[dst] = (src, action) + pending.append(dst) + return distance, parent + + +def describe_path( + graph: Graph, target: str, cached: tuple[dict[str, int], dict[str, tuple[str, str]]] +) -> str: + distance, parent = cached + if target not in distance: + return f"unreachable target key {target}" + actions: list[str] = [] + cursor = target + while cursor != graph.initial: + cursor, action = parent[cursor] + actions.append(action) + actions.reverse() + rendered = "\n".join( + f" {index}. {action}" for index, action in enumerate(actions, 1) + ) + return f"target: {target}\n{rendered or ' '}" + + +def mismatch(rust: Graph, lean: Graph) -> str: + if rust.initial != lean.initial: + return f"initial state mismatch\nRust: {rust.initial}\nLean: {lean.initial}" + + rust_paths = lean_paths = None + rust_states = set(rust.valuations) + lean_states = set(lean.valuations) + if rust_states != lean_states: + rust_only = rust_states - lean_states + lean_only = lean_states - rust_states + candidates: list[tuple[int, str, str, Graph]] = [] + if rust_only: + rust_paths = shortest_paths(rust) + state = min( + rust_only, key=lambda key: (rust_paths[0].get(key, sys.maxsize), key) + ) + candidates.append( + (rust_paths[0].get(state, sys.maxsize), "Rust-only", state, rust) + ) + if lean_only: + lean_paths = shortest_paths(lean) + state = min( + lean_only, key=lambda key: (lean_paths[0].get(key, sys.maxsize), key) + ) + candidates.append( + (lean_paths[0].get(state, sys.maxsize), "Lean-only", state, lean) + ) + _, side, state, graph = min(candidates) + paths = rust_paths if graph is rust else lean_paths + return ( + f"reachable state mismatch ({len(rust_only)} Rust-only, " + f"{len(lean_only)} Lean-only); shortest is {side}\n" + f"{describe_path(graph, state, paths)}" + ) + + rust_only_edges = rust.edges - lean.edges + lean_only_edges = lean.edges - rust.edges + if rust_only_edges or lean_only_edges: + candidates = [] + if rust_only_edges: + rust_paths = shortest_paths(rust) + edge = min( + rust_only_edges, + key=lambda value: (rust_paths[0].get(value[0], sys.maxsize), value), + ) + candidates.append( + ( + rust_paths[0].get(edge[0], sys.maxsize), + "Rust-only", + edge, + rust, + ) + ) + if lean_only_edges: + lean_paths = shortest_paths(lean) + edge = min( + lean_only_edges, + key=lambda value: (lean_paths[0].get(value[0], sys.maxsize), value), + ) + candidates.append( + ( + lean_paths[0].get(edge[0], sys.maxsize), + "Lean-only", + edge, + lean, + ) + ) + _, side, (src, action, dst), graph = min(candidates) + paths = rust_paths if graph is rust else lean_paths + return ( + f"labeled edge mismatch ({len(rust_only_edges)} Rust-only, " + f"{len(lean_only_edges)} Lean-only); shortest source is {side}\n" + f"{describe_path(graph, src, paths)}\n" + f"missing edge action: {action}\ndestination: {dst}" + ) + + differing = { + key for key in rust_states if rust.valuations[key] != lean.valuations[key] + } + if differing: + rust_paths = shortest_paths(rust) + state = min( + differing, key=lambda key: (rust_paths[0].get(key, sys.maxsize), key) + ) + rust_bits = rust.valuations[state] + lean_bits = lean.valuations[state] + details = [ + f" {index + 1}. {name}: Rust={rust_bits[index]} Lean={lean_bits[index]}" + for index, name in enumerate(PROPERTY_NAMES) + if rust_bits[index] != lean_bits[index] + ] + return ( + f"property valuation mismatch in {len(differing)} states\n" + f"{describe_path(rust, state, rust_paths)}\n" + "\n".join(details) + ) + + return "canonical files differ despite identical graph content" + + +def compare(nodes: int, lean_dir: Path, rust_dir: Path, temporary: Path) -> Summary: + rust_path = temporary / f"rust-{nodes}.tsv" + lean_path = temporary / f"lean-{nodes}.tsv" + run( + [ + "cargo", + "run", + "--quiet", + "--", + "export", + "--nodes", + str(nodes), + "-o", + str(rust_path), + ], + rust_dir, + ) + run( + ["lake", "exe", "disaster-recovery", "export", "--nodes", str(nodes)], + lean_dir, + lean_path, + ) + rust_summary = validate(rust_path, nodes) + lean_summary = validate(lean_path, nodes) + if rust_summary != lean_summary or not filecmp.cmp( + rust_path, lean_path, shallow=False + ): + raise AssertionError(mismatch(load(rust_path), load(lean_path))) + return rust_summary + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Exhaustively compare Rust/Stateright and Lean legacy DR graphs" + ) + parser.add_argument("--nodes", type=int, nargs="+", default=[1, 2, 3]) + args = parser.parse_args() + if any(nodes < 1 for nodes in args.nodes): + parser.error("node counts must be positive") + + lean_dir = Path(__file__).resolve().parent + rust_dir = lean_dir.parents[1] / "tla" / "disaster-recovery" + try: + with tempfile.TemporaryDirectory(prefix="ccf-legacy-dr-") as directory: + for nodes in args.nodes: + summary = compare(nodes, lean_dir, rust_dir, Path(directory)) + print( + f"n={nodes}: equivalent initial state, {summary.states} states, " + f"{summary.edges} labeled edges compared in both directions, " + f"{len(PROPERTY_NAMES)} valuations/state" + ) + except (AssertionError, OSError, RuntimeError, ValueError) as error: + print(f"equivalence failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/lean/disaster-recovery/fixtures/accepted-failover.ndjson b/lean/disaster-recovery/fixtures/accepted-failover.ndjson new file mode 100644 index 000000000000..0a1ff06789e5 --- /dev/null +++ b/lean/disaster-recovery/fixtures/accepted-failover.ndjson @@ -0,0 +1,8 @@ +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":1,"kind":"gossip_accepted","message_id":"gossip-a","source":"A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":2,"kind":"timeout","pre":"GOSSIPING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":3,"kind":"vote_accepted","message_id":"vote-a","source":"A","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":4,"kind":"timeout","pre":"VOTING","post":"OPENING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":5,"kind":"open","open_kind":"FAILOVER","pre":"OPENING","post":"OPENING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":6,"kind":"timeout","pre":"OPENING","post":"OPEN"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":7,"kind":"complete","pre":"OPEN","post":"OPEN"} diff --git a/lean/disaster-recovery/fixtures/accepted-multinode.ndjson b/lean/disaster-recovery/fixtures/accepted-multinode.ndjson new file mode 100644 index 000000000000..93b58b6dd448 --- /dev/null +++ b/lean/disaster-recovery/fixtures/accepted-multinode.ndjson @@ -0,0 +1,5 @@ +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":1,"kind":"retry","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":2,"kind":"send","message_id":"send-a-b","send":"gossip:B","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":1,"kind":"gossip_accepted","message_id":"receive-a-b","caused_by":"send-a-b","source":"A","view":1,"seqno":10,"pre":"GOSSIPING","post":"GOSSIPING"} diff --git a/lean/disaster-recovery/fixtures/accepted.ndjson b/lean/disaster-recovery/fixtures/accepted.ndjson new file mode 100644 index 000000000000..c53d64c21454 --- /dev/null +++ b/lean/disaster-recovery/fixtures/accepted.ndjson @@ -0,0 +1,10 @@ +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":1,"kind":"retry","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":2,"kind":"send","message_id":"send-gossip-1","send":"gossip:A","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":3,"kind":"gossip_accepted","message_id":"recv-gossip-1","caused_by":"send-gossip-1","source":"A","view":1,"seqno":10,"pre":"GOSSIPING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":4,"kind":"vote_accepted","message_id":"recv-vote-1","source":"A","pre":"VOTING","post":"OPENING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":5,"kind":"open","open_kind":"QUORUM","pre":"OPENING","post":"OPENING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":6,"kind":"timeout","pre":"OPENING","post":"OPENING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":7,"kind":"timeout","pre":"OPENING","post":"OPENING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":8,"kind":"timeout","pre":"OPENING","post":"OPEN"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":9,"kind":"complete","pre":"OPEN","post":"OPEN"} diff --git a/lean/disaster-recovery/fixtures/rejected-cause.ndjson b/lean/disaster-recovery/fixtures/rejected-cause.ndjson new file mode 100644 index 000000000000..07a2c30eecf9 --- /dev/null +++ b/lean/disaster-recovery/fixtures/rejected-cause.ndjson @@ -0,0 +1,4 @@ +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-cause","expected_locations":["A"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-cause","expected_locations":["A"],"node":"A","sequence":1,"kind":"retry","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-cause","expected_locations":["A"],"node":"A","sequence":2,"kind":"send","message_id":"send-gossip","send":"gossip:A","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-cause","expected_locations":["A"],"node":"A","sequence":3,"kind":"vote_accepted","message_id":"receive-vote","caused_by":"send-gossip","source":"A","pre":"GOSSIPING","post":"GOSSIPING"} diff --git a/lean/disaster-recovery/fixtures/rejected.ndjson b/lean/disaster-recovery/fixtures/rejected.ndjson new file mode 100644 index 000000000000..8a54beb2d427 --- /dev/null +++ b/lean/disaster-recovery/fixtures/rejected.ndjson @@ -0,0 +1,2 @@ +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":1,"kind":"gossip_accepted","message_id":"bad-gossip","source":"A","view":1,"seqno":10,"pre":"GOSSIPING","post":"OPEN"} diff --git a/lean/disaster-recovery/lake-manifest.json b/lean/disaster-recovery/lake-manifest.json new file mode 100644 index 000000000000..4df3dace4b34 --- /dev/null +++ b/lean/disaster-recovery/lake-manifest.json @@ -0,0 +1,116 @@ +{ + "version": "1.1.0", + "packagesDir": ".lake/packages", + "packages": [ + { + "url": "https://github.com/leanprover-community/mathlib4.git", + "type": "git", + "subDir": null, + "scope": "", + "rev": "8f9d9cff6bd728b17a24e163c9402775d9e6a365", + "name": "mathlib", + "manifestFile": "lake-manifest.json", + "inputRev": "v4.28.0", + "inherited": false, + "configFile": "lakefile.lean" + }, + { + "url": "https://github.com/leanprover-community/plausible", + "type": "git", + "subDir": null, + "scope": "leanprover-community", + "rev": "55c8532eb21ec9f6d565d51d96b8ca50bd1fbef3", + "name": "plausible", + "manifestFile": "lake-manifest.json", + "inputRev": "main", + "inherited": true, + "configFile": "lakefile.toml" + }, + { + "url": "https://github.com/leanprover-community/LeanSearchClient", + "type": "git", + "subDir": null, + "scope": "leanprover-community", + "rev": "c5d5b8fe6e5158def25cd28eb94e4141ad97c843", + "name": "LeanSearchClient", + "manifestFile": "lake-manifest.json", + "inputRev": "main", + "inherited": true, + "configFile": "lakefile.toml" + }, + { + "url": "https://github.com/leanprover-community/import-graph", + "type": "git", + "subDir": null, + "scope": "leanprover-community", + "rev": "85b59af46828c029a9168f2f9c35119bd0721e6e", + "name": "importGraph", + "manifestFile": "lake-manifest.json", + "inputRev": "main", + "inherited": true, + "configFile": "lakefile.toml" + }, + { + "url": "https://github.com/leanprover-community/ProofWidgets4", + "type": "git", + "subDir": null, + "scope": "leanprover-community", + "rev": "be3b2e63b1bbf496c478cef98b86972a37c1417d", + "name": "proofwidgets", + "manifestFile": "lake-manifest.json", + "inputRev": "v0.0.87", + "inherited": true, + "configFile": "lakefile.lean" + }, + { + "url": "https://github.com/leanprover-community/aesop", + "type": "git", + "subDir": null, + "scope": "leanprover-community", + "rev": "f642a64c76df8ba9cb53dba3b919425a0c2aeaf1", + "name": "aesop", + "manifestFile": "lake-manifest.json", + "inputRev": "master", + "inherited": true, + "configFile": "lakefile.toml" + }, + { + "url": "https://github.com/leanprover-community/quote4", + "type": "git", + "subDir": null, + "scope": "leanprover-community", + "rev": "b8f98e9087e02c8553945a2c5abf07cec8e798c3", + "name": "Qq", + "manifestFile": "lake-manifest.json", + "inputRev": "master", + "inherited": true, + "configFile": "lakefile.toml" + }, + { + "url": "https://github.com/leanprover-community/batteries", + "type": "git", + "subDir": null, + "scope": "leanprover-community", + "rev": "495c008c3e3f4fb4256ff5582ddb3abf3198026f", + "name": "batteries", + "manifestFile": "lake-manifest.json", + "inputRev": "main", + "inherited": true, + "configFile": "lakefile.toml" + }, + { + "url": "https://github.com/leanprover/lean4-cli", + "type": "git", + "subDir": null, + "scope": "leanprover", + "rev": "4f10f47646cb7d5748d6f423f4a07f98f7bbcc9e", + "name": "Cli", + "manifestFile": "lake-manifest.json", + "inputRev": "v4.28.0", + "inherited": true, + "configFile": "lakefile.toml" + } + ], + "name": "disaster_recovery", + "lakeDir": ".lake" +} diff --git a/lean/disaster-recovery/lakefile.toml b/lean/disaster-recovery/lakefile.toml new file mode 100644 index 000000000000..634272dd805a --- /dev/null +++ b/lean/disaster-recovery/lakefile.toml @@ -0,0 +1,38 @@ +name = "disaster_recovery" +version = "0.1.0" +defaultTargets = [ + "DisasterRecovery", + "disaster-recovery", + "semantic-checks", + "canonical-checks", + "trace-checks", + "trace-validator", +] + +[[require]] +name = "mathlib" +git = "https://github.com/leanprover-community/mathlib4.git" +rev = "v4.28.0" + +[[lean_lib]] +name = "DisasterRecovery" + +[[lean_exe]] +name = "disaster-recovery" +root = "Main" + +[[lean_exe]] +name = "semantic-checks" +root = "Tests" + +[[lean_exe]] +name = "canonical-checks" +root = "CanonicalTests" + +[[lean_exe]] +name = "trace-checks" +root = "TraceTests" + +[[lean_exe]] +name = "trace-validator" +root = "TraceMain" diff --git a/lean/disaster-recovery/lean-toolchain b/lean/disaster-recovery/lean-toolchain new file mode 100644 index 000000000000..4c685fa085fa --- /dev/null +++ b/lean/disaster-recovery/lean-toolchain @@ -0,0 +1 @@ +leanprover/lean4:v4.28.0 diff --git a/tla/disaster-recovery/Readme.md b/tla/disaster-recovery/Readme.md index e337787e0027..d13a98b9ac84 100644 --- a/tla/disaster-recovery/Readme.md +++ b/tla/disaster-recovery/Readme.md @@ -9,3 +9,51 @@ The specification can be checked from the command line via `cargo run check`. However, a more useful UX is via the web-view which is hosted locally via `cargo run serve`. This allows you to explore the specification actions interactively, and the checker can be exhaustively run using the `Run to completion` button, which should find several useful examples of states where the network is opened, and where a deadlock is reached. + +## Exporting the state graph + +`cargo run --quiet -- export --nodes [-o ]` exhaustively enumerates the reachable +state graph (via the public `stateright::Model` interface, i.e. `init_states`/`next_steps`) +and writes it to `` (or stdout) in the shared `ccf-legacy-dr-graph-v1` TSV contract, so +it can be diffed against an independent re-implementation of the same model (e.g. in Python +or Lean). The encoder (`src/export.rs`) never uses `Debug` formatting, so output is insulated +from field order, hash-set iteration order, and library-version changes. Full grammar and +design notes are documented in the module doc comment at the top of `src/export.rs`; summary: + +```text +format ccf-legacy-dr-graph-v1 +nodes +init +state (one per reachable state, ascending ) +edge (one per reachable transition, sorted) +``` + +- `` is a dense integer (`0..N_STATES`) assigned by sorting every reachable state's + `` lexicographically -- _not_ BFS/discovery order -- so numbering is a pure + function of the reachable state set. Edges reference states only by `` (not by + repeating ``), since e.g. `n=3` already has 105,558 states / 552,282 edges and + repeating full state keys per edge does not scale. `edge` records are sorted by the tuple + `(, text, )` -- numeric on the ids, lexicographic on the action -- + and de-duplicated. +- `` is 9 chars of `1`/`0`, one per predicate registered via + `ActorModel::property` (`model.properties`), in registration order -- the exact same `fn` + pointers used by `check`/`serve`, so the export can never drift from their semantics. +- `` is `S([ACTORS],[TIMERS],[ENVELOPES])`: `ACTORS` are semicolon-separated + `s(PHASE,[GOSSIPS],[VOTES],SUBMITTED,txid)` records (`PHASE` in `vote`/`openjoin`/`open0`/ + `open1`/`join`), `TIMERS` are the ids of actors with an active election timeout, and + `ENVELOPES` are `e(src,dst,msg)#count` in flight. `history`/`random_choices`/ + `actor_storages` (always the unit value `()` for this model) and `crashed` (always all + `false`, since `max_crashes` is never set above `0`) are omitted, as they carry no + information here. +- `` is `deliver(src,dst,msg)` or `timeout(id,election)`. +- Set elements (`GOSSIPS`, `VOTES`) and `ENVELOPES` are ordered using the real Rust + `#[derive(Ord)]` implementations of `GossipStruct`/`VoteStruct`/`Envelope` (not a + string sort). +- Per `Model::next_state`'s "`None` = no-op" contract, only actions where `next_state` + returns `Some` produce an `edge` line (`Model::next_steps`'s default implementation + already filters these out). + +`--nodes` is an alias for the existing `--n-nodes`/`-n` flag, and (being a global clap +argument) is accepted either before or after the subcommand, so existing invocations +(`cargo run -- --n-nodes 3 check`, `cargo run check`, `cargo run -- --n-nodes 3 serve`) keep +working unchanged alongside `cargo run --quiet -- export --nodes `. diff --git a/tla/disaster-recovery/src/export.rs b/tla/disaster-recovery/src/export.rs new file mode 100644 index 000000000000..f9988717b9ee --- /dev/null +++ b/tla/disaster-recovery/src/export.rs @@ -0,0 +1,370 @@ +//! Dependency-free canonical export of the reachable state graph. +//! +//! Implements the shared `ccf-legacy-dr-graph-v1` TSV contract: the model is +//! enumerated exhaustively using only the public `stateright::Model` +//! interface (`init_states`, `next_steps`, `within_boundary`), and every +//! state/action is serialized with an explicit hand-written grammar (never +//! `Debug`), so the output is stable across compiler/library versions and +//! diffable byte-for-byte against an independent re-implementation (e.g. +//! Python, Lean) of the same state machine. +//! +//! No new dependencies are introduced: only `stateright` (already a direct +//! dependency) and `std` are used. +//! +//! # Format +//! +//! ```text +//! format\tccf-legacy-dr-graph-v1 +//! nodes\t +//! init\t +//! state\t\t\t (one per reachable state) +//! edge\t\t\t (one per reachable transition) +//! ``` +//! +//! `` is a canonical, dense integer id (`0..N_STATES`) assigned by +//! sorting every reachable state's `` (see below) lexicographically +//! and numbering them in that order -- *not* BFS/discovery order -- so ids are +//! reproducible independent of traversal strategy. `state` records are +//! emitted in ascending `` order (equivalently, ascending `` +//! order). `edge` records are emitted sorted by the tuple +//! `(, text, )` (numeric on the ids, lexicographic on +//! the action text), and de-duplicated. Repeating the full `` in +//! every edge does not scale (e.g. `n=3` already has 105,558 states / 552,282 +//! edges), so edges reference states only by ``; a reader reconstructs the +//! `` for any `` via the `state` block. +//! +//! `` is exactly 9 characters of `1`/`0`, one per predicate +//! currently registered on the model via `ActorModel::property` +//! (`model.properties`), in registration order (liveness, then invariant, +//! then reachable properties -- *not* alphabetical). Each bit is the exact +//! existing `Property::condition` closure evaluated on that state, so the +//! export can never drift from `check`/`serve` behaviour, and preserves each +//! predicate's existing (sometimes misleadingly worded) name/meaning even +//! though names themselves are not repeated in the TSV output. +//! +//! Grammar for ``/`` tokens (no token contains whitespace): +//! +//! - gossip: `g(src,txid)` +//! - vote: `v(src,[GOSSIPS])` where `GOSSIPS` is a comma-separated gossip list +//! - msg: a gossip, a vote, or `o(id)` (`IAmOpen`) +//! - envelope: `e(src,dst,msg)` +//! - submitted vote: `none` or `some(dst,vote)` +//! - actor: `s(PHASE,[GOSSIPS],[VOTES],SUBMITTED,txid)` where `PHASE` is one +//! of `vote`, `openjoin`, `open0` (`Open { timeout: false }`), `open1` +//! (`Open { timeout: true }`), `join` +//! - global state: `S([ACTORS],[TIMERS],[ENVELOPES])` where `ACTORS` is +//! semicolon-separated (positional, by actor index), `TIMERS` is a +//! comma-separated list of actor ids with an active election timeout, and +//! `ENVELOPES` is a comma-separated list of `envelope#count` (count being +//! the in-flight multiplicity of that exact envelope) +//! - action: `deliver(src,dst,msg)` or `timeout(id,election)` +//! +//! `history` (`H = ()`), `random_choices` (`Node::Random = ()`), +//! `actor_storages` (`Node::Storage = ()`), and `crashed` (always all-`false`, +//! since `max_crashes` is never configured above `0`) are all omitted from +//! `S(...)`: for this model they are always constant/empty and carry no +//! information. +//! +//! Set elements (`GOSSIPS`, `VOTES`) are ordered using the actual Rust +//! `#[derive(Ord)]` implementation of `GossipStruct`/`VoteStruct` (not a +//! string sort), and `ENVELOPES` are ordered using `Envelope`'s derived +//! `Ord`. Per `Model::next_state`'s documented contract ("`None` indicates +//! the action does not change state"), only actions for which `next_state` +//! returns `Some` produce an edge; this is preserved by using +//! `Model::next_steps`, whose default implementation already filters out +//! `None` results. + +use crate::model::{GossipStruct, ModelCfg, Msg, NextStep, Node, State, Timer, VoteStruct}; +use stateright::actor::{ + ActorModel, ActorModelAction, ActorModelState, Envelope, Id, Network, Timers, +}; +use stateright::Model; +use std::collections::{HashMap, VecDeque}; +use std::io::{self, Write}; + +fn fmt_id(id: Id) -> String { + usize::from(id).to_string() +} + +fn fmt_gossip(g: &GossipStruct) -> String { + format!("g({},{})", fmt_id(g.src), g.txid) +} + +/// Clones and sorts a gossip set using `GossipStruct`'s derived `Ord` +/// (compares `src` then `txid`), per the shared contract's "sort set +/// elements by Rust derived Ord". +fn sorted_gossips(set: &stateright::util::HashableHashSet) -> Vec { + let mut v: Vec = set.iter().cloned().collect(); + v.sort(); + v +} + +fn fmt_gossip_list(set: &stateright::util::HashableHashSet) -> String { + let items: Vec = sorted_gossips(set).iter().map(fmt_gossip).collect(); + format!("[{}]", items.join(",")) +} + +fn fmt_vote(v: &VoteStruct) -> String { + format!("v({},{})", fmt_id(v.src), fmt_gossip_list(&v.recv)) +} + +/// Clones and sorts a vote set using `VoteStruct`'s derived `Ord` (compares +/// `src` then `recv`). +fn sorted_votes(set: &stateright::util::HashableHashSet) -> Vec { + let mut v: Vec = set.iter().cloned().collect(); + v.sort(); + v +} + +fn fmt_vote_list(set: &stateright::util::HashableHashSet) -> String { + let items: Vec = sorted_votes(set).iter().map(fmt_vote).collect(); + format!("[{}]", items.join(",")) +} + +fn fmt_submitted(sv: &Option<(Id, VoteStruct)>) -> String { + match sv { + None => "none".to_string(), + Some((dst, vote)) => format!("some({},{})", fmt_id(*dst), fmt_vote(vote)), + } +} + +fn fmt_phase(n: &NextStep) -> &'static str { + match n { + NextStep::Vote => "vote", + NextStep::OpenJoin => "openjoin", + NextStep::Open { timeout: false } => "open0", + NextStep::Open { timeout: true } => "open1", + NextStep::Join => "join", + } +} + +fn fmt_actor(s: &State) -> String { + format!( + "s({},{},{},{},{})", + fmt_phase(&s.next_step), + fmt_gossip_list(&s.gossips), + fmt_vote_list(&s.votes), + fmt_submitted(&s.submitted_vote), + s.txid, + ) +} + +fn fmt_msg(m: &Msg) -> String { + match m { + Msg::Gossip(g) => fmt_gossip(g), + Msg::Vote(v) => fmt_vote(v), + Msg::IAmOpen(id) => format!("o({})", fmt_id(*id)), + } +} + +fn fmt_envelope(env: &Envelope) -> String { + format!( + "e({},{},{})", + fmt_id(env.src), + fmt_id(env.dst), + fmt_msg(&env.msg) + ) +} + +/// Tallies in-flight multiplicity per distinct envelope. `Network::iter_all` +/// yields one item per unit of multiplicity regardless of the underlying +/// `Network` variant (this model only ever uses +/// `new_unordered_nonduplicating`, whose internal representation already +/// tracks a count directly), so tallying via `iter_all` is variant-agnostic +/// and stays correct if the network configuration ever changes. +fn network_counts(network: &Network) -> Vec<(Envelope, usize)> { + let mut counts: HashMap, usize> = HashMap::new(); + for env in network.iter_all() { + *counts.entry(env.to_cloned_msg()).or_insert(0) += 1; + } + let mut v: Vec<(Envelope, usize)> = counts.into_iter().collect(); + // Envelope's derived Ord (src, dst, msg), per the shared contract. + v.sort_by(|a, b| a.0.cmp(&b.0)); + v +} + +fn fmt_network(network: &Network) -> String { + let items: Vec = network_counts(network) + .iter() + .map(|(env, count)| format!("{}#{}", fmt_envelope(env), count)) + .collect(); + format!("[{}]", items.join(",")) +} + +/// Comma-separated, ascending list of actor ids with an active election +/// timeout. `Timer` currently has a single variant, so presence alone is +/// significant (no timer-kind tag is emitted). +fn fmt_timers(timers_set: &[Timers]) -> String { + let mut ids: Vec = timers_set + .iter() + .enumerate() + .filter(|(_, t)| t.iter().next().is_some()) + .map(|(i, _)| i) + .collect(); + ids.sort_unstable(); + let items: Vec = ids.iter().map(|i| i.to_string()).collect(); + format!("[{}]", items.join(",")) +} + +/// Canonical `S(...)` encoding of a full `ActorModelState`. Used both +/// as the state field in `state` records and as the basis of the canonical +/// state id, so two independent implementations that compute the same +/// reachable state always produce the same key, regardless of traversal order. +pub fn fmt_state(state: &ActorModelState) -> String { + let actors: Vec = state.actor_states.iter().map(|s| fmt_actor(s)).collect(); + format!( + "S([{}],{},{})", + actors.join(";"), + fmt_timers(&state.timers_set), + fmt_network(&state.network), + ) +} + +/// Canonical encoding of an `ActorModelAction`. Only `Deliver` and `Timeout` +/// are part of the `ccf-legacy-dr-graph-v1` contract: this model never +/// produces `Drop` (`LossyNetwork::No`), `Crash`/`Recover` (`max_crashes == +/// 0`), or `SelectRandom` (no `Actor` ever issues a `ChooseRandom` command), +/// so encountering one is a bug (e.g. a future model config change) rather +/// than a case the contract needs to define. +pub fn fmt_action(action: &ActorModelAction) -> String { + match action { + ActorModelAction::Deliver { src, dst, msg } => { + format!( + "deliver({},{},{})", + fmt_id(*src), + fmt_id(*dst), + fmt_msg(msg) + ) + } + ActorModelAction::Timeout(id, Timer::ElectionTimeout) => { + format!("timeout({},election)", fmt_id(*id)) + } + other => unreachable!( + "action variant {:?} is outside the ccf-legacy-dr-graph-v1 contract \ + (only Deliver/Timeout are ever produced by this model's configuration)", + other + ), + } +} + +/// The 9-character `1`/`0` bitstring for `state`, one bit per predicate +/// currently registered on `model` (`model.properties`) in registration +/// order -- i.e. exactly the same `fn` pointers used by `check`/`serve`, so +/// this can never drift from their semantics. +fn predicate_bitstring( + model: &ActorModel, + state: &ActorModelState, +) -> String { + model + .properties + .iter() + .map(|p| { + if (p.condition)(model, state) { + '1' + } else { + '0' + } + }) + .collect() +} + +/// Exhaustively enumerates the reachable state graph of `model` via the +/// public `stateright::Model` interface (`init_states`, `next_steps`, +/// `within_boundary`) and writes it to `out` as `ccf-legacy-dr-graph-v1`. +/// +/// States are discovered by BFS (for traversal only), but ``s are +/// assigned afterwards by sorting all discovered ``s +/// lexicographically -- so the numbering is a pure function of the reachable +/// state set, independent of traversal order. Edges reference states by +/// `` only, keeping output size linear in (states + edges) rather than +/// (edges * average state size). +pub fn export_graph( + model: &ActorModel, + out: &mut W, +) -> io::Result<()> { + // Indexed by BFS discovery order (a "discovery id"); remapped to the + // canonical sorted-key id only once the full state set is known. + let mut visited: HashMap, usize> = HashMap::new(); + let mut keys: Vec = Vec::new(); + let mut bits: Vec = Vec::new(); + let mut frontier: VecDeque> = VecDeque::new(); + // (discovery src id, action text, discovery dst id) + let mut edges: Vec<(usize, String, usize)> = Vec::new(); + + let mut init_states = model.init_states(); + assert_eq!( + init_states.len(), + 1, + "ccf-legacy-dr-graph-v1 assumes a single deterministic init state" + ); + let init_state = init_states.remove(0); + assert!( + model.within_boundary(&init_state), + "ccf-legacy-dr-graph-v1 assumes the init state is within the model boundary" + ); + let init_discovery_id = keys.len(); + keys.push(fmt_state(&init_state)); + bits.push(predicate_bitstring(model, &init_state)); + visited.insert(init_state.clone(), init_discovery_id); + frontier.push_back(init_state); + + while let Some(s) = frontier.pop_front() { + let src_discovery_id = *visited + .get(&s) + .expect("every frontier state was inserted into `visited` before being queued"); + // `next_steps` (default `Model` trait method) already filters out + // actions for which `next_state` returns `None`, preserving the + // documented no-op-suppression contract. + for (action, ns) in model.next_steps(&s) { + if !model.within_boundary(&ns) { + continue; + } + let action_key = fmt_action(&action); + let dst_discovery_id = if let Some(&id) = visited.get(&ns) { + id + } else { + let id = keys.len(); + keys.push(fmt_state(&ns)); + bits.push(predicate_bitstring(model, &ns)); + visited.insert(ns.clone(), id); + frontier.push_back(ns); + id + }; + edges.push((src_discovery_id, action_key, dst_discovery_id)); + } + } + + // Canonical id assignment: number every discovered state by the + // lexicographic order of its ``, not by discovery order. + let mut order: Vec = (0..keys.len()).collect(); + order.sort_by(|&a, &b| keys[a].cmp(&keys[b])); + let mut canonical_id: Vec = vec![0; keys.len()]; + for (id, &discovery_id) in order.iter().enumerate() { + canonical_id[discovery_id] = id; + } + + // Remap edges to canonical ids, then sort by (SRC_ID, ACTION, DST_ID) -- + // numeric on the ids (real `usize` comparison, not string comparison), + // lexicographic on the action text -- and de-duplicate. + let mut canonical_edges: Vec<(usize, String, usize)> = edges + .into_iter() + .map(|(src, action, dst)| (canonical_id[src], action, canonical_id[dst])) + .collect(); + canonical_edges.sort(); + canonical_edges.dedup(); + + writeln!(out, "format\tccf-legacy-dr-graph-v1")?; + writeln!(out, "nodes\t{}", model.actors.len())?; + writeln!(out, "init\t{}", canonical_id[init_discovery_id])?; + for (id, &discovery_id) in order.iter().enumerate() { + writeln!( + out, + "state\t{}\t{}\t{}", + id, keys[discovery_id], bits[discovery_id] + )?; + } + for (src, action, dst) in &canonical_edges { + writeln!(out, "edge\t{src}\t{action}\t{dst}")?; + } + Ok(()) +} diff --git a/tla/disaster-recovery/src/main.rs b/tla/disaster-recovery/src/main.rs index d92767d1a0f0..15bcef28f7ad 100644 --- a/tla/disaster-recovery/src/main.rs +++ b/tla/disaster-recovery/src/main.rs @@ -1,7 +1,9 @@ extern crate clap; extern crate stateright; use clap::Parser; +mod export; mod model; +use export::export_graph; use model::{ModelCfg, Msg, NextStep, Node, State}; use stateright::{actor::*, report::WriteReporter, util::HashableHashSet, Checker, Model}; use std::sync::Arc; @@ -198,7 +200,11 @@ fn properties(model: ActorModel) -> ActorModel, + }, } fn check(model: ActorModel) { @@ -227,6 +241,21 @@ fn serve(model: ActorModel) { checker.serve("localhost:8080"); } +fn export(model: ActorModel, out: Option) { + match out { + Some(path) => { + let mut file = std::fs::File::create(&path) + .unwrap_or_else(|e| panic!("failed to create '{}': {}", path, e)); + export_graph(&model, &mut file).expect("failed to write model export"); + } + None => { + let stdout = std::io::stdout(); + let mut handle = stdout.lock(); + export_graph(&model, &mut handle).expect("failed to write model export"); + } + } +} + fn main() { let args = CliArgs::parse(); @@ -240,5 +269,6 @@ fn main() { match args.command { Commands::Check => check(model), Commands::Serve => serve(model), + Commands::Export { out } => export(model, out), } } From 177fac81e13b90530bfe4c0f6366e510de13b5d7 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Sun, 30 Aug 2026 10:02:07 +0100 Subject: [PATCH 02/14] Record migration attribution Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 46d56f8a-935f-4551-abe3-84bd4f951865 From f400dd6e3d00f64aa0e232205eb361aacff00d6e Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Sun, 30 Aug 2026 13:39:41 +0100 Subject: [PATCH 03/14] Validate C++ recovery traces with Lean Add commit-aware recovery protocol instrumentation, causal log merging, and terminal trace checks for quorum, failover, and multiple-timeout SNP scenarios. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 46d56f8a-935f-4551-abe3-84bd4f951865 --- .github/workflows/README.md | 7 +- .github/workflows/ci.yml | 38 ++- .github/workflows/lean-shallow.yml | 6 + CMakeLists.txt | 9 + .../ccf/service/tables/self_healing_open.h | 26 ++ .../DisasterRecovery/Protocol/Trace.lean | 23 +- lean/disaster-recovery/README.md | 23 +- lean/disaster-recovery/TRACE_FORMAT_V1.md | 31 +- lean/disaster-recovery/TraceTests.lean | 27 ++ src/node/recovery_decision_protocol.cpp | 294 +++++++++++++++++- src/node/recovery_decision_protocol.h | 48 +++ src/node/rpc/self_healing_open_handlers.h | 59 +++- tests/e2e_operations.py | 10 + tests/infra/recovery_trace.py | 234 ++++++++++++++ tests/infra/recovery_trace_test.py | 172 ++++++++++ 15 files changed, 978 insertions(+), 29 deletions(-) create mode 100644 tests/infra/recovery_trace.py create mode 100644 tests/infra/recovery_trace_test.py diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 8d0da5157804..5b47a8acedbb 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -103,9 +103,10 @@ File: `tla-shallow.yml` # Lean Shallow Verification -Builds and checks the Lean disaster-recovery models, validates trace fixtures, -and compares the bounded Lean legacy model with Stateright on relevant pull -requests. +Builds and checks the Lean disaster-recovery models, validates trace fixtures +and the causal log merger, and compares the bounded Lean legacy model with +Stateright on relevant pull requests. The SNP jobs in `ci.yml` additionally +validate committed C++ recovery traces. File: `lean-shallow.yml` 3rd party dependencies: None diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0ff26ae8765..843b6ceb250a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -292,13 +292,28 @@ jobs: python3 tests/infra/platform_detection.py snp milan shell: bash + - name: "Build Lean recovery trace validator" + run: | + set -euo pipefail + curl --proto '=https' --tlsv1.2 -sSf \ + https://raw.githubusercontent.com/leanprover/elan/58e8d545e33641f66dbcbd22c4283109e71757be/elan-init.sh \ + -o /tmp/elan-init.sh + sh /tmp/elan-init.sh -y --default-toolchain none + rm /tmp/elan-init.sh + export PATH="${HOME}/.elan/bin:${PATH}" + elan toolchain install "$(cat lean/disaster-recovery/lean-toolchain)" + cd lean/disaster-recovery + lake exe cache get + lake build trace-validator + shell: bash + - name: "Build Debug" run: | set -ex git config --global --add safe.directory /__w/CCF/CCF mkdir build cd build - cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DWORKER_THREADS=1 .. + cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DWORKER_THREADS=1 -DCCF_RECOVERY_TRACE=ON .. ninja shell: bash @@ -314,6 +329,7 @@ jobs: shell: bash env: CCF_TEST_SYNC_AFTER_SETUP: 1 + CCF_LEAN_TRACE_VALIDATOR: ${{ github.workspace }}/lean/disaster-recovery/.lake/build/bin/trace-validator ELECTION_TIMEOUT_MS: 10000 - name: "Capture dmesg" @@ -336,6 +352,7 @@ jobs: build/workspace/*/out build/workspace/*/err build/workspace/*/*.ledger/* + build/workspace/**/*.recovery.ndjson build/workspace/*/stack_trace build/workspace/**/openapi_coverage.json if-no-files-found: ignore @@ -378,13 +395,28 @@ jobs: python3 tests/infra/platform_detection.py snp genoa shell: bash + - name: "Build Lean recovery trace validator" + run: | + set -euo pipefail + curl --proto '=https' --tlsv1.2 -sSf \ + https://raw.githubusercontent.com/leanprover/elan/58e8d545e33641f66dbcbd22c4283109e71757be/elan-init.sh \ + -o /tmp/elan-init.sh + sh /tmp/elan-init.sh -y --default-toolchain none + rm /tmp/elan-init.sh + export PATH="${HOME}/.elan/bin:${PATH}" + elan toolchain install "$(cat lean/disaster-recovery/lean-toolchain)" + cd lean/disaster-recovery + lake exe cache get + lake build trace-validator + shell: bash + - name: "Build Debug" run: | set -ex git config --global --add safe.directory /__w/CCF/CCF mkdir build cd build - cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DWORKER_THREADS=1 .. + cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DWORKER_THREADS=1 -DCCF_RECOVERY_TRACE=ON .. ninja shell: bash @@ -400,6 +432,7 @@ jobs: shell: bash env: CCF_TEST_SYNC_AFTER_SETUP: 1 + CCF_LEAN_TRACE_VALIDATOR: ${{ github.workspace }}/lean/disaster-recovery/.lake/build/bin/trace-validator ELECTION_TIMEOUT_MS: 10000 - name: "Capture dmesg" @@ -422,6 +455,7 @@ jobs: build/workspace/*/out build/workspace/*/err build/workspace/*/*.ledger/* + build/workspace/**/*.recovery.ndjson build/workspace/*/stack_trace build/workspace/**/openapi_coverage.json if-no-files-found: ignore diff --git a/.github/workflows/lean-shallow.yml b/.github/workflows/lean-shallow.yml index 3e2e72593c52..fbb84404151e 100644 --- a/.github/workflows/lean-shallow.yml +++ b/.github/workflows/lean-shallow.yml @@ -10,6 +10,10 @@ on: - "src/node/recovery_decision_protocol.cpp" - "src/node/recovery_decision_protocol.h" - "src/node/rpc/self_healing_open_handlers.h" + - "tests/e2e_operations.py" + - "tests/infra/recovery_trace.py" + - "CMakeLists.txt" + - ".github/workflows/ci.yml" - ".github/workflows/lean-shallow.yml" concurrency: @@ -69,6 +73,8 @@ jobs: echo "Rejected causal trace was accepted" exit 1 fi + PYTHONPATH=../../tests python3 -m unittest discover \ + -s ../../tests/infra -p recovery_trace_test.py - name: Compare Lean and Stateright working-directory: lean/disaster-recovery diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f075dd6a70b..b99d7ac34f7d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -458,6 +458,15 @@ if(CCF_RAFT_TRACING) add_compile_definitions(CCF_RAFT_TRACING) endif() +option( + CCF_RECOVERY_TRACE + "Enable committed recovery-decision-protocol tracing" + OFF +) +if(CCF_RECOVERY_TRACE) + add_compile_definitions(CCF_RECOVERY_TRACE) +endif() + # Build common library for CCF enclaves set( CCF_IMPL_SOURCE diff --git a/include/ccf/service/tables/self_healing_open.h b/include/ccf/service/tables/self_healing_open.h index 20066451e8ee..21f6dfed8561 100644 --- a/include/ccf/service/tables/self_healing_open.h +++ b/include/ccf/service/tables/self_healing_open.h @@ -94,6 +94,28 @@ namespace ccf using TimeoutSMState = ServiceValue; using OpenKind = ServiceValue; + +#ifdef CCF_RECOVERY_TRACE + struct TraceEvent + { + std::string kind; + std::optional message_id = std::nullopt; + std::optional caused_by = std::nullopt; + std::optional source = std::nullopt; + std::optional view = std::nullopt; + std::optional seqno = std::nullopt; + std::string pre; + std::string post; + std::optional open_kind = std::nullopt; + std::optional send = std::nullopt; + }; + DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(TraceEvent); + DECLARE_JSON_REQUIRED_FIELDS(TraceEvent, kind, pre, post); + DECLARE_JSON_OPTIONAL_FIELDS( + TraceEvent, message_id, caused_by, source, view, seqno, open_kind, send); + + using TraceEvents = ServiceMap; +#endif } namespace Tables @@ -112,5 +134,9 @@ namespace ccf "public:ccf.gov.recovery_decision_protocol.timeout_sm_state"; static constexpr auto RECOVERY_DECISION_PROTOCOL_OPEN_KIND = "public:ccf.gov.recovery_decision_protocol.open_kind"; +#ifdef CCF_RECOVERY_TRACE + static constexpr auto RECOVERY_DECISION_PROTOCOL_TRACE_EVENTS = + "public:ccf.internal.recovery_decision_protocol.trace_events"; +#endif } } diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/Trace.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/Trace.lean index af486ee32185..5611fabf9aef 100644 --- a/lean/disaster-recovery/DisasterRecovery/Protocol/Trace.lean +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/Trace.lean @@ -375,6 +375,19 @@ private def observationCompatible state.phase == .open | _ => false +private def historicalSendCompatible + (candidate : Candidate) + (event : TraceEvent) : Bool := + match nodeState candidate.system event.node, event.send with + | some state, some description => + phaseMatches event.pre state.phase && + phaseMatches event.post state.phase && + candidate.hiddenSends.contains { + source := event.node + description + } + | _, _ => false + private def removePendingEffect (node : Location) (target : Effect) : List PendingEffect -> Option (List PendingEffect) | [] => none @@ -404,19 +417,25 @@ private def consumeObservation (config : Config) (candidate : Candidate) (event : TraceEvent) : Option Candidate := do - guard (observationCompatible config candidate.system event) match event.kind with - | .send => some candidate + | .send => do + guard ( + observationCompatible config candidate.system event || + historicalSendCompatible candidate event) + some candidate | .open => do + guard (observationCompatible config candidate.system event) let kind <- event.openKind let pendingEffects <- removePendingEffect event.node (.opening kind) candidate.pendingEffects some { candidate with pendingEffects } | .joinRestart => do + guard (observationCompatible config candidate.system event) let pendingEffects <- removePendingRestart event.node candidate.pendingEffects some { candidate with pendingEffects } | .complete => do + guard (observationCompatible config candidate.system event) let pendingEffects <- removePendingEffect event.node .completed candidate.pendingEffects some { candidate with pendingEffects } diff --git a/lean/disaster-recovery/README.md b/lean/disaster-recovery/README.md index 9cd56d1b2fb7..fa446d02e090 100644 --- a/lean/disaster-recovery/README.md +++ b/lean/disaster-recovery/README.md @@ -317,6 +317,9 @@ incompatible histories. Candidates also retain one-shot committed effects; for the node that produced them. On failure the validator reports the shortest failing prefix and expected compatible events. +Explicit sends may match an accumulated earlier capability when a retry task +selected work before a concurrent state commit and dispatched it afterward. + The canonical v1 relation is deterministic, so a successful v1 prefix currently retains one candidate. The candidate list and separate histories are explicit for future under-observed or nondeterministic refinements. @@ -327,6 +330,20 @@ unique message IDs, feasible accepted receives from configured sources, and one receive per supplied causal send ID. Observable `pre`, `post`, and open-kind values are checked against every remaining candidate state. +### Implementation trace validation + +`CCF_RECOVERY_TRACE` enables C++ tracing without changing default builds. +Protocol transactions append semantic events to an internal public trace map; +its global commit hook emits `RDP_TRACE` records only after commit. Sends are +logged before dispatch with causal IDs propagated to accepted receive records. + +[`tests/infra/recovery_trace.py`](../../tests/infra/recovery_trace.py) extracts +records from all recovery nodes and topologically orders them from per-node +sequence and causal edges, without comparing clocks. It then invokes the Lean +validator. The quorum, failover, and multiple-timeout recovery scenarios call +this helper, and both SNP CI jobs build CCF with tracing enabled and provide the +Lean validator binary. + ## Commands From this directory: @@ -391,6 +408,6 @@ The Stateright model can be removed only after: 3. committed traces from the quorum, failover, and multiple-timeout C++ e2e scenarios validate against the canonical Lean model. -This change establishes the first two migration mechanisms and freezes the -trace contract needed by the third. It does not claim implementation -conformance before C++ emits committed semantic events. +The migration now implements all three mechanisms. Stateright should remain +until the bounded comparison and SNP implementation-trace jobs have established +a stable green history. diff --git a/lean/disaster-recovery/TRACE_FORMAT_V1.md b/lean/disaster-recovery/TRACE_FORMAT_V1.md index a274264b0194..3b17976bb921 100644 --- a/lean/disaster-recovery/TRACE_FORMAT_V1.md +++ b/lean/disaster-recovery/TRACE_FORMAT_V1.md @@ -100,6 +100,9 @@ allows delayed delivery after a sender changes phase without merging incompatible executions. `open`, `join_restart`, and `complete` each consume one matching pending effect from the observed node, so one node cannot replay its transition or consume another node's effect. +An explicit send may also match an earlier send capability: this models a +retry task that selected its work before a concurrent protocol commit and +dispatched it afterward. The validator computes a finite hidden closure over retry/send stuttering before and after each observation. It does not guess protocol-state changes. `pre` and `post` filter all candidates. Message IDs constrain causal matching @@ -115,14 +118,28 @@ prints the observed incompatible kind plus expected compatible events from the last nonempty candidate set. A successful parse with no compatible execution is never reported as success. -## Instrumentation transaction rule +## C++ instrumentation -Future C++ instrumentation must emit only after the transaction containing the -modeled state change commits. It must not log a handler mutation, open -transition, timeout-lane advance, or restart side effect from a transaction -that later aborts. Validation/HTTP rejection events that perform no state -mutation may be emitted at their final rejection boundary. This slice defines -the contract and Lean validator only; it does not instrument C++. +Configure CCF with `-DCCF_RECOVERY_TRACE=ON` to enable implementation tracing. +Accepted receive and timeout events are written to +`public:ccf.internal.recovery_decision_protocol.trace_events` in the same +transaction as the modeled state change. A global commit hook emits them only +after commit, followed by any `open`, `join_restart`, or `complete` effect from +that transition. Aborted transactions therefore emit nothing. + +The committed start hook emits `start` before scheduling retry and failover +tasks. Transport sends are emitted immediately before dispatch and propagate +their generated `message_id` in the internal request as `trace_message_id`; +the committed receive records it as `caused_by`. + +Each log record contains `RDP_TRACE ` followed by the event object. +`tests/infra/recovery_trace.py` extracts records from all participating node +logs, topologically orders them by per-node sequence and causal send edges, +writes NDJSON, and invokes the Lean validator. The quorum, failover, and +multiple-timeout SNP e2e scenarios call this helper. + +Validation/HTTP rejection events that perform no state mutation remain +optional; the current C++ instrumentation records successful committed paths. ## Example diff --git a/lean/disaster-recovery/TraceTests.lean b/lean/disaster-recovery/TraceTests.lean index 457cf33a6f39..220edac7887a 100644 --- a/lean/disaster-recovery/TraceTests.lean +++ b/lean/disaster-recovery/TraceTests.lean @@ -156,6 +156,33 @@ def main : IO UInt32 := do [start, retry, send, received, openingVote, openedOnce, openedTwice] 7) "one opening transition produced multiple committed open observations" + let votingGossip := { + baseEvent single "A" 1 .gossipAccepted with + messageId := some "voting-gossip" + source := some "A" + txid := some { view := 1, seqno := 1 } + pre := some .gossiping + post := some .voting + } + let staleOpeningVote := { + baseEvent single "A" 2 .voteAccepted with + messageId := some "voting-vote" + source := some "A" + pre := some .voting + post := some .opening + } + let delayedVoteSend := { + baseEvent single "A" 3 .send with + messageId := some "delayed-vote-send" + pre := some .opening + post := some .opening + send := some "vote:A" + } + match validate [start, votingGossip, staleOpeningVote, delayedVoteSend] with + | .ok 1 => pure () + | result => + throw (IO.userError s!"stale retry send was rejected: {repr result}") + let hiddenReceive := { baseEvent single "A" 1 .gossipAccepted with messageId := some "receive-hidden" diff --git a/src/node/recovery_decision_protocol.cpp b/src/node/recovery_decision_protocol.cpp index 4a2d56eac381..974953380421 100644 --- a/src/node/recovery_decision_protocol.cpp +++ b/src/node/recovery_decision_protocol.cpp @@ -19,12 +19,257 @@ namespace ccf { +#ifdef CCF_RECOVERY_TRACE + static constexpr auto RECOVERY_TRACE_VERSION = + "ccf.recovery_decision_protocol.trace/1"; + static constexpr auto RECOVERY_TRACE_MARKER = "RDP_TRACE"; + + static std::string trace_state_name( + recovery_decision_protocol::StateMachine state) + { + switch (state) + { + case recovery_decision_protocol::StateMachine::GOSSIPING: + return "GOSSIPING"; + case recovery_decision_protocol::StateMachine::VOTING: + return "VOTING"; + case recovery_decision_protocol::StateMachine::OPENING: + return "OPENING"; + case recovery_decision_protocol::StateMachine::JOINING: + return "JOINING"; + case recovery_decision_protocol::StateMachine::OPEN: + return "OPEN"; + default: + throw std::logic_error("Unknown recovery-decision-protocol state"); + } + } + + static std::string trace_open_kind_name( + recovery_decision_protocol::OpenKinds kind) + { + switch (kind) + { + case recovery_decision_protocol::OpenKinds::QUORUM: + return "QUORUM"; + case recovery_decision_protocol::OpenKinds::FAILOVER: + return "FAILOVER"; + default: + throw std::logic_error("Unknown recovery-decision-protocol open kind"); + } + } +#endif RecoveryDecisionProtocolSubsystem::RecoveryDecisionProtocolSubsystem( NodeState* node_state_) : node_state(node_state_) {} +#ifdef CCF_RECOVERY_TRACE + void RecoveryDecisionProtocolSubsystem::initialise_trace(ccf::kv::Tx& tx) + { + const auto previous_service_cert = + tx.ro(node_state->network.previous_service_identity)->get(); + if (!previous_service_cert.has_value()) + { + throw std::logic_error( + "Previous service identity not found while initialising " + "recovery-decision-protocol tracing"); + } + + { + std::lock_guard guard(trace_lock); + next_trace_record_id = 0; + next_trace_sequence = 0; + next_trace_message_number = 0; + trace_instance_id = + recovery_decision_protocol::service_fingerprint_from_pem( + previous_service_cert.value()); + trace_node = get_location().name; + trace_committed_state = "GOSSIPING"; + trace_expected_locations.clear(); + for (const auto& location : get_config().expected_locations) + { + trace_expected_locations.push_back(location.name); + } + } + + node_state->network.tables->set_global_hook( + Tables::RECOVERY_DECISION_PROTOCOL_TRACE_EVENTS, + recovery_decision_protocol::TraceEvents::wrap_commit_hook( + [this]( + ccf::kv::Version, + const recovery_decision_protocol::TraceEvents::Write& writes) { + for (const auto& [_, event] : writes) + { + if (event.has_value()) + { + emit_trace_event(event.value()); + } + } + })); + } + + void RecoveryDecisionProtocolSubsystem::emit_trace_event( + recovery_decision_protocol::TraceEvent event) + { + std::lock_guard guard(trace_lock); + if (event.kind == "send") + { + event.pre = trace_committed_state; + event.post = trace_committed_state; + } + else + { + trace_committed_state = event.post; + } + nlohmann::json trace = event; + trace["version"] = RECOVERY_TRACE_VERSION; + trace["instance"] = trace_instance_id; + trace["expected_locations"] = trace_expected_locations; + trace["node"] = trace_node; + trace["sequence"] = next_trace_sequence++; + LOG_INFO_FMT("{} {}", RECOVERY_TRACE_MARKER, trace.dump()); + } + + void RecoveryDecisionProtocolSubsystem::record_trace_event( + ccf::kv::Tx& tx, recovery_decision_protocol::TraceEvent event) + { + uint64_t record_id = 0; + { + std::lock_guard guard(trace_lock); + record_id = next_trace_record_id++; + } + tx.rw( + Tables::RECOVERY_DECISION_PROTOCOL_TRACE_EVENTS) + ->put(record_id, std::move(event)); + } + + std::string RecoveryDecisionProtocolSubsystem::new_trace_message_id() + { + std::lock_guard guard(trace_lock); + return fmt::format( + "{}:{}:{}", trace_instance_id, trace_node, next_trace_message_number++); + } + + recovery_decision_protocol::StateMachine RecoveryDecisionProtocolSubsystem:: + get_trace_state(kv::ReadOnlyTx& tx) + { + const auto state = tx.ro( + Tables::RECOVERY_DECISION_PROTOCOL_SM_STATE) + ->get(); + if (!state.has_value()) + { + throw std::logic_error( + "Recovery-decision-protocol state not set while tracing"); + } + return state.value(); + } + + void RecoveryDecisionProtocolSubsystem::emit_trace_send( + const std::string& message_id, const std::string& description) + { + emit_trace_event({ + .kind = "send", + .message_id = message_id, + .pre = "", + .post = "", + .send = description, + }); + } + + void RecoveryDecisionProtocolSubsystem::record_trace_effects( + ccf::kv::Tx& tx, + recovery_decision_protocol::StateMachine pre, + recovery_decision_protocol::StateMachine post) + { + if ( + post == recovery_decision_protocol::StateMachine::OPENING && + pre != recovery_decision_protocol::StateMachine::OPENING) + { + const auto open_kind = tx.ro( + Tables::RECOVERY_DECISION_PROTOCOL_OPEN_KIND) + ->get(); + if (!open_kind.has_value()) + { + throw std::logic_error( + "Recovery-decision-protocol open kind not set while tracing"); + } + record_trace_event( + tx, + { + .kind = "open", + .pre = "OPENING", + .post = "OPENING", + .open_kind = trace_open_kind_name(open_kind.value()), + }); + } + + if (post == recovery_decision_protocol::StateMachine::JOINING) + { + record_trace_event( + tx, + { + .kind = "join_restart", + .pre = "JOINING", + .post = "JOINING", + }); + } + + if ( + pre == recovery_decision_protocol::StateMachine::OPENING && + post == recovery_decision_protocol::StateMachine::OPEN) + { + record_trace_event( + tx, + { + .kind = "complete", + .pre = "OPEN", + .post = "OPEN", + }); + } + } + + void RecoveryDecisionProtocolSubsystem::record_trace_receive( + ccf::kv::Tx& tx, + const std::string& kind, + const std::optional& caused_by, + const std::string& source, + const std::optional& txid, + recovery_decision_protocol::StateMachine pre) + { + const auto post = get_trace_state(tx); + recovery_decision_protocol::TraceEvent event{ + .kind = kind, + .message_id = new_trace_message_id(), + .caused_by = caused_by, + .source = source, + .pre = trace_state_name(pre), + .post = trace_state_name(post), + }; + if (txid.has_value()) + { + event.view = txid->view; + event.seqno = txid->seqno; + } + record_trace_event(tx, std::move(event)); + record_trace_effects(tx, pre, post); + } + + void RecoveryDecisionProtocolSubsystem::record_trace_timeout( + ccf::kv::Tx& tx, recovery_decision_protocol::StateMachine pre) + { + const auto post = get_trace_state(tx); + record_trace_event( + tx, + { + .kind = "timeout", + .pre = trace_state_name(pre), + .post = trace_state_name(post), + }); + record_trace_effects(tx, pre, post); + } +#endif + void RecoveryDecisionProtocolSubsystem::reset_state(ccf::kv::Tx& tx) { // Clear any previous state @@ -49,6 +294,11 @@ namespace ccf tx.rw( Tables::RECOVERY_DECISION_PROTOCOL_OPEN_KIND) ->clear(); +#ifdef CCF_RECOVERY_TRACE + tx.rw( + Tables::RECOVERY_DECISION_PROTOCOL_TRACE_EVENTS) + ->clear(); +#endif } void RecoveryDecisionProtocolSubsystem::try_start( @@ -69,6 +319,10 @@ namespace ccf LOG_INFO_FMT("Starting recovery-decision-protocol"); +#ifdef CCF_RECOVERY_TRACE + initialise_trace(tx); +#endif + tx.rw( Tables::RECOVERY_DECISION_PROTOCOL_SM_STATE) ->put(recovery_decision_protocol::StateMachine::GOSSIPING); @@ -87,6 +341,13 @@ namespace ccf w.has_value() && w.value() == recovery_decision_protocol::StateMachine::GOSSIPING) { +#ifdef CCF_RECOVERY_TRACE + emit_trace_event({ + .kind = "start", + .pre = "GOSSIPING", + .post = "GOSSIPING", + }); +#endif start_message_retry_timers(); start_failover_timers(); } @@ -590,14 +851,21 @@ namespace ccf recovery_decision_protocol::GossipRequest request; request.info = get_node_info(tx); request.txid = get_last_recovered_signed_txid(); - nlohmann::json request_json = request; const auto self_signed_node_cert = node_state->get_self_signed_certificate(); const auto node_private_key = node_state->node_sign_kp->private_key_pem(); - for (auto& target : config.expected_locations) { auto target_address = target.address; +#ifdef CCF_RECOVERY_TRACE + request.trace_message_id = new_trace_message_id(); +#endif + nlohmann::json request_json = request; +#ifdef CCF_RECOVERY_TRACE + emit_trace_send( + request.trace_message_id.value(), + fmt::format("gossip:{}", target.name)); +#endif dispatch_authenticated_message( request_json, target_address, @@ -617,10 +885,18 @@ namespace ccf recovery_decision_protocol::TaggedWithNodeInfo request{ .info = get_node_info(tx)}; - nlohmann::json request_json = request; const auto self_signed_node_cert = node_state->get_self_signed_certificate(); +#ifdef CCF_RECOVERY_TRACE + request.trace_message_id = new_trace_message_id(); +#endif + nlohmann::json request_json = request; +#ifdef CCF_RECOVERY_TRACE + emit_trace_send( + request.trace_message_id.value(), + fmt::format("vote:{}", node_info.location.name)); +#endif dispatch_authenticated_message( request_json, node_info.location.address, @@ -674,11 +950,10 @@ namespace ccf LOG_TRACE_FMT("Sending recovery-decision-protocol iamopen"); - nlohmann::json request_json = get_iamopen_request(tx); + auto request = get_iamopen_request(tx); const auto self_signed_node_cert = node_state->get_self_signed_certificate(); const auto node_private_key = node_state->node_sign_kp->private_key_pem(); - for (auto& target : config.expected_locations) { if (target.name == location.name) @@ -686,6 +961,15 @@ namespace ccf // Don't send to self continue; } +#ifdef CCF_RECOVERY_TRACE + request.trace_message_id = new_trace_message_id(); +#endif + nlohmann::json request_json = request; +#ifdef CCF_RECOVERY_TRACE + emit_trace_send( + request.trace_message_id.value(), + fmt::format("iamopen:{}", target.name)); +#endif dispatch_authenticated_message( request_json, target.address, diff --git a/src/node/recovery_decision_protocol.h b/src/node/recovery_decision_protocol.h index 0a2b7d7896f7..f1b524f7cf85 100644 --- a/src/node/recovery_decision_protocol.h +++ b/src/node/recovery_decision_protocol.h @@ -16,9 +16,18 @@ namespace ccf::recovery_decision_protocol { public: RequestNodeInfo info; +#ifdef CCF_RECOVERY_TRACE + std::optional trace_message_id = std::nullopt; +#endif }; +#ifdef CCF_RECOVERY_TRACE + DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(TaggedWithNodeInfo); + DECLARE_JSON_REQUIRED_FIELDS(TaggedWithNodeInfo, info); + DECLARE_JSON_OPTIONAL_FIELDS(TaggedWithNodeInfo, trace_message_id); +#else DECLARE_JSON_TYPE(TaggedWithNodeInfo); DECLARE_JSON_REQUIRED_FIELDS(TaggedWithNodeInfo, info); +#endif struct GossipRequest : public TaggedWithNodeInfo { @@ -56,6 +65,17 @@ namespace ccf std::optional iamopen_request_cache; +#ifdef CCF_RECOVERY_TRACE + pal::Mutex trace_lock; + uint64_t next_trace_record_id = 0; + uint64_t next_trace_sequence = 0; + uint64_t next_trace_message_number = 0; + std::string trace_instance_id; + std::vector trace_expected_locations; + std::string trace_node; + std::string trace_committed_state; +#endif + public: RecoveryDecisionProtocolSubsystem(NodeState* node_state); void reset_state(ccf::kv::Tx& tx); @@ -65,6 +85,20 @@ namespace ccf recovery_decision_protocol::IAmOpenRequest& get_iamopen_request( kv::ReadOnlyTx& tx); +#ifdef CCF_RECOVERY_TRACE + recovery_decision_protocol::StateMachine get_trace_state( + kv::ReadOnlyTx& tx); + void record_trace_receive( + ccf::kv::Tx& tx, + const std::string& kind, + const std::optional& caused_by, + const std::string& source, + const std::optional& txid, + recovery_decision_protocol::StateMachine pre); + void record_trace_timeout( + ccf::kv::Tx& tx, recovery_decision_protocol::StateMachine pre); +#endif + private: // Start path void start_message_retry_timers(); @@ -85,5 +119,19 @@ namespace ccf RecoveryDecisionProtocolConfig& get_config(); sealing_recovery::Location& get_location(); ccf::TxID get_last_recovered_signed_txid(); + +#ifdef CCF_RECOVERY_TRACE + void initialise_trace(ccf::kv::Tx& tx); + void record_trace_event( + ccf::kv::Tx& tx, recovery_decision_protocol::TraceEvent event); + void record_trace_effects( + ccf::kv::Tx& tx, + recovery_decision_protocol::StateMachine pre, + recovery_decision_protocol::StateMachine post); + void emit_trace_event(recovery_decision_protocol::TraceEvent event); + std::string new_trace_message_id(); + void emit_trace_send( + const std::string& message_id, const std::string& description); +#endif }; } diff --git a/src/node/rpc/self_healing_open_handlers.h b/src/node/rpc/self_healing_open_handlers.h index e61dfc659369..a2057864694e 100644 --- a/src/node/rpc/self_healing_open_handlers.h +++ b/src/node/rpc/self_healing_open_handlers.h @@ -15,6 +15,8 @@ #include "node/recovery_decision_protocol.h" #include "node/rpc/node_frontend_utils.h" +#include + namespace ccf::node { template @@ -25,9 +27,10 @@ namespace ccf::node template static HandlerJsonParamsAndForward wrap_recovery_decision_protocol( RecoveryDecisionProtocolHandler cb, - ccf::AbstractNodeContext& node_context) + ccf::AbstractNodeContext& node_context, + const std::string& trace_kind) { - return [cb = std::move(cb), &node_context]( + return [cb = std::move(cb), &node_context, trace_kind]( endpoints::EndpointContext& args, const nlohmann::json& params) { auto config = node_context.get_subsystem(); auto node_operation = node_context.get_subsystem(); @@ -112,6 +115,22 @@ namespace ccf::node node_info_handle->put(info.location.name, src_info); } +#ifdef CCF_RECOVERY_TRACE + const auto trace_pre = args.tx + .ro( + Tables::RECOVERY_DECISION_PROTOCOL_SM_STATE) + ->get(); + if (!trace_pre.has_value()) + { + return make_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + "Recovery-decision-protocol state not set while tracing"); + } +#else + (void)trace_kind; +#endif + // ---- Run callback ---- auto ret = cb(args, in); @@ -125,7 +144,24 @@ namespace ccf::node try { - node_operation->recovery_decision_protocol().advance(args.tx, false); + auto& protocol = node_operation->recovery_decision_protocol(); + protocol.advance(args.tx, false); +#ifdef CCF_RECOVERY_TRACE + std::optional trace_txid = std::nullopt; + if constexpr (std::is_same_v< + Input, + recovery_decision_protocol::GossipRequest>) + { + trace_txid = in.txid; + } + protocol.record_trace_receive( + args.tx, + trace_kind, + in.trace_message_id, + info.location.name, + trace_txid, + trace_pre.value()); +#endif } catch (const std::logic_error& e) { @@ -186,7 +222,7 @@ namespace ccf::node HTTP_PUT, json_adapter(wrap_recovery_decision_protocol< recovery_decision_protocol::GossipRequest>( - recovery_decision_protocol_gossip, node_context)), + recovery_decision_protocol_gossip, node_context, "gossip_accepted")), no_auth_required) .set_forwarding_required(endpoints::ForwardingRequired::Never) .set_openapi_hidden(true) @@ -212,7 +248,7 @@ namespace ccf::node HTTP_PUT, json_adapter(wrap_recovery_decision_protocol< recovery_decision_protocol::TaggedWithNodeInfo>( - recovery_decision_protocol_vote, node_context)), + recovery_decision_protocol_vote, node_context, "vote_accepted")), no_auth_required) .set_forwarding_required(endpoints::ForwardingRequired::Never) .set_openapi_hidden(true) @@ -284,7 +320,9 @@ namespace ccf::node HTTP_PUT, json_adapter(wrap_recovery_decision_protocol< recovery_decision_protocol::IAmOpenRequest>( - recovery_decision_protocol_iamopen, node_context)), + recovery_decision_protocol_iamopen, + node_context, + "iamopen_accepted")), no_auth_required) .set_forwarding_required(endpoints::ForwardingRequired::Never) .set_openapi_hidden(true) @@ -343,7 +381,14 @@ namespace ccf::node try { - node_operation->recovery_decision_protocol().advance(args.tx, true); + auto& protocol = node_operation->recovery_decision_protocol(); +#ifdef CCF_RECOVERY_TRACE + const auto trace_pre = protocol.get_trace_state(args.tx); +#endif + protocol.advance(args.tx, true); +#ifdef CCF_RECOVERY_TRACE + protocol.record_trace_timeout(args.tx, trace_pre); +#endif } catch (const std::logic_error& e) { diff --git a/tests/e2e_operations.py b/tests/e2e_operations.py index b93834c91635..dcbcfdf75dcc 100644 --- a/tests/e2e_operations.py +++ b/tests/e2e_operations.py @@ -38,6 +38,7 @@ import infra.path import infra.platform_detection import infra.proc +import infra.recovery_trace import infra.utils import suite.test_requirements as reqs from ccf.tx_id import TxID @@ -2890,6 +2891,9 @@ def run_recovery_decision_protocol(const_args): assert ( recovery_type == '"Quorum"' ), f"Network self-healing open type was {recovery_type} instead of Quorum" + infra.recovery_trace.validate_recovery_trace_if_enabled( + recovered_network, args.label, "QUORUM" + ) def run_recovery_decision_protocol_timeout_path(const_args): @@ -2942,6 +2946,9 @@ def run_recovery_decision_protocol_timeout_path(const_args): assert ( recovery_type == '"Failover"' ), f"Network self-healing open type was {recovery_type} instead of Failover" + infra.recovery_trace.validate_recovery_trace_if_enabled( + recovered_network, args.label, "FAILOVER" + ) def run_recovery_decision_protocol_multiple_timeout(const_args): @@ -2994,6 +3001,9 @@ def run_recovery_decision_protocol_multiple_timeout(const_args): node.refresh_network_state(verify_ca=False) assert len(recovered_network.get_joined_nodes()) == len(args.nodes) + infra.recovery_trace.validate_recovery_trace_if_enabled( + recovered_network, args.label, "FAILOVER" + ) def run_read_ledger_on_testdata(args): diff --git a/tests/infra/recovery_trace.py b/tests/infra/recovery_trace.py new file mode 100644 index 000000000000..9bda16864b0c --- /dev/null +++ b/tests/infra/recovery_trace.py @@ -0,0 +1,234 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +import heapq +import itertools +import json +import logging +import os +import pathlib +import subprocess +import time + +TRACE_MARKER = "RDP_TRACE " +TRACE_VALIDATOR_ENV = "CCF_LEAN_TRACE_VALIDATOR" +LOG = logging.getLogger(__name__) + + +def _event_from_log_line(line, path, line_number): + message = line + try: + outer = json.loads(line) + if isinstance(outer, dict) and isinstance(outer.get("msg"), str): + message = outer["msg"] + except json.JSONDecodeError: + pass + + marker = message.find(TRACE_MARKER) + if marker < 0: + return None + + payload = message[marker + len(TRACE_MARKER) :].lstrip() + try: + event, _ = json.JSONDecoder().raw_decode(payload) + except json.JSONDecodeError as error: + raise ValueError( + f"{path}:{line_number}: invalid recovery trace JSON: {error}" + ) from error + if not isinstance(event, dict): + raise TypeError(f"{path}:{line_number}: recovery trace is not an object") + return event + + +def extract_events(nodes): + events = [] + for node in nodes: + out_path, _ = node.get_logs() + if out_path is None or not os.path.isfile(out_path): + continue + with open(out_path, encoding="utf-8", errors="replace") as log: + for line_number, line in enumerate(log, 1): + event = _event_from_log_line(line, out_path, line_number) + if event is not None: + events.append(event) + if not events: + raise ValueError("no recovery-decision-protocol trace events found") + return events + + +def linearize(events): + successors = [set() for _ in events] + indegree = [0 for _ in events] + + def add_edge(source, destination): + if destination not in successors[source]: + successors[source].add(destination) + indegree[destination] += 1 + + by_node = {} + message_ids = {} + for index, event in enumerate(events): + try: + node = event["node"] + sequence = event["sequence"] + kind = event["kind"] + except KeyError as error: + raise ValueError( + f"trace event {index} is missing {error.args[0]}" + ) from error + if not isinstance(node, str) or not isinstance(sequence, int): + raise TypeError(f"trace event {index} has an invalid node or sequence") + by_node.setdefault(node, []).append((sequence, index)) + + message_id = event.get("message_id") + if message_id is not None: + if message_id in message_ids: + raise ValueError(f"duplicate trace message_id {message_id}") + message_ids[message_id] = (index, kind) + + for node, node_events in by_node.items(): + node_events.sort() + sequences = [sequence for sequence, _ in node_events] + if sequences != list(range(len(node_events))): + raise ValueError( + f"node {node} trace sequence is not contiguous from zero: {sequences}" + ) + for (_, previous), (_, current) in itertools.pairwise(node_events): + add_edge(previous, current) + + for index, event in enumerate(events): + caused_by = event.get("caused_by") + if caused_by is None: + continue + if caused_by not in message_ids: + raise ValueError(f"caused_by {caused_by} has no matching send event") + source, kind = message_ids[caused_by] + if kind != "send": + raise ValueError(f"caused_by {caused_by} does not identify a send event") + add_edge(source, index) + + ready = [] + for index, degree in enumerate(indegree): + if degree == 0: + event = events[index] + heapq.heappush( + ready, (event["node"], event["sequence"], event["kind"], index) + ) + + ordered = [] + while ready: + _, _, _, index = heapq.heappop(ready) + ordered.append(events[index]) + for successor in successors[index]: + indegree[successor] -= 1 + if indegree[successor] == 0: + event = events[successor] + heapq.heappush( + ready, + (event["node"], event["sequence"], event["kind"], successor), + ) + + if len(ordered) != len(events): + raise ValueError("recovery trace contains a causal cycle") + return ordered + + +def _validator_path(): + configured = os.getenv(TRACE_VALIDATOR_ENV) + if configured: + return pathlib.Path(configured) + repository = pathlib.Path(__file__).resolve().parents[2] + return ( + repository + / "lean" + / "disaster-recovery" + / ".lake" + / "build" + / "bin" + / "trace-validator" + ) + + +def _participating_nodes(nodes): + return { + node.get_sealing_recovery_location()["name"] + for node in nodes + if node.remote is not None + } + + +def wait_for_terminal_events(network, expected_open_kind, timeout): + expected_nodes = _participating_nodes(network.nodes) + end_time = time.time() + timeout + events = [] + while time.time() < end_time: + try: + events = extract_events(network.nodes) + except ValueError: + time.sleep(0.1) + continue + + started = {event["node"] for event in events if event["kind"] == "start"} + completed = {event["node"] for event in events if event["kind"] == "complete"} + terminal = completed | { + event["node"] for event in events if event["kind"] == "join_restart" + } + opened = [event for event in events if event["kind"] == "open"] + if ( + expected_nodes <= started + and expected_nodes <= terminal + and completed + and opened + and all(event.get("open_kind") == expected_open_kind for event in opened) + ): + return events + time.sleep(0.1) + + raise TimeoutError( + "timed out waiting for terminal recovery trace events: " + f"expected_nodes={sorted(expected_nodes)}, " + f"expected_open_kind={expected_open_kind}, events={events}" + ) + + +def validate_recovery_trace(network, label, expected_open_kind=None, timeout=20): + if expected_open_kind is None: + events = extract_events(network.nodes) + else: + events = wait_for_terminal_events(network, expected_open_kind, timeout) + events = linearize(events) + trace_path = pathlib.Path(network.common_dir) / f"{label}.recovery.ndjson" + with open(trace_path, "w", encoding="utf-8") as trace: + for event in events: + trace.write(json.dumps(event, separators=(",", ":"), sort_keys=True)) + trace.write("\n") + + validator = _validator_path() + if not validator.is_file(): + raise FileNotFoundError( + f"Lean trace validator not found at {validator}; set {TRACE_VALIDATOR_ENV}" + ) + result = subprocess.run( + [validator, trace_path], + text=True, + capture_output=True, + check=False, + ) + if result.returncode != 0: + raise AssertionError( + f"Lean recovery trace validation failed for {trace_path}:\n" + f"{result.stdout}{result.stderr}" + ) + LOG.info(result.stdout.strip()) + return trace_path + + +def validate_recovery_trace_if_enabled(network, label, expected_open_kind, timeout=20): + if not os.getenv(TRACE_VALIDATOR_ENV): + return None + return validate_recovery_trace( + network, + label, + expected_open_kind=expected_open_kind, + timeout=timeout, + ) diff --git a/tests/infra/recovery_trace_test.py b/tests/infra/recovery_trace_test.py new file mode 100644 index 000000000000..5069a8b30171 --- /dev/null +++ b/tests/infra/recovery_trace_test.py @@ -0,0 +1,172 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +import json +import pathlib +import tempfile +import unittest +from unittest import mock + +import infra.recovery_trace + +VERSION = "ccf.recovery_decision_protocol.trace/1" +EXPECTED_LOCATIONS = ["A", "B"] + + +def event(node, sequence, kind, **extra): + value = { + "version": VERSION, + "instance": "synthetic", + "expected_locations": EXPECTED_LOCATIONS, + "node": node, + "sequence": sequence, + "kind": kind, + "pre": "GOSSIPING", + "post": "GOSSIPING", + } + value.update(extra) + return value + + +class FakeNode: + def __init__(self, path, name=None): + self.path = path + self.name = name + self.remote = object() if name is not None else None + + def get_logs(self): + return str(self.path), None + + def get_sealing_recovery_location(self): + return {"name": self.name} + + +class FakeNetwork: + def __init__(self, nodes, common_dir): + self.nodes = nodes + self.common_dir = common_dir + + +class RecoveryTraceTest(unittest.TestCase): + def test_extract_linearize_and_validate(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + a_log = root / "a.out" + b_log = root / "b.out" + a_events = [ + event("A", 0, "start"), + event( + "A", + 1, + "send", + message_id="send-a-b", + send="gossip:B", + ), + ] + b_events = [ + event("B", 0, "start"), + event( + "B", + 1, + "gossip_accepted", + message_id="receive-a-b", + caused_by="send-a-b", + source="A", + view=1, + seqno=1, + ), + ] + a_log.write_text( + "".join( + f"[info] RDP_TRACE {json.dumps(trace_event)}\n" + for trace_event in a_events + ), + encoding="utf-8", + ) + b_log.write_text( + "".join( + json.dumps({"msg": f"RDP_TRACE {json.dumps(trace_event)}"}) + "\n" + for trace_event in b_events + ), + encoding="utf-8", + ) + network = FakeNetwork( + [FakeNode(b_log), FakeNode(a_log)], + directory, + ) + + extracted = infra.recovery_trace.extract_events(network.nodes) + ordered = infra.recovery_trace.linearize(extracted) + self.assertEqual( + [(item["node"], item["sequence"]) for item in ordered], + [("A", 0), ("A", 1), ("B", 0), ("B", 1)], + ) + + trace_path = infra.recovery_trace.validate_recovery_trace( + network, "synthetic" + ) + self.assertTrue(trace_path.is_file()) + + def test_rejects_non_contiguous_sequence(self): + broken = [ + event("A", 0, "start"), + event("A", 2, "timeout"), + ] + with self.assertRaisesRegex(ValueError, "not contiguous"): + infra.recovery_trace.linearize(broken) + + def test_rejects_unresolved_cause(self): + broken = [ + event("A", 0, "start"), + event( + "A", + 1, + "gossip_accepted", + message_id="receive", + caused_by="missing-send", + source="B", + view=1, + seqno=1, + ), + ] + with self.assertRaisesRegex(ValueError, "no matching send"): + infra.recovery_trace.linearize(broken) + + def test_disabled_validation_preserves_default_tests(self): + with mock.patch.dict( + "os.environ", + {infra.recovery_trace.TRACE_VALIDATOR_ENV: ""}, + clear=False, + ): + self.assertIsNone( + infra.recovery_trace.validate_recovery_trace_if_enabled( + FakeNetwork([], "."), "disabled", "QUORUM" + ) + ) + + def test_waits_for_terminal_scenario_evidence(self): + with tempfile.TemporaryDirectory() as directory: + log_path = pathlib.Path(directory) / "a.out" + events = [ + event("A", 0, "start"), + event("A", 1, "open", open_kind="QUORUM"), + event("A", 2, "complete"), + ] + log_path.write_text( + "".join( + f"RDP_TRACE {json.dumps(trace_event)}\n" for trace_event in events + ), + encoding="utf-8", + ) + network = FakeNetwork( + [FakeNode(log_path, "A")], + directory, + ) + self.assertEqual( + infra.recovery_trace.wait_for_terminal_events(network, "QUORUM", 0.1), + events, + ) + + +if __name__ == "__main__": + unittest.main() From 935650e1cee52d247586b66456ddc6089c1f91b0 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Sun, 30 Aug 2026 14:24:59 +0100 Subject: [PATCH 04/14] Emit join traces before restart Defer trace-enabled joiner restart until the committed receive and join effect have been emitted, while preserving the default immediate path. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 46d56f8a-935f-4551-abe3-84bd4f951865 --- lean/disaster-recovery/README.md | 2 ++ lean/disaster-recovery/TRACE_FORMAT_V1.md | 3 +++ src/node/recovery_decision_protocol.cpp | 7 +++++++ 3 files changed, 12 insertions(+) diff --git a/lean/disaster-recovery/README.md b/lean/disaster-recovery/README.md index fa446d02e090..c9e2fc31b60a 100644 --- a/lean/disaster-recovery/README.md +++ b/lean/disaster-recovery/README.md @@ -336,6 +336,8 @@ values are checked against every remaining candidate state. Protocol transactions append semantic events to an internal public trace map; its global commit hook emits `RDP_TRACE` records only after commit. Sends are logged before dispatch with causal IDs propagated to accepted receive records. +For trace-enabled joiners, the hook emits the committed receive and +`join_restart` records before requesting host restart. [`tests/infra/recovery_trace.py`](../../tests/infra/recovery_trace.py) extracts records from all recovery nodes and topologically orders them from per-node diff --git a/lean/disaster-recovery/TRACE_FORMAT_V1.md b/lean/disaster-recovery/TRACE_FORMAT_V1.md index 3b17976bb921..555b72d6d141 100644 --- a/lean/disaster-recovery/TRACE_FORMAT_V1.md +++ b/lean/disaster-recovery/TRACE_FORMAT_V1.md @@ -126,6 +126,9 @@ Accepted receive and timeout events are written to transaction as the modeled state change. A global commit hook emits them only after commit, followed by any `open`, `join_restart`, or `complete` effect from that transition. Aborted transactions therefore emit nothing. +In trace-enabled builds the joiner restart request is issued by this hook after +the committed receive and `join_restart` records are emitted; default builds +retain the existing immediate restart path. The committed start hook emits `start` before scheduling retry and failover tasks. Transport sends are emitted immediately before dispatch and propagate diff --git a/src/node/recovery_decision_protocol.cpp b/src/node/recovery_decision_protocol.cpp index 974953380421..5c6a4e9b3901 100644 --- a/src/node/recovery_decision_protocol.cpp +++ b/src/node/recovery_decision_protocol.cpp @@ -104,6 +104,11 @@ namespace ccf if (event.has_value()) { emit_trace_event(event.value()); + if (event->kind == "join_restart") + { + RINGBUFFER_WRITE_MESSAGE( + AdminMessage::restart, node_state->to_host); + } } } })); @@ -507,7 +512,9 @@ namespace ccf ccf::crypto::cert_der_to_pem(node_config->service_cert_der); LOG_INFO_FMT("{}", service_cert.str()); +#ifndef CCF_RECOVERY_TRACE RINGBUFFER_WRITE_MESSAGE(AdminMessage::restart, node_state->to_host); +#endif } case recovery_decision_protocol::StateMachine::OPENING: { From c8dd40a7e41be5d8bccaee25e97627b6caca3756 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Sun, 30 Aug 2026 15:01:22 +0100 Subject: [PATCH 05/14] Order traced retries after global commits Defer trace-enabled retry work until its local protocol phase is globally visible, preventing sends from preceding their committed transition event. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 46d56f8a-935f-4551-abe3-84bd4f951865 --- lean/disaster-recovery/README.md | 2 ++ lean/disaster-recovery/TRACE_FORMAT_V1.md | 3 +++ src/node/recovery_decision_protocol.cpp | 14 ++++++++++++++ src/node/recovery_decision_protocol.h | 2 ++ tests/infra/recovery_trace.py | 17 +++++++---------- 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/lean/disaster-recovery/README.md b/lean/disaster-recovery/README.md index c9e2fc31b60a..05cdd934a12a 100644 --- a/lean/disaster-recovery/README.md +++ b/lean/disaster-recovery/README.md @@ -338,6 +338,8 @@ its global commit hook emits `RDP_TRACE` records only after commit. Sends are logged before dispatch with causal IDs propagated to accepted receive records. For trace-enabled joiners, the hook emits the committed receive and `join_restart` records before requesting host restart. +Trace-enabled periodic retries defer work while their locally committed phase +is ahead of the globally visible trace phase, then send on the next invocation. [`tests/infra/recovery_trace.py`](../../tests/infra/recovery_trace.py) extracts records from all recovery nodes and topologically orders them from per-node diff --git a/lean/disaster-recovery/TRACE_FORMAT_V1.md b/lean/disaster-recovery/TRACE_FORMAT_V1.md index 555b72d6d141..61fcf07ef86e 100644 --- a/lean/disaster-recovery/TRACE_FORMAT_V1.md +++ b/lean/disaster-recovery/TRACE_FORMAT_V1.md @@ -134,6 +134,9 @@ The committed start hook emits `start` before scheduling retry and failover tasks. Transport sends are emitted immediately before dispatch and propagate their generated `message_id` in the internal request as `trace_message_id`; the committed receive records it as `caused_by`. +If a retry observes a locally committed phase that is not yet globally visible +to the trace hook, tracing defers that retry invocation. The periodic task sends +on its next run after the phase event is emitted. Each log record contains `RDP_TRACE ` followed by the event object. `tests/infra/recovery_trace.py` extracts records from all participating node diff --git a/src/node/recovery_decision_protocol.cpp b/src/node/recovery_decision_protocol.cpp index 5c6a4e9b3901..a4b87b0ee829 100644 --- a/src/node/recovery_decision_protocol.cpp +++ b/src/node/recovery_decision_protocol.cpp @@ -156,6 +156,13 @@ namespace ccf "{}:{}:{}", trace_instance_id, trace_node, next_trace_message_number++); } + bool RecoveryDecisionProtocolSubsystem::is_trace_state_committed( + recovery_decision_protocol::StateMachine state) + { + std::lock_guard guard(trace_lock); + return trace_committed_state == trace_state_name(state); + } + recovery_decision_protocol::StateMachine RecoveryDecisionProtocolSubsystem:: get_trace_state(kv::ReadOnlyTx& tx) { @@ -592,6 +599,13 @@ namespace ccf } auto& sm_state = sm_state_opt.value(); +#ifdef CCF_RECOVERY_TRACE + if (!is_trace_state_committed(sm_state)) + { + return; + } +#endif + // Stop if recovery-decision-protocol is complete if (sm_state == recovery_decision_protocol::StateMachine::OPEN) { diff --git a/src/node/recovery_decision_protocol.h b/src/node/recovery_decision_protocol.h index f1b524f7cf85..a0ec1d49d0cc 100644 --- a/src/node/recovery_decision_protocol.h +++ b/src/node/recovery_decision_protocol.h @@ -130,6 +130,8 @@ namespace ccf recovery_decision_protocol::StateMachine post); void emit_trace_event(recovery_decision_protocol::TraceEvent event); std::string new_trace_message_id(); + bool is_trace_state_committed( + recovery_decision_protocol::StateMachine state); void emit_trace_send( const std::string& message_id, const std::string& description); #endif diff --git a/tests/infra/recovery_trace.py b/tests/infra/recovery_trace.py index 9bda16864b0c..531548332723 100644 --- a/tests/infra/recovery_trace.py +++ b/tests/infra/recovery_trace.py @@ -149,16 +149,12 @@ def _validator_path(): ) -def _participating_nodes(nodes): - return { - node.get_sealing_recovery_location()["name"] - for node in nodes - if node.remote is not None - } +def _participating_node_count(nodes): + return sum(node.remote is not None for node in nodes) def wait_for_terminal_events(network, expected_open_kind, timeout): - expected_nodes = _participating_nodes(network.nodes) + expected_node_count = _participating_node_count(network.nodes) end_time = time.time() + timeout events = [] while time.time() < end_time: @@ -175,8 +171,8 @@ def wait_for_terminal_events(network, expected_open_kind, timeout): } opened = [event for event in events if event["kind"] == "open"] if ( - expected_nodes <= started - and expected_nodes <= terminal + len(started) == expected_node_count + and started <= terminal and completed and opened and all(event.get("open_kind") == expected_open_kind for event in opened) @@ -186,7 +182,8 @@ def wait_for_terminal_events(network, expected_open_kind, timeout): raise TimeoutError( "timed out waiting for terminal recovery trace events: " - f"expected_nodes={sorted(expected_nodes)}, " + f"expected_node_count={expected_node_count}, " + f"started={sorted(started) if events else []}, " f"expected_open_kind={expected_open_kind}, events={events}" ) From aa46b8227dd400ef5eb1f6914eb67add204ab241 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Sun, 30 Aug 2026 15:51:12 +0100 Subject: [PATCH 06/14] Document Lean proof coverage Summarize kernel-checked safety, progress, and refinement results, distinguish bounded executable properties, and link successful Lean and SNP validation runs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 46d56f8a-935f-4551-abe3-84bd4f951865 --- lean/disaster-recovery/README.md | 68 ++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/lean/disaster-recovery/README.md b/lean/disaster-recovery/README.md index 05cdd934a12a..573ffa573bb0 100644 --- a/lean/disaster-recovery/README.md +++ b/lean/disaster-recovery/README.md @@ -117,6 +117,60 @@ Opening eventually reaches Open when timeout firing is weakly fair. These theorems relate the canonical and legacy Lean abstractions. They do not claim that the canonical C++-aligned model is identical to the Rust model. +## Property coverage + +### Kernel-checked canonical properties + +| Property | Lean theorem(s) | +| ------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------- | +| A valid timeout is aligned with the protocol timeout lane | `valid_timeout_requires_alignment` | +| Once a node chooses a recovery source, later gossip is rejected without changing state | `gossip_freezes_after_choice` | +| Gossip rejected by validation does not change protocol state | `rejected_gossip_stutters` | +| Receiving the same vote again is idempotent | `duplicate_vote_is_idempotent` | +| Opening and Open nodes reject `IAmOpen` without changing state | `opening_rejects_iamopen`, `open_rejects_iamopen` | +| An aligned Voting timeout with no votes cannot open | `aligned_voting_timeout_without_votes_stutters` | +| An aligned Gossiping timeout with no gossip aborts without changing state | `aligned_empty_gossip_timeout_aborts` | +| A quorum in Voting transitions to Opening with `QUORUM` and the opening effect | `quorum_advance_opens` | +| An aligned Opening timeout transitions to Open and emits completion | `aligned_opening_timeout_completes`, `aligned_timeout_transitions_to_open` | +| Every non-timeout event preserves aligned Opening | `non_timeout_step_preserves_aligned_opening` | +| Weak timeout fairness makes an execution starting in aligned Opening eventually reach Open | `fair_aligned_opening_progress` | + +These are theorem-checked for arbitrary configurations and states satisfying +their explicit premises. The progress theorem assumes weak fairness; the safety +theorems do not. + +### Kernel-checked refinement properties + +| Property | Lean theorem(s) | +| ---------------------------------------------------------------- | ------------------------------------------------------------------- | +| Every canonical step projects to zero or more legacy phase steps | `canonical_step_simulates`, `compatibility_step_simulates` | +| Finite canonical traces project to legacy weak phase traces | `compatibility_trace_simulates` | +| Canonical and legacy initial phases correspond | `initial_phase_correspondence`, `three_node_initial_correspondence` | +| Canonical and legacy quorum thresholds agree for odd node counts | `odd_quorum_matches_legacy` | +| Canonical quorum is one larger for even node counts | `even_quorum_exceeds_legacy_by_one` | +| Canonical Opening/Open is exactly projected legacy Open | `reached_open_is_preserved` | +| Canonical quorum opening projects to non-timeout legacy Open | `quorum_kind_projects_to_non_timeout_open` | +| Splitting canonical Opening and Open is a legacy stutter | `opening_to_open_is_stuttering` | +| Full single-node initial states intentionally differ | `single_node_full_initial_models_differ` | + +The refinement is phase-level. Gossip sets, votes, timeout-lane state, node +metadata, and retry effects are not claimed to be data-bisimilar. + +### Executable bounded properties + +The legacy BFS checker retains all nine Stateright expectations: + +- eventual Open, plus the unanimous-vote and majority-vote non-timeout + implications; +- always no pre-failover fork, no all-OpenJoin/no-vote deadlock, and persistence + of the legacy committed transaction threshold; and +- reachability of Open, timeout Open, and majority non-timeout Open. + +For the canonical model, exhaustive one- and two-node checks additionally +assert that Voting has a chosen node, Opening/Open has an open kind, and a +restart request occurs only in Joining. These are executable finite-state +checks, not general Lean theorems. + ## Legacy source mapping | Rust/Stateright source | Lean definition | @@ -401,6 +455,20 @@ Rust/Lean comparison on relevant pull requests. The weekly continuous verification workflow additionally runs the exhaustive three-node comparison. The Rust job remains in place until the replacement criteria below are met. +### Current CI evidence + +Draft PR [microsoft/CCF#8241](https://github.com/microsoft/CCF/pull/8241) +validated commit `c8dd40a7` on both proof and hardware paths: + +- [Lean shallow verification](https://github.com/microsoft/CCF/actions/runs/33315731130/job/99268715043) + built every Lean target, checked trace fixtures and merger behavior, and + compared the bounded Rust/Lean graphs. +- [Milan SNP](https://github.com/microsoft/CCF/actions/runs/33315731084/job/99268714883) + validated committed quorum, failover, and multiple-timeout implementation + traces. +- [Genoa SNP](https://github.com/microsoft/CCF/actions/runs/33315731084/job/99268714839) + validated the same implementation traces on the second SNP generation. + ## Replacement criteria The Stateright model can be removed only after: From 865d08e43f454dc7a7a0cdab1547ada926592ac8 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Sun, 30 Aug 2026 19:47:09 +0100 Subject: [PATCH 07/14] Simplify recovery trace replay Replace generalized under-observation search with deterministic replay of complete instrumented traces, enforce exact retry send batches and terminal effects, and split format parsing from replay. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 46d56f8a-935f-4551-abe3-84bd4f951865 --- .../DisasterRecovery/Protocol/Trace.lean | 793 +----------------- .../Protocol/Trace/Format.lean | 141 ++++ .../Protocol/Trace/Replay.lean | 445 ++++++++++ lean/disaster-recovery/README.md | 75 +- lean/disaster-recovery/TRACE_FORMAT_V1.md | 117 +-- lean/disaster-recovery/TraceMain.lean | 4 +- lean/disaster-recovery/TraceTests.lean | 497 ++++------- .../fixtures/accepted-failover.ndjson | 21 +- .../fixtures/accepted-multinode.ndjson | 27 +- .../fixtures/accepted.ndjson | 17 +- .../fixtures/rejected-cause.ndjson | 5 +- src/node/recovery_decision_protocol.cpp | 137 +-- src/node/recovery_decision_protocol.h | 22 +- src/node/rpc/self_healing_open_handlers.h | 10 + tests/infra/recovery_trace_test.py | 46 +- 15 files changed, 995 insertions(+), 1362 deletions(-) create mode 100644 lean/disaster-recovery/DisasterRecovery/Protocol/Trace/Format.lean create mode 100644 lean/disaster-recovery/DisasterRecovery/Protocol/Trace/Replay.lean diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/Trace.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/Trace.lean index 5611fabf9aef..a5e7fc60bb9b 100644 --- a/lean/disaster-recovery/DisasterRecovery/Protocol/Trace.lean +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/Trace.lean @@ -1,791 +1,2 @@ -import DisasterRecovery.Protocol.Model -import Lean.Data.Json -import Lean.Data.Json.FromToJson - -namespace DisasterRecovery.Protocol.Trace - -open Lean - -def contractVersion : String := - "ccf.recovery_decision_protocol.trace/1" - -inductive Kind where - | start - | gossipAccepted - | gossipRejected - | voteAccepted - | voteRejected - | iAmOpenAccepted - | iAmOpenRejected - | timeout - | retry - | send - | open - | joinRestart - | complete -deriving Repr, BEq, Inhabited - -structure TraceEvent where - version : String - instanceId : String - expectedLocations : List Location - node : Location - sequence : Nat - kind : Kind - messageId : Option String - causedBy : Option String - source : Option Location - txid : Option TxID - pre : Option Phase - post : Option Phase - openKind : Option OpenKind - send : Option String -deriving Repr, BEq, Inhabited - -structure Failure where - prefixLength : Nat - message : String - expected : List String -deriving Repr, BEq - -structure ObservedSend where - messageId : String - source : Location - description : String -deriving Repr, BEq - -structure HiddenSend where - source : Location - description : String -deriving Repr, BEq - -structure PendingEffect where - node : Location - effect : Effect -deriving Repr, BEq - -structure Candidate where - system : SystemState - hiddenSends : List HiddenSend := [] - pendingEffects : List PendingEffect := [] -deriving Repr, BEq - -private def parseKind : String -> Except String Kind - | "start" => pure .start - | "gossip_accepted" => pure .gossipAccepted - | "gossip_rejected" => pure .gossipRejected - | "vote_accepted" => pure .voteAccepted - | "vote_rejected" => pure .voteRejected - | "iamopen_accepted" => pure .iAmOpenAccepted - | "iamopen_rejected" => pure .iAmOpenRejected - | "timeout" => pure .timeout - | "retry" => pure .retry - | "send" => pure .send - | "open" => pure .open - | "join_restart" => pure .joinRestart - | "complete" => pure .complete - | value => throw s!"unknown kind '{value}'" - -private def parsePhase : String -> Except String Phase - | "GOSSIPING" => pure .gossiping - | "VOTING" => pure .voting - | "OPENING" => pure .opening - | "JOINING" => pure .joining - | "OPEN" => pure .open - | value => throw s!"unknown phase '{value}'" - -private def parseOpenKind : String -> Except String OpenKind - | "QUORUM" => pure .quorum - | "FAILOVER" => pure .failover - | value => throw s!"unknown open kind '{value}'" - -private def optionalString (json : Json) (key : String) : - Except String (Option String) := - match json.getObjVal? key with - | .error _ => pure none - | .ok .null => pure none - | .ok value => do - let parsed <- value.getStr? - pure (some parsed) - -private def optionalNat (json : Json) (key : String) : - Except String (Option Nat) := - match json.getObjVal? key with - | .error _ => pure none - | .ok .null => pure none - | .ok value => do - let parsed <- value.getNat? - pure (some parsed) - -private def optionalPhase (json : Json) (key : String) : - Except String (Option Phase) := do - let value <- optionalString json key - match value with - | none => pure none - | some name => do - let phase <- parsePhase name - pure (some phase) - -private def optionalOpenKind (json : Json) (key : String) : - Except String (Option OpenKind) := do - let value <- optionalString json key - match value with - | none => pure none - | some name => do - let kind <- parseOpenKind name - pure (some kind) - -def parseEvent (line : String) : Except String TraceEvent := do - let json <- Json.parse line - let version <- json.getObjValAs? String "version" - let instanceId <- json.getObjValAs? String "instance" - let expectedLocations <- json.getObjValAs? (List String) "expected_locations" - let node <- json.getObjValAs? String "node" - let sequence <- json.getObjValAs? Nat "sequence" - let kindName <- json.getObjValAs? String "kind" - let kind <- parseKind kindName - let messageId <- optionalString json "message_id" - let causedBy <- optionalString json "caused_by" - let source <- optionalString json "source" - let view <- optionalNat json "view" - let seqno <- optionalNat json "seqno" - let txid := - match view, seqno with - | some view, some seqno => some { view, seqno } - | none, none => none - | _, _ => none - let pre <- optionalPhase json "pre" - let post <- optionalPhase json "post" - let openKind <- optionalOpenKind json "open_kind" - let send <- optionalString json "send" - if version != contractVersion then - throw s!"unsupported version '{version}'" - if (view.isSome != seqno.isSome) then - throw "view and seqno must appear together" - pure { - version - instanceId - expectedLocations - node - sequence - kind - messageId - causedBy - source - txid - pre - post - openKind - send - } - -def parseNDJSON (input : String) : Except String (List TraceEvent) := do - let lines := (input.splitOn "\n").filter - (fun line => !line.trimAscii.isEmpty) - let mut events := [] - for (line, index) in lines.zipIdx do - match parseEvent line with - | .ok event => events := event :: events - | .error message => throw s!"line {index + 1}: {message}" - pure events.reverse - -private def nodeState (system : SystemState) (node : Location) : Option NodeState := - (system.nodes.find? fun entry => entry.1 == node).map Prod.snd - -private def phaseMatches (expected : Option Phase) (actual : Phase) : Bool := - match expected with - | none => true - | some phase => phase == actual - -private def requiresMessageId : Kind -> Bool - | .gossipAccepted | .gossipRejected - | .voteAccepted | .voteRejected - | .iAmOpenAccepted | .iAmOpenRejected - | .send => true - | _ => false - -private def requiresSource : Kind -> Bool - | .gossipAccepted | .gossipRejected - | .voteAccepted | .voteRejected - | .iAmOpenAccepted | .iAmOpenRejected => true - | _ => false - -private def missingRequiredFields (event : TraceEvent) : List String := - let phases := - (if event.pre.isNone then ["pre"] else []) ++ - (if event.post.isNone then ["post"] else []) - let message := - if requiresMessageId event.kind && event.messageId.isNone then - ["message_id"] - else - [] - let source := - if requiresSource event.kind && event.source.isNone then ["source"] else [] - let txid := - match event.kind with - | .gossipAccepted | .gossipRejected => - if event.txid.isNone then ["view", "seqno"] else [] - | _ => [] - let send := - match event.kind with - | .send => if event.send.isNone then ["send"] else [] - | _ => [] - let openKind := - match event.kind with - | .open => if event.openKind.isNone then ["open_kind"] else [] - | _ => [] - phases ++ message ++ source ++ txid ++ send ++ openKind - -private def configError (config : Config) : Option String := - if config.instanceId.isEmpty then - some "instance must not be empty" - else if config.expectedLocations.isEmpty then - some "expected_locations must not be empty" - else if config.expectedLocations.any String.isEmpty then - some "expected_locations must not contain an empty name" - else if config.expectedLocations.eraseDups.length != - config.expectedLocations.length then - some "expected_locations must not contain duplicates" - else - none - -private def eventInputs (event : TraceEvent) : List Event := - let validations : List Validation := - match event.kind with - | .gossipRejected | .voteRejected | .iAmOpenRejected => - [Validation.rejected, Validation.accepted] - | _ => [Validation.accepted] - match event.kind, event.source, event.txid with - | .gossipAccepted, some source, some txid => - [.receiveGossip source txid .accepted] - | .gossipRejected, some source, some txid => - validations.map fun validation => .receiveGossip source txid validation - | .voteAccepted, some source, _ => - [.receiveVote source .accepted] - | .voteRejected, some source, _ => - validations.map fun validation => .receiveVote source validation - | .iAmOpenAccepted, some source, _ => - [.receiveIAmOpen source .accepted] - | .iAmOpenRejected, some source, _ => - validations.map fun validation => .receiveIAmOpen source validation - | .timeout, _, _ => [.timeout] - | .retry, _, _ => [.retry] - | _, _, _ => [] - -private def expectsAcceptance : Kind -> Option Bool - | .gossipAccepted | .voteAccepted | .iAmOpenAccepted => some true - | .gossipRejected | .voteRejected | .iAmOpenRejected => some false - | .timeout | .retry => some true - | _ => none - -private def effectName : Effect -> Option String - | .sendGossip destination => some s!"gossip:{destination}" - | .sendVote destination => some s!"vote:{destination}" - | .sendIAmOpen destination => some s!"iamopen:{destination}" - | _ => none - -private def hiddenSendBefore (left right : HiddenSend) : Bool := - left.source < right.source || - (left.source == right.source && left.description <= right.description) - -private def uniqueHiddenSends (sends : List HiddenSend) : List HiddenSend := - (sends.foldl (fun result send => - if result.contains send then result else send :: result) []).mergeSort - hiddenSendBefore - -private def uniqueCandidates (candidates : List Candidate) : List Candidate := - candidates.foldl (fun result candidate => - if result.contains candidate then result else candidate :: result) [] - -private def hiddenSendsAt - (config : Config) - (startedNodes : List Location) - (system : SystemState) : List HiddenSend := - startedNodes.flatMap fun source => - match nodeState system source with - | none => [] - | some sender => - (step config sender .retry).effects.filterMap fun effect => do - let description <- effectName effect - pure { source, description } - -def hiddenClosure - (config : Config) - (startedNodes : List Location) - (candidates : List Candidate) : List Candidate := - uniqueCandidates (candidates.map fun candidate => { - candidate with - hiddenSends := uniqueHiddenSends - (candidate.hiddenSends ++ - hiddenSendsAt config startedNodes candidate.system) - }) - -private def expectedReceiveSend (event : TraceEvent) : Option String := - match event.kind with - | .gossipAccepted | .gossipRejected => - some s!"gossip:{event.node}" - | .voteAccepted | .voteRejected => - some s!"vote:{event.node}" - | .iAmOpenAccepted | .iAmOpenRejected => - some s!"iamopen:{event.node}" - | _ => none - -private def observedSendCompatible - (send : ObservedSend) - (event : TraceEvent) : Bool := - match event.source, expectedReceiveSend event with - | some source, some expected => - send.source == source && send.description == expected - | _, _ => false - -private def hiddenSendCompatible - (candidate : Candidate) - (event : TraceEvent) : Bool := - match event.source, expectedReceiveSend event with - | some source, some expected => - candidate.hiddenSends.contains { - source - description := expected - } - | _, _ => false - -private def observationCompatible - (config : Config) - (system : SystemState) - (event : TraceEvent) : Bool := - match nodeState system event.node with - | none => false - | some state => - if !phaseMatches event.pre state.phase || - !phaseMatches event.post state.phase then - false - else - match event.kind with - | .send => - match event.send with - | none => false - | some expected => - (step config state .retry).effects.any - (fun effect => effectName effect == some expected) - | .open => - state.phase == .opening && event.openKind == state.openKind - | .joinRestart => - state.phase == .joining && state.restartRequested - | .complete => - state.phase == .open - | _ => false - -private def historicalSendCompatible - (candidate : Candidate) - (event : TraceEvent) : Bool := - match nodeState candidate.system event.node, event.send with - | some state, some description => - phaseMatches event.pre state.phase && - phaseMatches event.post state.phase && - candidate.hiddenSends.contains { - source := event.node - description - } - | _, _ => false - -private def removePendingEffect (node : Location) (target : Effect) : - List PendingEffect -> Option (List PendingEffect) - | [] => none - | pending :: rest => - if pending.node == node && pending.effect == target then - some rest - else - (removePendingEffect node target rest).map (fun remaining => - pending :: remaining) - -private def removePendingRestart (node : Location) : - List PendingEffect -> Option (List PendingEffect) - | [] => none - | pending :: rest => - match pending.effect with - | .restart _ => - if pending.node == node then - some rest - else - (removePendingRestart node rest).map (fun remaining => - pending :: remaining) - | _ => - (removePendingRestart node rest).map (fun remaining => - pending :: remaining) - -private def consumeObservation - (config : Config) - (candidate : Candidate) - (event : TraceEvent) : Option Candidate := do - match event.kind with - | .send => do - guard ( - observationCompatible config candidate.system event || - historicalSendCompatible candidate event) - some candidate - | .open => do - guard (observationCompatible config candidate.system event) - let kind <- event.openKind - let pendingEffects <- removePendingEffect event.node (.opening kind) - candidate.pendingEffects - some { candidate with pendingEffects } - | .joinRestart => do - guard (observationCompatible config candidate.system event) - let pendingEffects <- removePendingRestart event.node - candidate.pendingEffects - some { candidate with pendingEffects } - | .complete => do - guard (observationCompatible config candidate.system event) - let pendingEffects <- removePendingEffect event.node .completed - candidate.pendingEffects - some { candidate with pendingEffects } - | _ => none - -private def isOneShotEffect : Effect -> Bool - | .opening _ | .restart _ | .completed => true - | _ => false - -private def transitionCandidates - (config : Config) - (system : SystemState) - (event : TraceEvent) : List (Prod SystemState StepOutput) := - match nodeState system event.node with - | none => [] - | some before => - if !phaseMatches event.pre before.phase then [] - else - (eventInputs event).filterMap fun input => do - let (nextSystem, output) <- systemStep config system event.node input - let acceptanceOk := - match expectsAcceptance event.kind with - | none => true - | some expected => output.accepted == expected - if acceptanceOk && phaseMatches event.post output.state.phase then - some (nextSystem, output) - else - none - -private def expectedEvents (candidates : List Candidate) (node : Location) : - List String := - let phases := - candidates.filterMap (fun candidate => nodeState candidate.system node) |>.map - (fun state => phaseName state.phase) - let phaseText := String.intercalate "/" phases.eraseDups - let common := [ - "timeout", "retry", "gossip_accepted|gossip_rejected", - "vote_accepted|vote_rejected", "iamopen_accepted|iamopen_rejected" - ] - let canOpen := candidates.any fun candidate => - candidate.pendingEffects.any fun pending => - pending.node == node && - match pending.effect with - | .opening _ => true - | _ => false - let canRestart := candidates.any fun candidate => - candidate.pendingEffects.any fun pending => - pending.node == node && - match pending.effect with - | .restart _ => true - | _ => false - let canComplete := candidates.any fun candidate => - candidate.pendingEffects.any fun pending => - pending.node == node && pending.effect == .completed - let observations := - (if canOpen then ["open(open_kind=QUORUM|FAILOVER)"] else []) ++ - (if phases.contains "OPENING" then ["send(iamopen:DEST)"] else []) ++ - (if canRestart then ["join_restart"] else []) ++ - (if canComplete then ["complete"] else []) - s!"state={phaseText}" :: common ++ observations - -structure ValidatorState where - config : Option Config := none - candidates : List Candidate := [] - nextSequence : List (Prod Location Nat) := [] - startedNodes : List Location := [] - seenMessageIds : List String := [] - observedSends : List ObservedSend := [] - consumedSendIds : List String := [] -deriving Inhabited - -private def expectedSequence (state : ValidatorState) (node : Location) : Nat := - (state.nextSequence.find? fun entry => entry.1 == node).map Prod.snd |>.getD 0 - -private def setSequence - (sequences : List (Prod Location Nat)) - (node : Location) - (next : Nat) : - List (Prod Location Nat) := - if sequences.any (fun entry => entry.1 == node) then - sequences.map fun entry => if entry.1 == node then (node, next) else entry - else - (node, next) :: sequences - -private def process - (index : Nat) - (state : ValidatorState) - (event : TraceEvent) : - Except Failure ValidatorState := do - let missing := missingRequiredFields event - if !missing.isEmpty then - throw { - prefixLength := index + 1 - message := s!"missing required fields: {String.intercalate ", " missing}" - expected := [] - } - if requiresSource event.kind && (event.source.getD "").isEmpty then - throw { - prefixLength := index + 1 - message := "source must not be empty" - expected := [] - } - if event.messageId.map String.isEmpty |>.getD false then - throw { - prefixLength := index + 1 - message := "message_id must not be empty" - expected := [] - } - if event.causedBy.map String.isEmpty |>.getD false then - throw { - prefixLength := index + 1 - message := "caused_by must not be empty" - expected := [] - } - if !requiresSource event.kind && event.causedBy.isSome then - throw { - prefixLength := index + 1 - message := "caused_by is only valid on receive events" - expected := [] - } - let expectedSeq := expectedSequence state event.node - if event.sequence != expectedSeq then - throw { - prefixLength := index + 1 - message := s!"node {event.node} sequence {event.sequence}, expected {expectedSeq}" - expected := [] - } - let config : Config := { - instanceId := event.instanceId - expectedLocations := event.expectedLocations - } - match configError config with - | some message => - throw { - prefixLength := index + 1 - message - expected := [] - } - | none => pure () - match event.messageId with - | some messageId => - if state.seenMessageIds.contains messageId || - state.consumedSendIds.contains messageId then - throw { - prefixLength := index + 1 - message := s!"message_id '{messageId}' was already used" - expected := [] - } - | none => pure () - if event.messageId.isSome && event.messageId == event.causedBy then - throw { - prefixLength := index + 1 - message := "message_id and caused_by must identify distinct observations" - expected := [] - } - match event.kind, state.config with - | .start, none => - if !config.expectedLocations.contains event.node then - throw { - prefixLength := index + 1 - message := s!"start node {event.node} is not expected" - expected := config.expectedLocations - } - let system := initialSystem config - let node := (nodeState system event.node).get! - if !phaseMatches event.pre node.phase || !phaseMatches event.post node.phase then - throw { - prefixLength := index + 1 - message := "start pre/post phase does not match GOSSIPING" - expected := ["pre=GOSSIPING", "post=GOSSIPING"] - } - pure { - config := some config - candidates := [{ system }] - nextSequence := setSequence state.nextSequence event.node (expectedSeq + 1) - startedNodes := [event.node] - seenMessageIds := event.messageId.toList - } - | .start, some established => - if established != config then - throw { - prefixLength := index + 1 - message := "instance or expected_locations changed" - expected := [] - } - if !config.expectedLocations.contains event.node then - throw { - prefixLength := index + 1 - message := s!"start node {event.node} is not expected" - expected := config.expectedLocations - } - if state.startedNodes.contains event.node then - throw { - prefixLength := index + 1 - message := s!"duplicate start event for node {event.node}" - expected := [] - } - let closure := - hiddenClosure config state.startedNodes state.candidates - let next := closure.filter fun candidate => - match nodeState candidate.system event.node with - | none => false - | some node => - phaseMatches event.pre node.phase && - phaseMatches event.post node.phase - if next.isEmpty then - throw { - prefixLength := index + 1 - message := "start pre/post phase does not match GOSSIPING" - expected := ["pre=GOSSIPING", "post=GOSSIPING"] - } - let startedNodes := event.node :: state.startedNodes - pure { - state with - candidates := hiddenClosure config startedNodes next - nextSequence := - setSequence state.nextSequence event.node (expectedSeq + 1) - startedNodes - seenMessageIds := event.messageId.toList ++ state.seenMessageIds - } - | _, none => - throw { - prefixLength := index + 1 - message := "trace must begin with start" - expected := ["start"] - } - | _, some established => - if established != config then - throw { - prefixLength := index + 1 - message := "instance or expected_locations changed" - expected := [] - } - if !state.startedNodes.contains event.node then - throw { - prefixLength := index + 1 - message := s!"node {event.node} has no start event" - expected := ["start"] - } - let closure := - hiddenClosure config state.startedNodes state.candidates - let mustMatchSend := - match event.kind with - | .gossipAccepted | .voteAccepted | .iAmOpenAccepted => true - | .gossipRejected | .voteRejected | .iAmOpenRejected => - event.causedBy.isSome - | _ => false - let sourceCandidates := - if mustMatchSend && - config.expectedLocations.contains (event.source.getD "") then - closure.filter - (fun (candidate : Candidate) => - hiddenSendCompatible candidate event) - else - closure - let causalCandidates := - match event.causedBy with - | none => sourceCandidates - | some cause => - if state.consumedSendIds.contains cause then - [] - else - match state.observedSends.find? - (fun (send : ObservedSend) => send.messageId == cause) with - | some send => - if observedSendCompatible send event then - sourceCandidates - else - [] - | none => - if state.seenMessageIds.contains cause then - [] - else - sourceCandidates - if event.causedBy.isSome && causalCandidates.isEmpty then - throw { - prefixLength := index + 1 - message := s!"caused_by '{event.causedBy.getD ""}' has no prior or hidden compatible send" - expected := expectedEvents closure event.node - } - let next := - match event.kind with - | .send | .open | .joinRestart | .complete => - causalCandidates.filterMap - (fun (candidate : Candidate) => - consumeObservation config candidate event) - | _ => - causalCandidates.flatMap fun (candidate : Candidate) => - (transitionCandidates config candidate.system event).map - (fun (system, output) => { - candidate with - system - pendingEffects := candidate.pendingEffects ++ - (output.effects.filter isOneShotEffect).map - (fun effect => { - node := event.node - effect - }) - }) - let next := hiddenClosure config state.startedNodes - (uniqueCandidates next) - if next.isEmpty then - throw { - prefixLength := index + 1 - message := s!"event {repr event.kind} is incompatible" - expected := expectedEvents closure event.node - } - pure { - config := some config - candidates := next - nextSequence := - setSequence state.nextSequence event.node (expectedSeq + 1) - startedNodes := state.startedNodes - seenMessageIds := event.messageId.toList ++ state.seenMessageIds - observedSends := - match event.kind, event.messageId, event.send with - | .send, some messageId, some description => - { - messageId - source := event.node - description - } :: state.observedSends - | _, _, _ => state.observedSends - consumedSendIds := event.causedBy.toList ++ state.consumedSendIds - } - -def validate (events : List TraceEvent) : Except Failure Nat := do - let mut state : ValidatorState := {} - for (event, index) in events.zipIdx do - state <- process index state event - if events.isEmpty then - throw { - prefixLength := 0 - message := "empty trace" - expected := ["start"] - } - match state.config with - | none => - throw { - prefixLength := events.length - message := "trace has no configuration" - expected := ["start"] - } - | some _ => pure () - pure state.candidates.length - -def renderFailure (failure : Failure) : String := - let expected := - if failure.expected.isEmpty then "" - else s!"\nexpected compatible events:\n {String.intercalate "\n " failure.expected}" - s!"shortest failing prefix: {failure.prefixLength}\n{failure.message}{expected}" - -end DisasterRecovery.Protocol.Trace +import DisasterRecovery.Protocol.Trace.Format +import DisasterRecovery.Protocol.Trace.Replay diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/Trace/Format.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/Trace/Format.lean new file mode 100644 index 000000000000..ef40da360268 --- /dev/null +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/Trace/Format.lean @@ -0,0 +1,141 @@ +import DisasterRecovery.Protocol.Model +import Lean.Data.Json +import Lean.Data.Json.FromToJson + +namespace DisasterRecovery.Protocol.Trace + +open Lean + +def contractVersion : String := + "ccf.recovery_decision_protocol.trace/1" + +inductive Kind where + | start + | gossipAccepted + | voteAccepted + | iAmOpenAccepted + | timeout + | send + | open + | joinRestart + | complete +deriving Repr, BEq, Inhabited + +structure TraceEvent where + version : String + instanceId : String + expectedLocations : List Location + node : Location + sequence : Nat + kind : Kind + messageId : Option String + causedBy : Option String + source : Option Location + txid : Option TxID + pre : Option Phase + post : Option Phase + openKind : Option OpenKind + send : Option String +deriving Repr, BEq, Inhabited + +private def parseKind : String -> Except String Kind + | "start" => pure .start + | "gossip_accepted" => pure .gossipAccepted + | "vote_accepted" => pure .voteAccepted + | "iamopen_accepted" => pure .iAmOpenAccepted + | "timeout" => pure .timeout + | "send" => pure .send + | "open" => pure .open + | "join_restart" => pure .joinRestart + | "complete" => pure .complete + | value => throw s!"unknown kind '{value}'" + +private def parsePhase : String -> Except String Phase + | "GOSSIPING" => pure .gossiping + | "VOTING" => pure .voting + | "OPENING" => pure .opening + | "JOINING" => pure .joining + | "OPEN" => pure .open + | value => throw s!"unknown phase '{value}'" + +private def parseOpenKind : String -> Except String OpenKind + | "QUORUM" => pure .quorum + | "FAILOVER" => pure .failover + | value => throw s!"unknown open kind '{value}'" + +private def optionalString (json : Json) (key : String) : + Except String (Option String) := + match json.getObjVal? key with + | .error _ | .ok .null => pure none + | .ok value => some <$> value.getStr? + +private def optionalNat (json : Json) (key : String) : + Except String (Option Nat) := + match json.getObjVal? key with + | .error _ | .ok .null => pure none + | .ok value => some <$> value.getNat? + +private def optionalParsed + (json : Json) + (key : String) + (parse : String -> Except String α) : + Except String (Option α) := do + match <- optionalString json key with + | none => pure none + | some value => some <$> parse value + +def parseEvent (line : String) : Except String TraceEvent := do + let json <- Json.parse line + let version <- json.getObjValAs? String "version" + if version != contractVersion then + throw s!"unsupported version '{version}'" + + let view <- optionalNat json "view" + let seqno <- optionalNat json "seqno" + if view.isSome != seqno.isSome then + throw "view and seqno must appear together" + + let instanceId <- json.getObjValAs? String "instance" + let expectedLocations <- + json.getObjValAs? (List String) "expected_locations" + let node <- json.getObjValAs? String "node" + let sequence <- json.getObjValAs? Nat "sequence" + let kindName <- json.getObjValAs? String "kind" + let kind <- parseKind kindName + let messageId <- optionalString json "message_id" + let causedBy <- optionalString json "caused_by" + let source <- optionalString json "source" + let pre <- optionalParsed json "pre" parsePhase + let post <- optionalParsed json "post" parsePhase + let openKind <- optionalParsed json "open_kind" parseOpenKind + let send <- optionalString json "send" + pure { + version + instanceId + expectedLocations + node + sequence + kind + messageId + causedBy + source + txid := match view, seqno with + | some view, some seqno => some { view, seqno } + | _, _ => none + pre + post + openKind + send + } + +def parseNDJSON (input : String) : Except String (List TraceEvent) := do + let lines := (input.splitOn "\n").filter + (fun line => !line.trimAscii.isEmpty) + let mut events := [] + for (line, index) in lines.zipIdx do + match parseEvent line with + | .ok event => events := event :: events + | .error message => throw s!"line {index + 1}: {message}" + pure events.reverse + +end DisasterRecovery.Protocol.Trace diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/Trace/Replay.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/Trace/Replay.lean new file mode 100644 index 000000000000..9921d09f6c76 --- /dev/null +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/Trace/Replay.lean @@ -0,0 +1,445 @@ +import DisasterRecovery.Protocol.Trace.Format + +namespace DisasterRecovery.Protocol.Trace + +structure Failure where + prefixLength : Nat + message : String + expected : List String +deriving Repr, BEq + +structure ObservedSend where + messageId : String + source : Location + description : String +deriving Repr, BEq + +structure PendingEffect where + node : Location + effect : Effect +deriving Repr, BEq + +structure PendingSendBatch where + node : Location + remaining : List String +deriving Repr, BEq + +structure ActiveReplay where + config : Config + system : SystemState + startedNodes : List Location + sends : List ObservedSend := [] + consumedSendIds : List String := [] + pendingEffects : List PendingEffect := [] + pendingSendBatches : List PendingSendBatch := [] + terminalNodes : List Location := [] + completedNodes : List Location := [] +deriving Repr, BEq + +structure ReplayState where + active : Option ActiveReplay := none + nextSequence : List (Prod Location Nat) := [] + seenMessageIds : List String := [] +deriving Repr, BEq, Inhabited + +private def nodeState (system : SystemState) (node : Location) : Option NodeState := + (system.nodes.find? fun entry => entry.1 == node).map Prod.snd + +private def phaseMatches (expected : Option Phase) (actual : Phase) : Bool := + expected == some actual + +private def isReceive : Kind -> Bool + | .gossipAccepted | .voteAccepted | .iAmOpenAccepted => true + | _ => false + +private def shapeError (event : TraceEvent) : Option String := + if event.pre.isNone || event.post.isNone then + some "pre and post are required" + else if isReceive event.kind && + (event.messageId.isNone || event.causedBy.isNone || event.source.isNone) then + some "message_id, caused_by, and source are required for receives" + else if event.kind == .gossipAccepted && event.txid.isNone then + some "view and seqno are required for gossip" + else if event.kind == .send && + (event.messageId.isNone || event.send.isNone) then + some "message_id and send are required for sends" + else if event.kind == .open && event.openKind.isNone then + some "open_kind is required for open" + else if !isReceive event.kind && event.causedBy.isSome then + some "caused_by is only valid on receive events" + else if event.messageId.map String.isEmpty |>.getD false then + some "message_id must not be empty" + else if event.causedBy.map String.isEmpty |>.getD false then + some "caused_by must not be empty" + else if event.source.map String.isEmpty |>.getD false then + some "source must not be empty" + else + none + +private def configError (config : Config) : Option String := + if config.instanceId.isEmpty then + some "instance must not be empty" + else if config.expectedLocations.isEmpty then + some "expected_locations must not be empty" + else if config.expectedLocations.any String.isEmpty then + some "expected_locations must not contain an empty name" + else if config.expectedLocations.eraseDups.length != + config.expectedLocations.length then + some "expected_locations must not contain duplicates" + else + none + +private def expectedSequence (state : ReplayState) (node : Location) : Nat := + (state.nextSequence.find? fun entry => entry.1 == node).map Prod.snd |>.getD 0 + +private def setSequence + (sequences : List (Prod Location Nat)) + (node : Location) + (next : Nat) : + List (Prod Location Nat) := + if sequences.any (fun entry => entry.1 == node) then + sequences.map fun entry => if entry.1 == node then (node, next) else entry + else + (node, next) :: sequences + +private def effectName : Effect -> Option String + | .sendGossip destination => some s!"gossip:{destination}" + | .sendVote destination => some s!"vote:{destination}" + | .sendIAmOpen destination => some s!"iamopen:{destination}" + | _ => none + +private def sendBatch (config : Config) (state : NodeState) : List String := + (step config state .retry).effects.filterMap effectName + +private def setPendingSendBatch + (node : Location) + (remaining : List String) + (batches : List PendingSendBatch) : + List PendingSendBatch := + let others := batches.filter (fun batch => batch.node != node) + if remaining.isEmpty then + others + else + { node, remaining } :: others + +private def receiveDescription (event : TraceEvent) : Option String := + match event.kind with + | .gossipAccepted => some s!"gossip:{event.node}" + | .voteAccepted => some s!"vote:{event.node}" + | .iAmOpenAccepted => some s!"iamopen:{event.node}" + | _ => none + +private def eventInput (event : TraceEvent) : Option Event := + match event.kind, event.source, event.txid with + | .gossipAccepted, some source, some txid => + some (.receiveGossip source txid .accepted) + | .voteAccepted, some source, _ => + some (.receiveVote source .accepted) + | .iAmOpenAccepted, some source, _ => + some (.receiveIAmOpen source .accepted) + | .timeout, _, _ => some .timeout + | _, _, _ => none + +private def isOneShotEffect : Effect -> Bool + | .opening _ | .restart _ | .completed => true + | _ => false + +private def addEffects + (node : Location) + (effects : List Effect) + (pending : List PendingEffect) : + List PendingEffect := + pending ++ (effects.filter isOneShotEffect).map fun effect => + ({ node := node, effect := effect } : PendingEffect) + +private def removeEffect (node : Location) (target : Effect) : + List PendingEffect -> Option (List PendingEffect) + | [] => none + | pending :: rest => + if pending.node == node && pending.effect == target then + some rest + else + (removeEffect node target rest).map (fun remaining => + pending :: remaining) + +private def removeRestart (node : Location) : + List PendingEffect -> Option (List PendingEffect) + | [] => none + | pending :: rest => + if pending.node == node then + match pending.effect with + | .restart _ => some rest + | _ => (removeRestart node rest).map (fun remaining => + pending :: remaining) + else + (removeRestart node rest).map (fun remaining => pending :: remaining) + +private def consumeCause + (active : ActiveReplay) + (event : TraceEvent) : Except String ActiveReplay := do + let cause := event.causedBy.getD "" + if active.consumedSendIds.contains cause then + throw s!"caused_by '{cause}' was already consumed" + let send <- match active.sends.find? (fun send => send.messageId == cause) with + | none => throw s!"caused_by '{cause}' has no prior send" + | some send => pure send + let source := event.source.getD "" + let description := receiveDescription event |>.getD "" + if send.source != source || send.description != description then + throw s!"caused_by '{cause}' has the wrong source, class, or destination" + pure { + active with + consumedSendIds := cause :: active.consumedSendIds + } + +private def applyTransition + (active : ActiveReplay) + (event : TraceEvent) : Except String ActiveReplay := do + let before <- match nodeState active.system event.node with + | none => throw s!"unknown node {event.node}" + | some state => pure state + if !phaseMatches event.pre before.phase then + throw s!"pre phase does not match {phaseName before.phase}" + let input <- match eventInput event with + | none => throw "event is not a protocol transition" + | some input => pure input + let (system, output) <- match + systemStep active.config active.system event.node input with + | none => throw s!"unknown node {event.node}" + | some result => pure result + if !output.accepted then + throw "protocol transition was rejected" + if !phaseMatches event.post output.state.phase then + throw s!"post phase does not match {phaseName output.state.phase}" + pure { + active with + system + pendingEffects := + addEffects event.node output.effects active.pendingEffects + } + +private def applyReceive + (active : ActiveReplay) + (event : TraceEvent) : Except String ActiveReplay := do + applyTransition (← consumeCause active event) event + +private def applySend + (active : ActiveReplay) + (event : TraceEvent) : Except String ActiveReplay := do + let state <- match nodeState active.system event.node with + | none => throw s!"unknown node {event.node}" + | some state => pure state + if !phaseMatches event.pre state.phase || !phaseMatches event.post state.phase then + throw s!"send phase does not match {phaseName state.phase}" + let description := event.send.getD "" + let batch := (active.pendingSendBatches.find? + (fun batch => batch.node == event.node)).map (fun batch => batch.remaining) + |>.getD (sendBatch active.config state) + let expected <- match batch with + | [] => throw "no retry send batch is enabled" + | expected :: _ => pure expected + if description != expected then + throw s!"expected send '{expected}', got '{description}'" + pure { + active with + sends := { + messageId := event.messageId.getD "" + source := event.node + description + } :: active.sends + pendingSendBatches := + setPendingSendBatch event.node batch.tail active.pendingSendBatches + } + +private def applyObservation + (active : ActiveReplay) + (event : TraceEvent) : Except String ActiveReplay := do + let state <- match nodeState active.system event.node with + | none => throw s!"unknown node {event.node}" + | some state => pure state + if !phaseMatches event.pre state.phase || !phaseMatches event.post state.phase then + throw s!"observation phase does not match {phaseName state.phase}" + match event.kind with + | .open => + if state.phase != .opening || event.openKind != state.openKind then + throw "open observation does not match state" + let pendingEffects <- match + removeEffect event.node (.opening event.openKind.get!) active.pendingEffects with + | none => throw "open observation has no pending opening effect" + | some pending => pure pending + pure { active with pendingEffects } + | .joinRestart => + if state.phase != .joining || !state.restartRequested then + throw "join_restart observation does not match state" + let pendingEffects <- match removeRestart event.node active.pendingEffects with + | none => throw "join_restart has no pending restart effect" + | some pending => pure pending + pure { + active with + pendingEffects + terminalNodes := event.node :: active.terminalNodes + } + | .complete => + if state.phase != .open then + throw "complete observation does not match state" + let pendingEffects <- match + removeEffect event.node .completed active.pendingEffects with + | none => throw "complete has no pending completion effect" + | some pending => pure pending + pure { + active with + pendingEffects + terminalNodes := event.node :: active.terminalNodes + completedNodes := event.node :: active.completedNodes + } + | _ => throw "event is not a protocol observation" + +private def expectedEvents (active : ActiveReplay) (node : Location) : + List String := + let phase := nodeState active.system node |>.map + (fun state => phaseName state.phase) |>.getD "UNKNOWN" + [s!"state={phase}", "send", "gossip_accepted", "vote_accepted", + "iamopen_accepted", "timeout", "open", "join_restart", "complete"] + +private def start + (active : Option ActiveReplay) + (config : Config) + (event : TraceEvent) : Except String ActiveReplay := do + if !config.expectedLocations.contains event.node then + throw s!"start node {event.node} is not expected" + let current := active.getD { + config + system := initialSystem config + startedNodes := [] + } + if current.config != config then + throw "instance or expected_locations changed" + if current.startedNodes.contains event.node then + throw s!"duplicate start event for node {event.node}" + let state <- match nodeState current.system event.node with + | none => throw s!"unknown node {event.node}" + | some state => pure state + if !phaseMatches event.pre state.phase || !phaseMatches event.post state.phase then + throw "start pre/post phase does not match GOSSIPING" + pure { + current with + startedNodes := event.node :: current.startedNodes + } + +private def processActive + (active : ActiveReplay) + (event : TraceEvent) : Except String ActiveReplay := do + if active.config.instanceId != event.instanceId || + active.config.expectedLocations != event.expectedLocations then + throw "instance or expected_locations changed" + if !active.startedNodes.contains event.node then + throw s!"node {event.node} has no start event" + if event.kind != .send && + active.pendingSendBatches.any (fun batch => batch.node == event.node) then + throw s!"node {event.node} has an incomplete retry send batch" + match event.kind with + | .gossipAccepted | .voteAccepted | .iAmOpenAccepted => + applyReceive active event + | .timeout => applyTransition active event + | .send => applySend active event + | .open | .joinRestart | .complete => applyObservation active event + | .start => throw "unexpected start event" + +private def fail + (index : Nat) + (message : String) + (expected : List String := []) : + Except Failure α := + throw { prefixLength := index + 1, message, expected } + +private def process + (index : Nat) + (state : ReplayState) + (event : TraceEvent) : Except Failure ReplayState := do + if let some message := shapeError event then + fail index message + let config : Config := { + instanceId := event.instanceId + expectedLocations := event.expectedLocations + } + if let some message := configError config then + fail index message + let expectedSeq := expectedSequence state event.node + if event.sequence != expectedSeq then + fail index s!"node {event.node} sequence {event.sequence}, expected {expectedSeq}" + if let some messageId := event.messageId then + if state.seenMessageIds.contains messageId then + fail index s!"message_id '{messageId}' was already used" + if event.causedBy == some messageId then + fail index "message_id and caused_by must identify distinct observations" + + let nextActive <- match event.kind with + | .start => + match start state.active config event with + | .ok active => pure active + | .error message => fail index message + | _ => + match state.active with + | none => fail index "trace must begin with start" ["start"] + | some active => + match processActive active event with + | .ok next => pure next + | .error message => fail index message (expectedEvents active event.node) + + pure { + active := some nextActive + nextSequence := + setSequence state.nextSequence event.node (expectedSeq + 1) + seenMessageIds := event.messageId.toList ++ state.seenMessageIds + } + +def validate (events : List TraceEvent) : Except Failure Unit := do + if events.isEmpty then + throw { + prefixLength := 0 + message := "empty trace" + expected := ["start"] + } + let mut state : ReplayState := {} + for (event, index) in events.zipIdx do + state <- process index state event + let active <- match state.active with + | none => + throw { + prefixLength := events.length + message := "trace has no start event" + expected := ["start"] + } + | some active => pure active + if !active.pendingEffects.isEmpty then + throw { + prefixLength := events.length + message := "trace ended with unobserved committed effects" + expected := ["open", "join_restart", "complete"] + } + if !active.pendingSendBatches.isEmpty then + throw { + prefixLength := events.length + message := "trace ended with incomplete retry send batches" + expected := ["send"] + } + if !active.startedNodes.all (fun node => active.terminalNodes.contains node) then + throw { + prefixLength := events.length + message := "trace ended before every participating node terminated" + expected := ["join_restart", "complete"] + } + if active.completedNodes.isEmpty then + throw { + prefixLength := events.length + message := "trace has no completed opener" + expected := ["complete"] + } + +def renderFailure (failure : Failure) : String := + let expected := + if failure.expected.isEmpty then "" + else s!"\nexpected compatible events:\n {String.intercalate "\n " failure.expected}" + s!"shortest failing prefix: {failure.prefixLength}\n{failure.message}{expected}" + +end DisasterRecovery.Protocol.Trace diff --git a/lean/disaster-recovery/README.md b/lean/disaster-recovery/README.md index 573ffa573bb0..64c7670b0a3a 100644 --- a/lean/disaster-recovery/README.md +++ b/lean/disaster-recovery/README.md @@ -24,31 +24,32 @@ intentionally differs from the legacy model in several places. It has formal phase-refinement and fairness-aware progress results, plus a versioned trace validator designed for future committed C++ instrumentation. -The initial migration is approximately 4,300 lines across 26 new files: +The initial migration is approximately 4,000 lines across 28 new files: | Area | Files | Approximate lines | Purpose | | ----------------------------------------- | ----: | ----------------: | --------------------------------------------------------------- | | Exact legacy Lean model | 4 | 650 | Stateright state, actions, timers, network, predicates, and BFS | | Canonical protocol and proofs | 4 | 1,000 | C++ behavior, phase refinement, quorum, and temporal proofs | -| Trace validation | 9 | 1,360 | NDJSON contract, validator, tests, and quorum/failover fixtures | +| Trace validation | 11 | 1,020 | NDJSON format, deterministic replay, tests, and fixtures | | Equivalence tooling | 2 | 710 | Rust graph exporter and bidirectional Rust/Lean comparison | | Project, documentation, and configuration | 6 | 530 | Lake/Mathlib setup, entry points, and documentation | | Pull-request CI | 1 | 80 | Lean model and bounded equivalence checks | ### Principal files -| File | Lines | Role | -| ---------------------------------------------------------------------------------------- | ----: | ------------------------------------------------------------------------ | -| [`DisasterRecovery/Model.lean`](DisasterRecovery/Model.lean) | 392 | Exact executable legacy semantics | -| [`DisasterRecovery/Checker.lean`](DisasterRecovery/Checker.lean) | 152 | BFS enumeration, property checking, and canonical graph export | -| [`DisasterRecovery/Protocol/Model.lean`](DisasterRecovery/Protocol/Model.lean) | 286 | Production-oriented C++ protocol model | -| [`DisasterRecovery/Protocol/Refinement.lean`](DisasterRecovery/Protocol/Refinement.lean) | 332 | Canonical-to-legacy formal phase refinement | -| [`DisasterRecovery/Protocol/Temporal.lean`](DisasterRecovery/Protocol/Temporal.lean) | 230 | Execution streams, fairness, safety, and progress proofs | -| [`DisasterRecovery/Protocol/Trace.lean`](DisasterRecovery/Protocol/Trace.lean) | 772 | Versioned implementation-trace parser and validator | -| [`TraceTests.lean`](TraceTests.lean) | 394 | Configuration, causality, delayed-send, and committed-effect regressions | -| [`compare.py`](compare.py) | 343 | Exhaustive canonical graph comparison | -| [`../../tla/disaster-recovery/src/export.rs`](../../tla/disaster-recovery/src/export.rs) | 370 | Canonical export of the actual Stateright model | -| [`TRACE_FORMAT_V1.md`](TRACE_FORMAT_V1.md) | 145 | Contract for future committed C++ trace events | +| File | Lines | Role | +| -------------------------------------------------------------------------------------------- | ----: | -------------------------------------------------------------- | +| [`DisasterRecovery/Model.lean`](DisasterRecovery/Model.lean) | 392 | Exact executable legacy semantics | +| [`DisasterRecovery/Checker.lean`](DisasterRecovery/Checker.lean) | 152 | BFS enumeration, property checking, and canonical graph export | +| [`DisasterRecovery/Protocol/Model.lean`](DisasterRecovery/Protocol/Model.lean) | 286 | Production-oriented C++ protocol model | +| [`DisasterRecovery/Protocol/Refinement.lean`](DisasterRecovery/Protocol/Refinement.lean) | 332 | Canonical-to-legacy formal phase refinement | +| [`DisasterRecovery/Protocol/Temporal.lean`](DisasterRecovery/Protocol/Temporal.lean) | 230 | Execution streams, fairness, safety, and progress proofs | +| [`DisasterRecovery/Protocol/Trace/Format.lean`](DisasterRecovery/Protocol/Trace/Format.lean) | 141 | Versioned NDJSON types and parser | +| [`DisasterRecovery/Protocol/Trace/Replay.lean`](DisasterRecovery/Protocol/Trace/Replay.lean) | 445 | Deterministic implementation-trace replay | +| [`TraceTests.lean`](TraceTests.lean) | 210 | Strict replay, causality, sequencing, and effect regressions | +| [`compare.py`](compare.py) | 343 | Exhaustive canonical graph comparison | +| [`../../tla/disaster-recovery/src/export.rs`](../../tla/disaster-recovery/src/export.rs) | 370 | Canonical export of the actual Stateright model | +| [`TRACE_FORMAT_V1.md`](TRACE_FORMAT_V1.md) | 145 | Contract for future committed C++ trace events | The migration also updates the existing Stateright CLI and documentation, adds weekly exhaustive verification, and adds @@ -294,9 +295,10 @@ satisfy a one-location threshold. The model preserves this behavior. The phase refinement theorem covers this step; expected-source restrictions appear only in `LegacyDataAssumptions` for richer data comparisons. -The trace validator treats a nonempty accepted source outside -`expected_locations` as an external input. It applies the C++-aligned receive -transition but cannot validate the unmodeled sender's behavior. +The strict version 1 trace requires every accepted receive to reference an +instrumented send, so it covers configured protocol participants rather than +the unexpected external-source path. That implementation discrepancy remains +explicit in the canonical model and its tests. ## Temporal results @@ -361,28 +363,20 @@ counterexample to full initial-state equality. ## Trace validation The versioned NDJSON contract is documented in -[`TRACE_FORMAT_V1.md`](TRACE_FORMAT_V1.md). `trace-validator` parses it without -another dependency, tracks a set of compatible canonical states, and closes -that set over hidden retry/send stuttering. Each candidate retains the send -classes that could have been emitted by earlier hidden retries, so delayed -messages remain matchable after a sender changes phase without combining -incompatible histories. Candidates also retain one-shot committed effects; -`open`, `join_restart`, and `complete` observations consume these exactly once -for the node that produced them. On failure the validator reports the shortest -failing prefix and expected compatible events. - -Explicit sends may match an accumulated earlier capability when a retry task -selected work before a concurrent state commit and dispatched it afterward. - -The canonical v1 relation is deterministic, so a successful v1 prefix currently -retains one candidate. The candidate list and separate histories are explicit -for future under-observed or nondeterministic refinements. - -The validator also enforces configuration validity, one ordered event sequence -per node, one start per participating node, required event fields, globally -unique message IDs, feasible accepted receives from configured sources, and one -receive per supplied causal send ID. Observable `pre`, `post`, and open-kind -values are checked against every remaining candidate state. +[`TRACE_FORMAT_V1.md`](TRACE_FORMAT_V1.md). Version 1 records complete +successful executions: sends, accepted receives, committed timeouts, and +one-shot effects are explicit. + +[`Trace/Format.lean`](DisasterRecovery/Protocol/Trace/Format.lean) parses the +wire format. [`Trace/Replay.lean`](DisasterRecovery/Protocol/Trace/Replay.lean) +then folds each event over one deterministic canonical `SystemState`. Replay +retains only per-node sequences, observed and consumed causal sends, and +node-scoped pending `open`, `join_restart`, or `complete` effects. + +The validator checks configuration consistency, starts, globally unique message +IDs, send enablement, exact send/receive causality, canonical pre/post phases, +and single consumption of committed effects. On failure it reports the shortest +failing prefix and current protocol phase. ### Implementation trace validation @@ -393,7 +387,8 @@ logged before dispatch with causal IDs propagated to accepted receive records. For trace-enabled joiners, the hook emits the committed receive and `join_restart` records before requesting host restart. Trace-enabled periodic retries defer work while their locally committed phase -is ahead of the globally visible trace phase, then send on the next invocation. +is ahead of the globally visible trace phase. Once phases match, the trace lock +serializes the complete send batch against later commit publication. [`tests/infra/recovery_trace.py`](../../tests/infra/recovery_trace.py) extracts records from all recovery nodes and topologically orders them from per-node diff --git a/lean/disaster-recovery/TRACE_FORMAT_V1.md b/lean/disaster-recovery/TRACE_FORMAT_V1.md index 61fcf07ef86e..306460db6175 100644 --- a/lean/disaster-recovery/TRACE_FORMAT_V1.md +++ b/lean/disaster-recovery/TRACE_FORMAT_V1.md @@ -40,83 +40,53 @@ All integers must be nonnegative Lean `Nat` values. ## Event kinds -| Kind | Required event fields | Canonical boundary | -| ------------------ | --------------------------------------------------------------- | ------------------------------------------- | -| `start` | `pre`, `post` | Protocol state initialized | -| `gossip_accepted` | `message_id`, `source`, `view`, `seqno`, `pre`, `post` | Validated gossip callback committed | -| `gossip_rejected` | `message_id`, `source`, `view`, `seqno`, `pre`, `post` | Validation or protocol rejection | -| `vote_accepted` | `message_id`, `source`, `pre`, `post` | Validated vote callback committed | -| `vote_rejected` | `message_id`, `source`, `pre`, `post` | Validation rejection | -| `iamopen_accepted` | `message_id`, `source`, `pre`, `post` | IAmOpen selected peer and Joining committed | -| `iamopen_rejected` | `message_id`, `source`, `pre`, `post` | Validation or Opening/Open rejection | -| `timeout` | `pre`, `post` | Timeout transaction committed | -| `retry` | `pre`, `post` | Retry task observed | -| `send` | `send` in `class:destination` form, `message_id`, `pre`, `post` | Transport send observed | -| `open` | `open_kind`, `pre`, `post` | Service-open transition committed | -| `join_restart` | `pre`, `post` | Joining/restart side effect committed | -| `complete` | `pre`, `post` | Opening-to-Open completion committed | - -Receive events may use `caused_by` to identify the observed send. A referenced -send is checked for the expected sender, destination, and message class. If the -send was not logged, the validator may match it against a compatible hidden -send from a node that has already started. Message IDs, causal IDs, and source -names must be nonempty. Message IDs cannot be reused. Rejected events branch -over rejection at the explicit validation boundary and rejection by the -protocol state. A send ID may cause at most one receive event. - -Even without `caused_by`, an accepted receive from a configured source must -match a send that source could have emitted earlier. A rejected receive without -`caused_by` may represent rejection at the untrusted validation boundary. A -nonempty accepted source outside `expected_locations` is treated as an external -input, matching the current C++ behavior; the validator checks the local receive -transition but cannot constrain that external sender. - -Non-receive events must omit `caused_by`. +| Kind | Required event fields | Canonical boundary | +| ------------------ | ------------------------------------------------------------------- | ------------------------------------------- | +| `start` | `pre`, `post` | Protocol state initialized | +| `gossip_accepted` | `message_id`, `caused_by`, `source`, `view`, `seqno`, `pre`, `post` | Validated gossip callback committed | +| `vote_accepted` | `message_id`, `caused_by`, `source`, `pre`, `post` | Validated vote callback committed | +| `iamopen_accepted` | `message_id`, `caused_by`, `source`, `pre`, `post` | IAmOpen selected peer and Joining committed | +| `timeout` | `pre`, `post` | Timeout transaction committed | +| `send` | `send` in `class:destination` form, `message_id`, `pre`, `post` | Transport send observed | +| `open` | `open_kind`, `pre`, `post` | Service-open transition committed | +| `join_restart` | `pre`, `post` | Joining/restart side effect committed | +| `complete` | `pre`, `post` | Opening-to-Open completion committed | + +Every receive uses `caused_by` to identify an earlier `send`. The validator +checks the sender, destination, message class, and single consumption of that +send. Message IDs, causal IDs, and source names must be nonempty, and message +IDs cannot be reused. Non-receive events must omit `caused_by`. Each participating configured node has one `start` event at sequence zero. The -first creates the initial candidate system; later starts activate other -configured nodes without resetting candidates. Non-start events for a node +first creates the replay system; later starts activate other configured nodes +without resetting it. Non-start events for a node before its start are rejected. Configured but unavailable nodes may have no start event. Subsequent records must preserve `instance` and `expected_locations`, refer to a configured node, and increment that node's sequence exactly. Empty instance IDs, empty configurations, empty location names, and duplicate configured names are rejected. -The NDJSON record order must be a topological linearization of the distributed -trace. It must order a node's start and any observed transition enabling an -omitted send before the receive caused by that send. The collector must retain -this hidden happens-before edge while merging, even if it omits the send record -from the final trace. Per-node `sequence`, message causality, and these hidden -edges define the ordering; wall-clock timestamps do not. Unknown fields may -carry collector-specific merge metadata. - -## Under-observation - -An implementation trace need not expose every retry or network send. The -validator maintains compatible candidates containing a `SystemState` and the -send classes that could have been emitted by earlier hidden retries, plus -unobserved one-shot effects produced by committed transitions. This history -allows delayed delivery after a sender changes phase without merging -incompatible executions. `open`, `join_restart`, and `complete` each consume -one matching pending effect from the observed node, so one node cannot replay -its transition or consume another node's effect. -An explicit send may also match an earlier send capability: this models a -retry task that selected its work before a concurrent protocol commit and -dispatched it afterward. -The validator computes a finite hidden closure over retry/send stuttering -before and after each observation. It does not guess protocol-state changes. -`pre` and `post` filter all candidates. Message IDs constrain causal matching -but are not protocol state. - -The canonical v1 transition relation is deterministic, so a successful v1 -prefix currently retains one candidate. Candidate sets and separate histories -remain explicit so later under-observed refinements can introduce genuine -alternatives without changing the validation architecture. - -If no candidate remains, validation stops at the shortest failing prefix and -prints the observed incompatible kind plus expected compatible events from the -last nonempty candidate set. A successful parse with no compatible execution is -never reported as success. +The NDJSON record order is a topological linearization of the distributed +trace. Per-node `sequence` and `caused_by` edges define the ordering; wall-clock +timestamps do not. + +## Strict replay + +Version 1 is a complete successful-execution trace: every transport send, +accepted receive, committed timeout, and one-shot effect is explicit. +`Trace/Replay.lean` folds these events over one deterministic `SystemState`. +It retains only observed sends, consumed causal IDs, per-node sequences, and +pending `open`, `join_restart`, or `complete` effects. + +The validator rejects the first event that is not enabled by the canonical +model or whose recorded pre/post state, cause, or effect does not match. It +reports this shortest failing prefix with the current phase and expected event +classes. + +Rejected HTTP/validation inputs do not mutate the modeled state and are not +part of version 1. A future need to validate rejection behavior or incomplete +traces should use a new contract version rather than adding implicit behavior +to this deterministic replay. ## C++ instrumentation @@ -135,8 +105,8 @@ tasks. Transport sends are emitted immediately before dispatch and propagate their generated `message_id` in the internal request as `trace_message_id`; the committed receive records it as `caused_by`. If a retry observes a locally committed phase that is not yet globally visible -to the trace hook, tracing defers that retry invocation. The periodic task sends -on its next run after the phase event is emitted. +to the trace hook, tracing defers that retry invocation. Once phases match, the +trace lock serializes the complete send batch against later commit publication. Each log record contains `RDP_TRACE ` followed by the event object. `tests/infra/recovery_trace.py` extracts records from all participating node @@ -144,8 +114,9 @@ logs, topologically orders them by per-node sequence and causal send edges, writes NDJSON, and invokes the Lean validator. The quorum, failover, and multiple-timeout SNP e2e scenarios call this helper. -Validation/HTTP rejection events that perform no state mutation remain -optional; the current C++ instrumentation records successful committed paths. +The e2e helper additionally requires scenario-specific terminal evidence before +accepting the trace: the expected open kind, at least one completed opener, and +a `complete` or `join_restart` event for every participating node. ## Example diff --git a/lean/disaster-recovery/TraceMain.lean b/lean/disaster-recovery/TraceMain.lean index b00316203cbd..334889dd97cf 100644 --- a/lean/disaster-recovery/TraceMain.lean +++ b/lean/disaster-recovery/TraceMain.lean @@ -15,8 +15,8 @@ def main (args : List String) : IO UInt32 := do | .error failure => IO.eprintln (renderFailure failure) pure 1 - | .ok candidates => - IO.println s!"trace accepted: {events.length} events, {candidates} compatible final state(s)" + | .ok () => + IO.println s!"trace accepted: {events.length} events" pure 0 | _ => IO.eprintln "usage: trace-validator TRACE.ndjson" diff --git a/lean/disaster-recovery/TraceTests.lean b/lean/disaster-recovery/TraceTests.lean index 220edac7887a..33462aaab623 100644 --- a/lean/disaster-recovery/TraceTests.lean +++ b/lean/disaster-recovery/TraceTests.lean @@ -10,8 +10,7 @@ private def baseEvent (locations : List Location) (node : Location) (sequence : Nat) - (kind : DisasterRecovery.Protocol.Trace.Kind) : - DisasterRecovery.Protocol.Trace.TraceEvent := { + (kind : Kind) : TraceEvent := { version := contractVersion instanceId := "trace-tests" expectedLocations := locations @@ -30,392 +29,182 @@ private def baseEvent private def startEvent (locations : List Location) - (node : Location) : DisasterRecovery.Protocol.Trace.TraceEvent := { + (node : Location) : TraceEvent := { baseEvent locations node 0 .start with pre := some .gossiping post := some .gossiping } -private def validationFailedAt - (events : List DisasterRecovery.Protocol.Trace.TraceEvent) - (expectedPrefix : Nat) : Bool := +private def sendEvent + (locations : List Location) + (sequence : Nat) + (messageId description : String) + (phase : Phase) : TraceEvent := { + baseEvent locations "A" sequence .send with + messageId := some messageId + pre := some phase + post := some phase + send := some description +} + +private def gossipEvent + (locations : List Location) + (sequence : Nat) + (messageId cause : String) + (post : Phase) : TraceEvent := { + baseEvent locations "A" sequence .gossipAccepted with + messageId := some messageId + causedBy := some cause + source := some "A" + txid := some { view := 1, seqno := 1 } + pre := some .gossiping + post := some post +} + +private def voteEvent + (locations : List Location) + (sequence : Nat) + (messageId cause : String) + (post : Phase) : TraceEvent := { + baseEvent locations "A" sequence .voteAccepted with + messageId := some messageId + causedBy := some cause + source := some "A" + pre := some .voting + post := some post +} + +private def timeoutEvent + (locations : List Location) + (sequence : Nat) + (pre post : Phase) : TraceEvent := { + baseEvent locations "A" sequence .timeout with + pre := some pre + post := some post +} + +private def openEvent + (locations : List Location) + (sequence : Nat) + (kind : OpenKind) : TraceEvent := { + baseEvent locations "A" sequence .open with + pre := some .opening + post := some .opening + openKind := some kind +} + +private def completeEvent + (locations : List Location) + (sequence : Nat) : TraceEvent := { + baseEvent locations "A" sequence .complete with + pre := some .open + post := some .open +} + +private def validationSucceeds (events : List TraceEvent) : Bool := + match validate events with + | .ok () => true + | .error _ => false + +private def failedAt (events : List TraceEvent) (expectedPrefix : Nat) : Bool := match validate events with | .error failure => failure.prefixLength == expectedPrefix + | .ok () => false + +private def parseFails (value : String) : Bool := + match parseEvent value with + | .error _ => true | .ok _ => false -def main : IO UInt32 := do - let locations := ["A", "B"] - match validate [startEvent locations "A", startEvent locations "B"] with - | .ok 1 => pure () - | result => - throw (IO.userError s!"multi-node starts were not accepted: {repr result}") +private def quorumTrace : List TraceEvent := + let locations := ["A"] + [ + startEvent locations "A", + sendEvent locations 1 "send-gossip" "gossip:A" .gossiping, + gossipEvent locations 2 "receive-gossip" "send-gossip" .voting, + sendEvent locations 3 "send-vote" "vote:A" .voting, + sendEvent locations 4 "send-voting-gossip" "gossip:A" .voting, + voteEvent locations 5 "receive-vote" "send-vote" .opening, + openEvent locations 6 .quorum, + timeoutEvent locations 7 .opening .opening, + timeoutEvent locations 8 .opening .opening, + timeoutEvent locations 9 .opening .open, + completeEvent locations 10 + ] - match validate [startEvent locations "A"] with - | .ok 1 => pure () - | result => - throw (IO.userError s!"unavailable configured node was required to start: {repr result}") +def main : IO UInt32 := do + expect (validationSucceeds quorumTrace) "complete quorum trace was rejected" + let locations := ["A", "B"] expect - (validationFailedAt - [startEvent locations "A", startEvent locations "A"] 2) - "duplicate start was not rejected at the second event" - + (failedAt [startEvent locations "A", startEvent locations "B"] 2) + "incomplete multi-node trace was accepted" expect - (validationFailedAt - [startEvent ["A", "A"] "A"] 1) - "duplicate expected_locations were not rejected" - + (failedAt [startEvent locations "A"] 1) + "incomplete single-node trace was accepted" expect - (validationFailedAt - [{ startEvent ["A"] "A" with instanceId := "" }] 1) - "empty recovery instance was not rejected" - + (failedAt [startEvent locations "A", startEvent locations "A"] 2) + "duplicate start was accepted" + expect + (failedAt [startEvent ["A", "A"] "A"] 1) + "duplicate expected locations were accepted" expect - (validationFailedAt - [{ baseEvent ["A"] "A" 0 .start with post := some .gossiping }] 1) - "missing required start pre-state was not rejected" + (failedAt [{ startEvent ["A"] "A" with instanceId := "" }] 1) + "empty recovery instance was accepted" let single := ["A"] let start := startEvent single "A" - let retry := { - baseEvent single "A" 1 .retry with - pre := some .gossiping - post := some .gossiping - } - let send := { - baseEvent single "A" 2 .send with - messageId := some "send-1" - pre := some .gossiping - post := some .gossiping - send := some "gossip:A" - } - let wrongCause := { - baseEvent single "A" 3 .voteAccepted with - messageId := some "receive-1" - causedBy := some "send-1" - source := some "A" - pre := some .gossiping - post := some .gossiping + let gossipSend := sendEvent single 1 "send-gossip" "gossip:A" .gossiping + let missingCause := { + gossipEvent single 2 "receive-gossip" "missing" .voting with + causedBy := none } expect - (validationFailedAt [start, retry, send, wrongCause] 4) - "caused_by accepted a send with the wrong message class" + (failedAt [start, gossipSend, missingCause] 3) + "receive without caused_by was accepted" - let duplicateId := { - baseEvent single "A" 3 .gossipAccepted with - messageId := some "send-1" - source := some "A" - txid := some { view := 1, seqno := 1 } + let wrongClass := { + voteEvent single 2 "receive-vote" "send-gossip" .voting with pre := some .gossiping - post := some .voting } expect - (validationFailedAt [start, retry, send, duplicateId] 4) - "duplicate message_id was not rejected" + (failedAt [start, gossipSend, wrongClass] 3) + "vote consumed a gossip send" - let received := { - baseEvent single "A" 3 .gossipAccepted with - messageId := some "receive-1" - causedBy := some "send-1" - source := some "A" - txid := some { view := 1, seqno := 1 } - pre := some .gossiping - post := some .voting - } + let received := gossipEvent single 2 "receive-gossip" "send-gossip" .voting let reusedCause := { - baseEvent single "A" 4 .gossipRejected with - messageId := some "receive-2" - causedBy := some "send-1" - source := some "A" - txid := some { view := 1, seqno := 1 } - pre := some .voting - post := some .voting - } - expect - (validationFailedAt [start, retry, send, received, reusedCause] 5) - "a send was accepted as the cause of multiple receives" - - let openingVote := { - baseEvent single "A" 4 .voteAccepted with - messageId := some "opening-vote" - source := some "A" - pre := some .voting - post := some .opening - } - let openedOnce := { - baseEvent single "A" 5 .open with - pre := some .opening - post := some .opening - openKind := some .quorum - } - let openedTwice := { - openedOnce with sequence := 6 - } - expect - (validationFailedAt - [start, retry, send, received, openingVote, openedOnce, openedTwice] 7) - "one opening transition produced multiple committed open observations" - - let votingGossip := { - baseEvent single "A" 1 .gossipAccepted with - messageId := some "voting-gossip" - source := some "A" - txid := some { view := 1, seqno := 1 } - pre := some .gossiping - post := some .voting - } - let staleOpeningVote := { - baseEvent single "A" 2 .voteAccepted with - messageId := some "voting-vote" - source := some "A" - pre := some .voting - post := some .opening - } - let delayedVoteSend := { - baseEvent single "A" 3 .send with - messageId := some "delayed-vote-send" - pre := some .opening - post := some .opening - send := some "vote:A" - } - match validate [start, votingGossip, staleOpeningVote, delayedVoteSend] with - | .ok 1 => pure () - | result => - throw (IO.userError s!"stale retry send was rejected: {repr result}") - - let hiddenReceive := { - baseEvent single "A" 1 .gossipAccepted with - messageId := some "receive-hidden" - causedBy := some "hidden-send" - source := some "A" - txid := some { view := 1, seqno := 1 } - pre := some .gossiping - post := some .voting - } - let lateHiddenSend := { - baseEvent single "A" 2 .send with - messageId := some "hidden-send" + voteEvent single 3 "receive-vote" "send-gossip" .voting with pre := some .voting - post := some .voting - send := some "gossip:A" } expect - (validationFailedAt [start, hiddenReceive, lateHiddenSend] 3) - "a hidden causal send ID was accepted later in the trace" + (failedAt [start, gossipSend, received, reusedCause] 4) + "one send caused multiple receives" - let selfCaused := { - hiddenReceive with - messageId := some "same-id" - causedBy := some "same-id" - } + let badSend := sendEvent single 1 "send-vote" "vote:A" .gossiping expect - (validationFailedAt [start, selfCaused] 2) - "one observation was accepted as both a send and its receive" + (failedAt [start, badSend] 2) + "Voting send was accepted while Gossiping" - let abortedTimeout := { - baseEvent single "A" 1 .timeout with - pre := some .gossiping - post := some .gossiping - } + let abortedTimeout := timeoutEvent single 1 .gossiping .gossiping expect - (validationFailedAt [start, abortedTimeout] 2) - "an aborted empty-gossip timeout was accepted as committed" + (failedAt [start, abortedTimeout] 2) + "aborted empty-gossip timeout was accepted" - let beforeSourceStart := { - baseEvent locations "A" 1 .gossipAccepted with - messageId := some "receive-before-start" - causedBy := some "hidden-before-start" - source := some "B" - txid := some { view := 1, seqno := 1 } - pre := some .gossiping - post := some .gossiping - } + let throughOpen := quorumTrace.take 7 expect - (validationFailedAt - [startEvent locations "A", beforeSourceStart, startEvent locations "B"] 2) - "a hidden send originated before its source node started" - - let impossibleVote := { - baseEvent locations "A" 1 .voteAccepted with - messageId := some "impossible-vote" - source := some "B" - pre := some .gossiping - post := some .gossiping - } - expect - (validationFailedAt - [startEvent locations "A", startEvent locations "B", impossibleVote] 3) - "an accepted receive bypassed hidden-send feasibility" - - let externalVote := { - baseEvent single "A" 1 .voteAccepted with - messageId := some "external-vote" - source := some "OUTSIDE" - pre := some .gossiping - post := some .gossiping - } - match validate [start, externalVote] with - | .ok 1 => pure () - | result => - throw (IO.userError s!"external accepted input was rejected: {repr result}") - - let emptySource := { - baseEvent single "A" 1 .gossipAccepted with - messageId := some "empty-source" - source := some "" - txid := some { view := 1, seqno := 1 } - pre := some .gossiping - post := some .voting - } - expect - (validationFailedAt [start, emptySource] 2) - "an empty receive source was accepted" - - let aGossipA := { - baseEvent locations "A" 1 .gossipAccepted with - messageId := some "a-gossip-a" - source := some "A" - txid := some { view := 2, seqno := 1 } - pre := some .gossiping - post := some .gossiping - } - let aGossipB := { - baseEvent locations "A" 2 .gossipAccepted with - messageId := some "a-gossip-b" - source := some "B" - txid := some { view := 1, seqno := 1 } - pre := some .gossiping - post := some .voting - } - let bGossipA := { - baseEvent locations "B" 1 .gossipAccepted with - messageId := some "b-gossip-a" - source := some "A" - txid := some { view := 2, seqno := 1 } - pre := some .gossiping - post := some .gossiping - } - let bGossipB := { - baseEvent locations "B" 2 .gossipAccepted with - messageId := some "b-gossip-b" - source := some "B" - txid := some { view := 1, seqno := 1 } - pre := some .gossiping - post := some .voting - } - let aVoteA := { - baseEvent locations "A" 3 .voteAccepted with - messageId := some "a-vote-a" - source := some "A" - pre := some .voting - post := some .voting - } - let aVoteB := { - baseEvent locations "A" 4 .voteAccepted with - messageId := some "a-vote-b" - source := some "B" - pre := some .voting - post := some .opening - } - let delayedGossip := { - baseEvent locations "B" 3 .gossipRejected with - messageId := some "b-delayed-gossip" - causedBy := some "hidden-delayed-gossip" - source := some "A" - txid := some { view := 2, seqno := 1 } - pre := some .voting - post := some .voting - } - let delayedTrace := [ - startEvent locations "A", - startEvent locations "B", - aGossipA, - aGossipB, - bGossipA, - bGossipB, - aVoteA, - aVoteB, - delayedGossip - ] - match validate delayedTrace with - | .ok 1 => pure () - | result => - throw (IO.userError s!"delayed hidden send was rejected: {repr result}") - - let failoverGossipA := { - baseEvent locations "A" 1 .gossipAccepted with - messageId := some "failover-gossip-a" - source := some "A" - txid := some { view := 1, seqno := 1 } - pre := some .gossiping - post := some .gossiping - } - let failoverTimeoutA1 := { - baseEvent locations "A" 2 .timeout with - pre := some .gossiping - post := some .voting - } - let failoverVoteA := { - baseEvent locations "A" 3 .voteAccepted with - messageId := some "failover-vote-a" - source := some "A" - pre := some .voting - post := some .voting - } - let failoverTimeoutA2 := { - baseEvent locations "A" 4 .timeout with - pre := some .voting - post := some .opening - } - let failoverGossipB := { - baseEvent locations "B" 1 .gossipAccepted with - messageId := some "failover-gossip-b" - source := some "B" - txid := some { view := 1, seqno := 1 } - pre := some .gossiping - post := some .gossiping - } - let failoverTimeoutB1 := { - baseEvent locations "B" 2 .timeout with - pre := some .gossiping - post := some .voting - } - let failoverVoteB := { - baseEvent locations "B" 3 .voteAccepted with - messageId := some "failover-vote-b" - source := some "B" - pre := some .voting - post := some .voting - } - let failoverTimeoutB2 := { - baseEvent locations "B" 4 .timeout with - pre := some .voting - post := some .opening - } - let openA := { - baseEvent locations "A" 5 .open with - pre := some .opening - post := some .opening - openKind := some .failover - } - let duplicateOpenA := { openA with sequence := 6 } - let twoFailovers := [ - startEvent locations "A", - startEvent locations "B", - failoverGossipA, - failoverTimeoutA1, - failoverVoteA, - failoverTimeoutA2, - failoverGossipB, - failoverTimeoutB1, - failoverVoteB, - failoverTimeoutB2, - openA, - duplicateOpenA - ] + (failedAt (throughOpen ++ [openEvent single 7 .quorum]) 8) + "one opening transition produced multiple open observations" expect - (validationFailedAt twoFailovers 12) - "node A consumed node B's pending open effect" - - IO.println "all trace contract checks passed" + (failedAt (quorumTrace.take 6) 6) + "trace with an unobserved opening effect was accepted" + + let rejectedJson := + "{\"version\":\"ccf.recovery_decision_protocol.trace/1\"," + ++ "\"instance\":\"x\",\"expected_locations\":[\"A\"]," + ++ "\"node\":\"A\",\"sequence\":0,\"kind\":\"gossip_rejected\"," + ++ "\"pre\":\"GOSSIPING\",\"post\":\"GOSSIPING\"}" + expect (parseFails rejectedJson) + "unused rejection event remains in the strict v1 format" + + IO.println "all strict trace replay checks passed" pure 0 diff --git a/lean/disaster-recovery/fixtures/accepted-failover.ndjson b/lean/disaster-recovery/fixtures/accepted-failover.ndjson index 0a1ff06789e5..87210e273c77 100644 --- a/lean/disaster-recovery/fixtures/accepted-failover.ndjson +++ b/lean/disaster-recovery/fixtures/accepted-failover.ndjson @@ -1,8 +1,15 @@ {"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":1,"kind":"gossip_accepted","message_id":"gossip-a","source":"A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":2,"kind":"timeout","pre":"GOSSIPING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":3,"kind":"vote_accepted","message_id":"vote-a","source":"A","pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":4,"kind":"timeout","pre":"VOTING","post":"OPENING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":5,"kind":"open","open_kind":"FAILOVER","pre":"OPENING","post":"OPENING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":6,"kind":"timeout","pre":"OPENING","post":"OPEN"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":7,"kind":"complete","pre":"OPEN","post":"OPEN"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":1,"kind":"send","message_id":"send-gossip-a","send":"gossip:A","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":2,"kind":"send","message_id":"send-gossip-b","send":"gossip:B","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":3,"kind":"send","message_id":"send-gossip-c","send":"gossip:C","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":4,"kind":"gossip_accepted","message_id":"gossip-a","caused_by":"send-gossip-a","source":"A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":5,"kind":"timeout","pre":"GOSSIPING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":6,"kind":"send","message_id":"send-vote-a","send":"vote:A","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":7,"kind":"send","message_id":"send-voting-gossip-a","send":"gossip:A","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":8,"kind":"send","message_id":"send-voting-gossip-b","send":"gossip:B","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":9,"kind":"send","message_id":"send-voting-gossip-c","send":"gossip:C","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":10,"kind":"vote_accepted","message_id":"vote-a","caused_by":"send-vote-a","source":"A","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":11,"kind":"timeout","pre":"VOTING","post":"OPENING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":12,"kind":"open","open_kind":"FAILOVER","pre":"OPENING","post":"OPENING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":13,"kind":"timeout","pre":"OPENING","post":"OPEN"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":14,"kind":"complete","pre":"OPEN","post":"OPEN"} diff --git a/lean/disaster-recovery/fixtures/accepted-multinode.ndjson b/lean/disaster-recovery/fixtures/accepted-multinode.ndjson index 93b58b6dd448..69df47f1fec5 100644 --- a/lean/disaster-recovery/fixtures/accepted-multinode.ndjson +++ b/lean/disaster-recovery/fixtures/accepted-multinode.ndjson @@ -1,5 +1,26 @@ {"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":1,"kind":"retry","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":2,"kind":"send","message_id":"send-a-b","send":"gossip:B","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":1,"kind":"gossip_accepted","message_id":"receive-a-b","caused_by":"send-a-b","source":"A","view":1,"seqno":10,"pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":1,"kind":"send","message_id":"a-gossip-a","send":"gossip:A","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":2,"kind":"send","message_id":"a-gossip-b","send":"gossip:B","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":1,"kind":"send","message_id":"b-gossip-a","send":"gossip:A","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":2,"kind":"send","message_id":"b-gossip-b","send":"gossip:B","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":3,"kind":"gossip_accepted","message_id":"a-receive-a","caused_by":"a-gossip-a","source":"A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":4,"kind":"gossip_accepted","message_id":"a-receive-b","caused_by":"b-gossip-a","source":"B","view":2,"seqno":1,"pre":"GOSSIPING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":3,"kind":"gossip_accepted","message_id":"b-receive-a","caused_by":"a-gossip-b","source":"A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":4,"kind":"gossip_accepted","message_id":"b-receive-b","caused_by":"b-gossip-b","source":"B","view":2,"seqno":1,"pre":"GOSSIPING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":5,"kind":"send","message_id":"a-vote-b","send":"vote:B","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":6,"kind":"send","message_id":"a-voting-gossip-a","send":"gossip:A","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":7,"kind":"send","message_id":"a-voting-gossip-b","send":"gossip:B","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":5,"kind":"send","message_id":"b-vote-b","send":"vote:B","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":6,"kind":"send","message_id":"b-voting-gossip-a","send":"gossip:A","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":7,"kind":"send","message_id":"b-voting-gossip-b","send":"gossip:B","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":8,"kind":"vote_accepted","message_id":"b-receive-vote-a","caused_by":"a-vote-b","source":"A","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":9,"kind":"vote_accepted","message_id":"b-receive-vote-b","caused_by":"b-vote-b","source":"B","pre":"VOTING","post":"OPENING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":10,"kind":"open","open_kind":"QUORUM","pre":"OPENING","post":"OPENING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":11,"kind":"send","message_id":"b-open-a","send":"iamopen:A","pre":"OPENING","post":"OPENING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":8,"kind":"iamopen_accepted","message_id":"a-receive-open","caused_by":"b-open-a","source":"B","pre":"VOTING","post":"JOINING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":9,"kind":"join_restart","pre":"JOINING","post":"JOINING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":12,"kind":"timeout","pre":"OPENING","post":"OPENING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":13,"kind":"timeout","pre":"OPENING","post":"OPENING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":14,"kind":"timeout","pre":"OPENING","post":"OPEN"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":15,"kind":"complete","pre":"OPEN","post":"OPEN"} diff --git a/lean/disaster-recovery/fixtures/accepted.ndjson b/lean/disaster-recovery/fixtures/accepted.ndjson index c53d64c21454..b6970a90a56c 100644 --- a/lean/disaster-recovery/fixtures/accepted.ndjson +++ b/lean/disaster-recovery/fixtures/accepted.ndjson @@ -1,10 +1,11 @@ {"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":1,"kind":"retry","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":2,"kind":"send","message_id":"send-gossip-1","send":"gossip:A","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":3,"kind":"gossip_accepted","message_id":"recv-gossip-1","caused_by":"send-gossip-1","source":"A","view":1,"seqno":10,"pre":"GOSSIPING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":4,"kind":"vote_accepted","message_id":"recv-vote-1","source":"A","pre":"VOTING","post":"OPENING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":5,"kind":"open","open_kind":"QUORUM","pre":"OPENING","post":"OPENING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":6,"kind":"timeout","pre":"OPENING","post":"OPENING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":1,"kind":"send","message_id":"send-gossip-1","send":"gossip:A","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":2,"kind":"gossip_accepted","message_id":"recv-gossip-1","caused_by":"send-gossip-1","source":"A","view":1,"seqno":10,"pre":"GOSSIPING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":3,"kind":"send","message_id":"send-vote-1","send":"vote:A","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":4,"kind":"send","message_id":"send-gossip-2","send":"gossip:A","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":5,"kind":"vote_accepted","message_id":"recv-vote-1","caused_by":"send-vote-1","source":"A","pre":"VOTING","post":"OPENING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":6,"kind":"open","open_kind":"QUORUM","pre":"OPENING","post":"OPENING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":7,"kind":"timeout","pre":"OPENING","post":"OPENING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":8,"kind":"timeout","pre":"OPENING","post":"OPEN"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":9,"kind":"complete","pre":"OPEN","post":"OPEN"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":8,"kind":"timeout","pre":"OPENING","post":"OPENING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":9,"kind":"timeout","pre":"OPENING","post":"OPEN"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":10,"kind":"complete","pre":"OPEN","post":"OPEN"} diff --git a/lean/disaster-recovery/fixtures/rejected-cause.ndjson b/lean/disaster-recovery/fixtures/rejected-cause.ndjson index 07a2c30eecf9..cc0332d2795b 100644 --- a/lean/disaster-recovery/fixtures/rejected-cause.ndjson +++ b/lean/disaster-recovery/fixtures/rejected-cause.ndjson @@ -1,4 +1,3 @@ {"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-cause","expected_locations":["A"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-cause","expected_locations":["A"],"node":"A","sequence":1,"kind":"retry","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-cause","expected_locations":["A"],"node":"A","sequence":2,"kind":"send","message_id":"send-gossip","send":"gossip:A","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-cause","expected_locations":["A"],"node":"A","sequence":3,"kind":"vote_accepted","message_id":"receive-vote","caused_by":"send-gossip","source":"A","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-cause","expected_locations":["A"],"node":"A","sequence":1,"kind":"send","message_id":"send-gossip","send":"gossip:A","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-cause","expected_locations":["A"],"node":"A","sequence":2,"kind":"vote_accepted","message_id":"receive-vote","caused_by":"send-gossip","source":"A","pre":"GOSSIPING","post":"GOSSIPING"} diff --git a/src/node/recovery_decision_protocol.cpp b/src/node/recovery_decision_protocol.cpp index a4b87b0ee829..ff03eaa36ab5 100644 --- a/src/node/recovery_decision_protocol.cpp +++ b/src/node/recovery_decision_protocol.cpp @@ -118,6 +118,12 @@ namespace ccf recovery_decision_protocol::TraceEvent event) { std::lock_guard guard(trace_lock); + emit_trace_event_unsafe(std::move(event)); + } + + void RecoveryDecisionProtocolSubsystem::emit_trace_event_unsafe( + recovery_decision_protocol::TraceEvent event) + { if (event.kind == "send") { event.pre = trace_committed_state; @@ -152,15 +158,13 @@ namespace ccf std::string RecoveryDecisionProtocolSubsystem::new_trace_message_id() { std::lock_guard guard(trace_lock); - return fmt::format( - "{}:{}:{}", trace_instance_id, trace_node, next_trace_message_number++); + return new_trace_message_id_unsafe(); } - bool RecoveryDecisionProtocolSubsystem::is_trace_state_committed( - recovery_decision_protocol::StateMachine state) + std::string RecoveryDecisionProtocolSubsystem::new_trace_message_id_unsafe() { - std::lock_guard guard(trace_lock); - return trace_committed_state == trace_state_name(state); + return fmt::format( + "{}:{}:{}", trace_instance_id, trace_node, next_trace_message_number++); } recovery_decision_protocol::StateMachine RecoveryDecisionProtocolSubsystem:: @@ -177,10 +181,10 @@ namespace ccf return state.value(); } - void RecoveryDecisionProtocolSubsystem::emit_trace_send( + void RecoveryDecisionProtocolSubsystem::emit_trace_send_unsafe( const std::string& message_id, const std::string& description) { - emit_trace_event({ + emit_trace_event_unsafe({ .kind = "send", .message_id = message_id, .pre = "", @@ -599,13 +603,6 @@ namespace ccf } auto& sm_state = sm_state_opt.value(); -#ifdef CCF_RECOVERY_TRACE - if (!is_trace_state_committed(sm_state)) - { - return; - } -#endif - // Stop if recovery-decision-protocol is complete if (sm_state == recovery_decision_protocol::StateMachine::OPEN) { @@ -615,10 +612,21 @@ namespace ccf return; } + std::optional + gossip_request = std::nullopt; + std::optional + vote_request = std::nullopt; + std::optional chosen_node_info = + std::nullopt; + std::optional + iamopen_request = std::nullopt; + switch (sm_state) { case recovery_decision_protocol::StateMachine::GOSSIPING: - send_gossip_unsafe(tx); + gossip_request = recovery_decision_protocol::GossipRequest{}; + gossip_request->info = get_node_info(tx); + gossip_request->txid = get_last_recovered_signed_txid(); break; case recovery_decision_protocol::StateMachine::VOTING: { @@ -633,7 +641,7 @@ namespace ccf throw std::logic_error( "Recovery-decision-protocol chosen node not set, cannot vote"); } - auto chosen_node_info = + chosen_node_info = node_info_handle->get(chosen_replica_handle->get().value()); if (!chosen_node_info.has_value()) { @@ -641,13 +649,15 @@ namespace ccf "Recovery-decision-protocol chosen node {} not found", chosen_replica_handle->get().value())); } - send_vote_unsafe(tx, chosen_node_info.value()); - // keep gossiping to allow lagging nodes to eventually vote - send_gossip_unsafe(tx); + vote_request = recovery_decision_protocol::TaggedWithNodeInfo{ + .info = get_node_info(tx)}; + gossip_request = recovery_decision_protocol::GossipRequest{}; + gossip_request->info = vote_request->info; + gossip_request->txid = get_last_recovered_signed_txid(); break; } case recovery_decision_protocol::StateMachine::OPENING: - send_iamopen_unsafe(tx); + iamopen_request = get_iamopen_request(tx); break; case recovery_decision_protocol::StateMachine::JOINING: case recovery_decision_protocol::StateMachine::OPEN: @@ -658,6 +668,47 @@ namespace ccf "Unknown recovery-decision-protocol state: {}", static_cast(sm_state))); } + + const auto self_signed_node_cert = + node_state->get_self_signed_certificate(); + const auto node_private_key = + node_state->node_sign_kp->private_key_pem(); + +#ifdef CCF_RECOVERY_TRACE + std::lock_guard trace_guard(trace_lock); + if (trace_committed_state != trace_state_name(sm_state)) + { + return; + } +#endif + + switch (sm_state) + { + case recovery_decision_protocol::StateMachine::GOSSIPING: + send_gossip_unsafe( + gossip_request.value(), self_signed_node_cert, node_private_key); + break; + case recovery_decision_protocol::StateMachine::VOTING: + send_vote_unsafe( + vote_request.value(), + chosen_node_info.value(), + self_signed_node_cert, + node_private_key); + // Keep gossiping to allow lagging nodes to eventually vote. + send_gossip_unsafe( + gossip_request.value(), self_signed_node_cert, node_private_key); + break; + case recovery_decision_protocol::StateMachine::OPENING: + send_iamopen_unsafe( + iamopen_request.value(), self_signed_node_cert, node_private_key); + break; + case recovery_decision_protocol::StateMachine::JOINING: + case recovery_decision_protocol::StateMachine::OPEN: + default: + throw std::logic_error(fmt::format( + "Unexpected prepared recovery-decision-protocol state: {}", + static_cast(sm_state))); + } }, "RecoveryDecisionProtocolRetry"); @@ -863,27 +914,24 @@ namespace ccf return node_info_cache.value(); } - void RecoveryDecisionProtocolSubsystem::send_gossip_unsafe(kv::ReadOnlyTx& tx) + void RecoveryDecisionProtocolSubsystem::send_gossip_unsafe( + recovery_decision_protocol::GossipRequest request, + const crypto::Pem& self_signed_node_cert, + const crypto::Pem& node_private_key) { auto& config = get_config(); LOG_TRACE_FMT("Broadcasting recovery-decision-protocol gossip"); - recovery_decision_protocol::GossipRequest request; - request.info = get_node_info(tx); - request.txid = get_last_recovered_signed_txid(); - const auto self_signed_node_cert = - node_state->get_self_signed_certificate(); - const auto node_private_key = node_state->node_sign_kp->private_key_pem(); for (auto& target : config.expected_locations) { auto target_address = target.address; #ifdef CCF_RECOVERY_TRACE - request.trace_message_id = new_trace_message_id(); + request.trace_message_id = new_trace_message_id_unsafe(); #endif nlohmann::json request_json = request; #ifdef CCF_RECOVERY_TRACE - emit_trace_send( + emit_trace_send_unsafe( request.trace_message_id.value(), fmt::format("gossip:{}", target.name)); #endif @@ -897,24 +945,22 @@ namespace ccf } void RecoveryDecisionProtocolSubsystem::send_vote_unsafe( - kv::ReadOnlyTx& tx, const recovery_decision_protocol::NodeInfo& node_info) + recovery_decision_protocol::TaggedWithNodeInfo request, + const recovery_decision_protocol::NodeInfo& node_info, + const crypto::Pem& self_signed_node_cert, + const crypto::Pem& node_private_key) { LOG_TRACE_FMT( "Sending recovery-decision-protocol vote to {} at {}", node_info.location.name, node_info.location.address); - recovery_decision_protocol::TaggedWithNodeInfo request{ - .info = get_node_info(tx)}; - const auto self_signed_node_cert = - node_state->get_self_signed_certificate(); - #ifdef CCF_RECOVERY_TRACE - request.trace_message_id = new_trace_message_id(); + request.trace_message_id = new_trace_message_id_unsafe(); #endif nlohmann::json request_json = request; #ifdef CCF_RECOVERY_TRACE - emit_trace_send( + emit_trace_send_unsafe( request.trace_message_id.value(), fmt::format("vote:{}", node_info.location.name)); #endif @@ -923,7 +969,7 @@ namespace ccf node_info.location.address, "vote", self_signed_node_cert, - node_state->node_sign_kp->private_key_pem()); + node_private_key); } recovery_decision_protocol::IAmOpenRequest& @@ -964,17 +1010,14 @@ namespace ccf } void RecoveryDecisionProtocolSubsystem::send_iamopen_unsafe( - ccf::kv::ReadOnlyTx& tx) + recovery_decision_protocol::IAmOpenRequest request, + const crypto::Pem& self_signed_node_cert, + const crypto::Pem& node_private_key) { auto& config = get_config(); auto& location = get_location(); LOG_TRACE_FMT("Sending recovery-decision-protocol iamopen"); - - auto request = get_iamopen_request(tx); - const auto self_signed_node_cert = - node_state->get_self_signed_certificate(); - const auto node_private_key = node_state->node_sign_kp->private_key_pem(); for (auto& target : config.expected_locations) { if (target.name == location.name) @@ -983,11 +1026,11 @@ namespace ccf continue; } #ifdef CCF_RECOVERY_TRACE - request.trace_message_id = new_trace_message_id(); + request.trace_message_id = new_trace_message_id_unsafe(); #endif nlohmann::json request_json = request; #ifdef CCF_RECOVERY_TRACE - emit_trace_send( + emit_trace_send_unsafe( request.trace_message_id.value(), fmt::format("iamopen:{}", target.name)); #endif diff --git a/src/node/recovery_decision_protocol.h b/src/node/recovery_decision_protocol.h index a0ec1d49d0cc..fd204735553d 100644 --- a/src/node/recovery_decision_protocol.h +++ b/src/node/recovery_decision_protocol.h @@ -110,11 +110,19 @@ namespace ccf // Steady state operations recovery_decision_protocol::RequestNodeInfo& get_node_info( kv::ReadOnlyTx& tx); - void send_gossip_unsafe(kv::ReadOnlyTx& tx); + void send_gossip_unsafe( + recovery_decision_protocol::GossipRequest request, + const crypto::Pem& self_signed_node_cert, + const crypto::Pem& node_private_key); void send_vote_unsafe( - kv::ReadOnlyTx& tx, - const recovery_decision_protocol::NodeInfo& node_info); - void send_iamopen_unsafe(kv::ReadOnlyTx& tx); + recovery_decision_protocol::TaggedWithNodeInfo request, + const recovery_decision_protocol::NodeInfo& node_info, + const crypto::Pem& self_signed_node_cert, + const crypto::Pem& node_private_key); + void send_iamopen_unsafe( + recovery_decision_protocol::IAmOpenRequest request, + const crypto::Pem& self_signed_node_cert, + const crypto::Pem& node_private_key); RecoveryDecisionProtocolConfig& get_config(); sealing_recovery::Location& get_location(); @@ -129,10 +137,10 @@ namespace ccf recovery_decision_protocol::StateMachine pre, recovery_decision_protocol::StateMachine post); void emit_trace_event(recovery_decision_protocol::TraceEvent event); + void emit_trace_event_unsafe(recovery_decision_protocol::TraceEvent event); std::string new_trace_message_id(); - bool is_trace_state_committed( - recovery_decision_protocol::StateMachine state); - void emit_trace_send( + std::string new_trace_message_id_unsafe(); + void emit_trace_send_unsafe( const std::string& message_id, const std::string& description); #endif }; diff --git a/src/node/rpc/self_healing_open_handlers.h b/src/node/rpc/self_healing_open_handlers.h index a2057864694e..6cbf1be6d578 100644 --- a/src/node/rpc/self_healing_open_handlers.h +++ b/src/node/rpc/self_healing_open_handlers.h @@ -57,6 +57,16 @@ namespace ccf::node auto in = params.get(); recovery_decision_protocol::RequestNodeInfo info = in.info; +#ifdef CCF_RECOVERY_TRACE + if (!in.trace_message_id.has_value()) + { + return make_error( + HTTP_STATUS_BAD_REQUEST, + ccf::errors::InvalidInput, + "Recovery trace message ID is required in trace-enabled builds"); + } +#endif + // ---- Validate the quote against our store and store the node info ---- auto cert_der = ccf::crypto::public_key_der_from_cert( diff --git a/tests/infra/recovery_trace_test.py b/tests/infra/recovery_trace_test.py index 5069a8b30171..46b52ed00b46 100644 --- a/tests/infra/recovery_trace_test.py +++ b/tests/infra/recovery_trace_test.py @@ -53,29 +53,17 @@ def test_extract_linearize_and_validate(self): root = pathlib.Path(directory) a_log = root / "a.out" b_log = root / "b.out" - a_events = [ - event("A", 0, "start"), - event( - "A", - 1, - "send", - message_id="send-a-b", - send="gossip:B", - ), - ] - b_events = [ - event("B", 0, "start"), - event( - "B", - 1, - "gossip_accepted", - message_id="receive-a-b", - caused_by="send-a-b", - source="A", - view=1, - seqno=1, - ), - ] + fixture = ( + pathlib.Path(__file__).resolve().parents[2] + / "lean" + / "disaster-recovery" + / "fixtures" + / "accepted-multinode.ndjson" + ) + with open(fixture, encoding="utf-8") as trace: + events = [json.loads(line) for line in trace if line.strip()] + a_events = [item for item in events if item["node"] == "A"] + b_events = [item for item in events if item["node"] == "B"] a_log.write_text( "".join( f"[info] RDP_TRACE {json.dumps(trace_event)}\n" @@ -97,10 +85,14 @@ def test_extract_linearize_and_validate(self): extracted = infra.recovery_trace.extract_events(network.nodes) ordered = infra.recovery_trace.linearize(extracted) - self.assertEqual( - [(item["node"], item["sequence"]) for item in ordered], - [("A", 0), ("A", 1), ("B", 0), ("B", 1)], - ) + positions = { + item["message_id"]: index + for index, item in enumerate(ordered) + if "message_id" in item + } + for index, item in enumerate(ordered): + if "caused_by" in item: + self.assertLess(positions[item["caused_by"]], index) trace_path = infra.recovery_trace.validate_recovery_trace( network, "synthetic" From 5d6828fcff701c8ab79317bbd3edfb934dab1539 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Sun, 30 Aug 2026 20:43:52 +0100 Subject: [PATCH 08/14] Tighten strict recovery traces Validate commands before enumeration, trigger CI for merger tests, bind gossip payloads to causal sends, and enforce atomic ordered retry batches. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 46d56f8a-935f-4551-abe3-84bd4f951865 --- .github/workflows/lean-shallow.yml | 1 + .../Protocol/Trace/Replay.lean | 7 ++++ lean/disaster-recovery/Main.lean | 3 +- lean/disaster-recovery/README.md | 18 ++++++----- lean/disaster-recovery/TRACE_FORMAT_V1.md | 32 ++++++++++--------- lean/disaster-recovery/TraceTests.lean | 17 ++++++++++ .../fixtures/accepted-failover.ndjson | 12 +++---- .../fixtures/accepted-multinode.ndjson | 16 +++++----- .../fixtures/accepted.ndjson | 4 +-- .../fixtures/rejected-cause.ndjson | 2 +- .../fixtures/rejected.ndjson | 5 +-- src/node/recovery_decision_protocol.cpp | 17 +++++++--- src/node/recovery_decision_protocol.h | 4 ++- 13 files changed, 90 insertions(+), 48 deletions(-) diff --git a/.github/workflows/lean-shallow.yml b/.github/workflows/lean-shallow.yml index fbb84404151e..80764b8b4c9f 100644 --- a/.github/workflows/lean-shallow.yml +++ b/.github/workflows/lean-shallow.yml @@ -12,6 +12,7 @@ on: - "src/node/rpc/self_healing_open_handlers.h" - "tests/e2e_operations.py" - "tests/infra/recovery_trace.py" + - "tests/infra/recovery_trace_test.py" - "CMakeLists.txt" - ".github/workflows/ci.yml" - ".github/workflows/lean-shallow.yml" diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/Trace/Replay.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/Trace/Replay.lean index 9921d09f6c76..8afb4dbf362f 100644 --- a/lean/disaster-recovery/DisasterRecovery/Protocol/Trace/Replay.lean +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/Trace/Replay.lean @@ -12,6 +12,7 @@ structure ObservedSend where messageId : String source : Location description : String + txid : Option TxID deriving Repr, BEq structure PendingEffect where @@ -63,6 +64,9 @@ private def shapeError (event : TraceEvent) : Option String := else if event.kind == .send && (event.messageId.isNone || event.send.isNone) then some "message_id and send are required for sends" + else if event.kind == .send && + (event.send.getD "").startsWith "gossip:" && event.txid.isNone then + some "view and seqno are required for gossip sends" else if event.kind == .open && event.openKind.isNone then some "open_kind is required for open" else if !isReceive event.kind && event.causedBy.isSome then @@ -187,6 +191,8 @@ private def consumeCause let description := receiveDescription event |>.getD "" if send.source != source || send.description != description then throw s!"caused_by '{cause}' has the wrong source, class, or destination" + if event.kind == .gossipAccepted && send.txid != event.txid then + throw s!"caused_by '{cause}' has the wrong gossip TxID" pure { active with consumedSendIds := cause :: active.consumedSendIds @@ -246,6 +252,7 @@ private def applySend messageId := event.messageId.getD "" source := event.node description + txid := event.txid } :: active.sends pendingSendBatches := setPendingSendBatch event.node batch.tail active.pendingSendBatches diff --git a/lean/disaster-recovery/Main.lean b/lean/disaster-recovery/Main.lean index b1fc46eceefe..6372a8eb0875 100644 --- a/lean/disaster-recovery/Main.lean +++ b/lean/disaster-recovery/Main.lean @@ -21,12 +21,13 @@ def main (args : List String) : IO UInt32 := do IO.eprintln message pure 2 | .ok n => - let graph <- enumerate n match command with | "export" => + let graph <- enumerate n exportGraph n graph pure 0 | "check" => + let graph <- enumerate n if <- checkGraph n graph then pure 0 else pure 1 | _ => IO.eprintln usage diff --git a/lean/disaster-recovery/README.md b/lean/disaster-recovery/README.md index 64c7670b0a3a..4b45af811e8e 100644 --- a/lean/disaster-recovery/README.md +++ b/lean/disaster-recovery/README.md @@ -30,7 +30,7 @@ The initial migration is approximately 4,000 lines across 28 new files: | ----------------------------------------- | ----: | ----------------: | --------------------------------------------------------------- | | Exact legacy Lean model | 4 | 650 | Stateright state, actions, timers, network, predicates, and BFS | | Canonical protocol and proofs | 4 | 1,000 | C++ behavior, phase refinement, quorum, and temporal proofs | -| Trace validation | 11 | 1,020 | NDJSON format, deterministic replay, tests, and fixtures | +| Trace validation | 11 | 1,040 | NDJSON format, deterministic replay, tests, and fixtures | | Equivalence tooling | 2 | 710 | Rust graph exporter and bidirectional Rust/Lean comparison | | Project, documentation, and configuration | 6 | 530 | Lake/Mathlib setup, entry points, and documentation | | Pull-request CI | 1 | 80 | Lean model and bounded equivalence checks | @@ -45,8 +45,8 @@ The initial migration is approximately 4,000 lines across 28 new files: | [`DisasterRecovery/Protocol/Refinement.lean`](DisasterRecovery/Protocol/Refinement.lean) | 332 | Canonical-to-legacy formal phase refinement | | [`DisasterRecovery/Protocol/Temporal.lean`](DisasterRecovery/Protocol/Temporal.lean) | 230 | Execution streams, fairness, safety, and progress proofs | | [`DisasterRecovery/Protocol/Trace/Format.lean`](DisasterRecovery/Protocol/Trace/Format.lean) | 141 | Versioned NDJSON types and parser | -| [`DisasterRecovery/Protocol/Trace/Replay.lean`](DisasterRecovery/Protocol/Trace/Replay.lean) | 445 | Deterministic implementation-trace replay | -| [`TraceTests.lean`](TraceTests.lean) | 210 | Strict replay, causality, sequencing, and effect regressions | +| [`DisasterRecovery/Protocol/Trace/Replay.lean`](DisasterRecovery/Protocol/Trace/Replay.lean) | 452 | Deterministic implementation-trace replay | +| [`TraceTests.lean`](TraceTests.lean) | 227 | Strict replay, causality, sequencing, and effect regressions | | [`compare.py`](compare.py) | 343 | Exhaustive canonical graph comparison | | [`../../tla/disaster-recovery/src/export.rs`](../../tla/disaster-recovery/src/export.rs) | 370 | Canonical export of the actual Stateright model | | [`TRACE_FORMAT_V1.md`](TRACE_FORMAT_V1.md) | 145 | Contract for future committed C++ trace events | @@ -370,13 +370,15 @@ one-shot effects are explicit. [`Trace/Format.lean`](DisasterRecovery/Protocol/Trace/Format.lean) parses the wire format. [`Trace/Replay.lean`](DisasterRecovery/Protocol/Trace/Replay.lean) then folds each event over one deterministic canonical `SystemState`. Replay -retains only per-node sequences, observed and consumed causal sends, and -node-scoped pending `open`, `join_restart`, or `complete` effects. +retains only per-node sequences, observed and consumed causal sends, exact +ordered retry-send batches, and node-scoped pending `open`, `join_restart`, or +`complete` effects. The validator checks configuration consistency, starts, globally unique message -IDs, send enablement, exact send/receive causality, canonical pre/post phases, -and single consumption of committed effects. On failure it reports the shortest -failing prefix and current protocol phase. +IDs, send-batch completeness, exact send/receive causality including gossip +TxIDs, canonical pre/post phases, terminal completion, and single consumption +of committed effects. On failure it reports the shortest failing prefix and +current protocol phase. ### Implementation trace validation diff --git a/lean/disaster-recovery/TRACE_FORMAT_V1.md b/lean/disaster-recovery/TRACE_FORMAT_V1.md index 306460db6175..37cad96f0e63 100644 --- a/lean/disaster-recovery/TRACE_FORMAT_V1.md +++ b/lean/disaster-recovery/TRACE_FORMAT_V1.md @@ -40,22 +40,23 @@ All integers must be nonnegative Lean `Nat` values. ## Event kinds -| Kind | Required event fields | Canonical boundary | -| ------------------ | ------------------------------------------------------------------- | ------------------------------------------- | -| `start` | `pre`, `post` | Protocol state initialized | -| `gossip_accepted` | `message_id`, `caused_by`, `source`, `view`, `seqno`, `pre`, `post` | Validated gossip callback committed | -| `vote_accepted` | `message_id`, `caused_by`, `source`, `pre`, `post` | Validated vote callback committed | -| `iamopen_accepted` | `message_id`, `caused_by`, `source`, `pre`, `post` | IAmOpen selected peer and Joining committed | -| `timeout` | `pre`, `post` | Timeout transaction committed | -| `send` | `send` in `class:destination` form, `message_id`, `pre`, `post` | Transport send observed | -| `open` | `open_kind`, `pre`, `post` | Service-open transition committed | -| `join_restart` | `pre`, `post` | Joining/restart side effect committed | -| `complete` | `pre`, `post` | Opening-to-Open completion committed | +| Kind | Required event fields | Canonical boundary | +| ------------------ | ----------------------------------------------------------------------------------------------------- | ------------------------------------------- | +| `start` | `pre`, `post` | Protocol state initialized | +| `gossip_accepted` | `message_id`, `caused_by`, `source`, `view`, `seqno`, `pre`, `post` | Validated gossip callback committed | +| `vote_accepted` | `message_id`, `caused_by`, `source`, `pre`, `post` | Validated vote callback committed | +| `iamopen_accepted` | `message_id`, `caused_by`, `source`, `pre`, `post` | IAmOpen selected peer and Joining committed | +| `timeout` | `pre`, `post` | Timeout transaction committed | +| `send` | `send` in `class:destination` form, `message_id`, `pre`, `post`; gossip also requires `view`, `seqno` | Transport send observed | +| `open` | `open_kind`, `pre`, `post` | Service-open transition committed | +| `join_restart` | `pre`, `post` | Joining/restart side effect committed | +| `complete` | `pre`, `post` | Opening-to-Open completion committed | Every receive uses `caused_by` to identify an earlier `send`. The validator -checks the sender, destination, message class, and single consumption of that -send. Message IDs, causal IDs, and source names must be nonempty, and message -IDs cannot be reused. Non-receive events must omit `caused_by`. +checks the sender, destination, message class, gossip TxID payload, and single +consumption of that send. Message IDs, causal IDs, and source names must be +nonempty, and message IDs cannot be reused. Non-receive events must omit +`caused_by`. Each participating configured node has one `start` event at sequence zero. The first creates the replay system; later starts activate other configured nodes @@ -76,7 +77,8 @@ Version 1 is a complete successful-execution trace: every transport send, accepted receive, committed timeout, and one-shot effect is explicit. `Trace/Replay.lean` folds these events over one deterministic `SystemState`. It retains only observed sends, consumed causal IDs, per-node sequences, and -pending `open`, `join_restart`, or `complete` effects. +pending ordered retry-send batches and `open`, `join_restart`, or `complete` +effects. The validator rejects the first event that is not enabled by the canonical model or whose recorded pre/post state, cause, or effect does not match. It diff --git a/lean/disaster-recovery/TraceTests.lean b/lean/disaster-recovery/TraceTests.lean index 33462aaab623..e719b5c76eeb 100644 --- a/lean/disaster-recovery/TraceTests.lean +++ b/lean/disaster-recovery/TraceTests.lean @@ -44,6 +44,10 @@ private def sendEvent messageId := some messageId pre := some phase post := some phase + txid := if description.startsWith "gossip:" then + some { view := 1, seqno := 1 } + else + none send := some description } @@ -171,6 +175,19 @@ def main : IO UInt32 := do (failedAt [start, gossipSend, wrongClass] 3) "vote consumed a gossip send" + let wrongTxid := { + gossipEvent single 2 "receive-gossip" "send-gossip" .voting with + txid := some { view := 9, seqno := 9 } + } + expect + (failedAt [start, gossipSend, wrongTxid] 3) + "gossip received a different TxID than its send" + + let wrongPost := gossipEvent single 2 "receive-gossip" "send-gossip" .open + expect + (failedAt [start, gossipSend, wrongPost] 3) + "invalid gossip post-state was accepted" + let received := gossipEvent single 2 "receive-gossip" "send-gossip" .voting let reusedCause := { voteEvent single 3 "receive-vote" "send-gossip" .voting with diff --git a/lean/disaster-recovery/fixtures/accepted-failover.ndjson b/lean/disaster-recovery/fixtures/accepted-failover.ndjson index 87210e273c77..27e560c8b59f 100644 --- a/lean/disaster-recovery/fixtures/accepted-failover.ndjson +++ b/lean/disaster-recovery/fixtures/accepted-failover.ndjson @@ -1,13 +1,13 @@ {"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":1,"kind":"send","message_id":"send-gossip-a","send":"gossip:A","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":2,"kind":"send","message_id":"send-gossip-b","send":"gossip:B","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":3,"kind":"send","message_id":"send-gossip-c","send":"gossip:C","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":1,"kind":"send","message_id":"send-gossip-a","send":"gossip:A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":2,"kind":"send","message_id":"send-gossip-b","send":"gossip:B","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":3,"kind":"send","message_id":"send-gossip-c","send":"gossip:C","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":4,"kind":"gossip_accepted","message_id":"gossip-a","caused_by":"send-gossip-a","source":"A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":5,"kind":"timeout","pre":"GOSSIPING","post":"VOTING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":6,"kind":"send","message_id":"send-vote-a","send":"vote:A","pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":7,"kind":"send","message_id":"send-voting-gossip-a","send":"gossip:A","pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":8,"kind":"send","message_id":"send-voting-gossip-b","send":"gossip:B","pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":9,"kind":"send","message_id":"send-voting-gossip-c","send":"gossip:C","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":7,"kind":"send","message_id":"send-voting-gossip-a","send":"gossip:A","view":1,"seqno":1,"pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":8,"kind":"send","message_id":"send-voting-gossip-b","send":"gossip:B","view":1,"seqno":1,"pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":9,"kind":"send","message_id":"send-voting-gossip-c","send":"gossip:C","view":1,"seqno":1,"pre":"VOTING","post":"VOTING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":10,"kind":"vote_accepted","message_id":"vote-a","caused_by":"send-vote-a","source":"A","pre":"VOTING","post":"VOTING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":11,"kind":"timeout","pre":"VOTING","post":"OPENING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":12,"kind":"open","open_kind":"FAILOVER","pre":"OPENING","post":"OPENING"} diff --git a/lean/disaster-recovery/fixtures/accepted-multinode.ndjson b/lean/disaster-recovery/fixtures/accepted-multinode.ndjson index 69df47f1fec5..906a46e2209b 100644 --- a/lean/disaster-recovery/fixtures/accepted-multinode.ndjson +++ b/lean/disaster-recovery/fixtures/accepted-multinode.ndjson @@ -1,19 +1,19 @@ {"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":1,"kind":"send","message_id":"a-gossip-a","send":"gossip:A","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":2,"kind":"send","message_id":"a-gossip-b","send":"gossip:B","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":1,"kind":"send","message_id":"b-gossip-a","send":"gossip:A","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":2,"kind":"send","message_id":"b-gossip-b","send":"gossip:B","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":1,"kind":"send","message_id":"a-gossip-a","send":"gossip:A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":2,"kind":"send","message_id":"a-gossip-b","send":"gossip:B","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":1,"kind":"send","message_id":"b-gossip-a","send":"gossip:A","view":2,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":2,"kind":"send","message_id":"b-gossip-b","send":"gossip:B","view":2,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":3,"kind":"gossip_accepted","message_id":"a-receive-a","caused_by":"a-gossip-a","source":"A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":4,"kind":"gossip_accepted","message_id":"a-receive-b","caused_by":"b-gossip-a","source":"B","view":2,"seqno":1,"pre":"GOSSIPING","post":"VOTING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":3,"kind":"gossip_accepted","message_id":"b-receive-a","caused_by":"a-gossip-b","source":"A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":4,"kind":"gossip_accepted","message_id":"b-receive-b","caused_by":"b-gossip-b","source":"B","view":2,"seqno":1,"pre":"GOSSIPING","post":"VOTING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":5,"kind":"send","message_id":"a-vote-b","send":"vote:B","pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":6,"kind":"send","message_id":"a-voting-gossip-a","send":"gossip:A","pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":7,"kind":"send","message_id":"a-voting-gossip-b","send":"gossip:B","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":6,"kind":"send","message_id":"a-voting-gossip-a","send":"gossip:A","view":1,"seqno":1,"pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":7,"kind":"send","message_id":"a-voting-gossip-b","send":"gossip:B","view":1,"seqno":1,"pre":"VOTING","post":"VOTING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":5,"kind":"send","message_id":"b-vote-b","send":"vote:B","pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":6,"kind":"send","message_id":"b-voting-gossip-a","send":"gossip:A","pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":7,"kind":"send","message_id":"b-voting-gossip-b","send":"gossip:B","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":6,"kind":"send","message_id":"b-voting-gossip-a","send":"gossip:A","view":2,"seqno":1,"pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":7,"kind":"send","message_id":"b-voting-gossip-b","send":"gossip:B","view":2,"seqno":1,"pre":"VOTING","post":"VOTING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":8,"kind":"vote_accepted","message_id":"b-receive-vote-a","caused_by":"a-vote-b","source":"A","pre":"VOTING","post":"VOTING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":9,"kind":"vote_accepted","message_id":"b-receive-vote-b","caused_by":"b-vote-b","source":"B","pre":"VOTING","post":"OPENING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":10,"kind":"open","open_kind":"QUORUM","pre":"OPENING","post":"OPENING"} diff --git a/lean/disaster-recovery/fixtures/accepted.ndjson b/lean/disaster-recovery/fixtures/accepted.ndjson index b6970a90a56c..182111180bfa 100644 --- a/lean/disaster-recovery/fixtures/accepted.ndjson +++ b/lean/disaster-recovery/fixtures/accepted.ndjson @@ -1,8 +1,8 @@ {"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":1,"kind":"send","message_id":"send-gossip-1","send":"gossip:A","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":1,"kind":"send","message_id":"send-gossip-1","send":"gossip:A","view":1,"seqno":10,"pre":"GOSSIPING","post":"GOSSIPING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":2,"kind":"gossip_accepted","message_id":"recv-gossip-1","caused_by":"send-gossip-1","source":"A","view":1,"seqno":10,"pre":"GOSSIPING","post":"VOTING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":3,"kind":"send","message_id":"send-vote-1","send":"vote:A","pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":4,"kind":"send","message_id":"send-gossip-2","send":"gossip:A","pre":"VOTING","post":"VOTING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":4,"kind":"send","message_id":"send-gossip-2","send":"gossip:A","view":1,"seqno":10,"pre":"VOTING","post":"VOTING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":5,"kind":"vote_accepted","message_id":"recv-vote-1","caused_by":"send-vote-1","source":"A","pre":"VOTING","post":"OPENING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":6,"kind":"open","open_kind":"QUORUM","pre":"OPENING","post":"OPENING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":7,"kind":"timeout","pre":"OPENING","post":"OPENING"} diff --git a/lean/disaster-recovery/fixtures/rejected-cause.ndjson b/lean/disaster-recovery/fixtures/rejected-cause.ndjson index cc0332d2795b..41afdf3eb2e5 100644 --- a/lean/disaster-recovery/fixtures/rejected-cause.ndjson +++ b/lean/disaster-recovery/fixtures/rejected-cause.ndjson @@ -1,3 +1,3 @@ {"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-cause","expected_locations":["A"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-cause","expected_locations":["A"],"node":"A","sequence":1,"kind":"send","message_id":"send-gossip","send":"gossip:A","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-cause","expected_locations":["A"],"node":"A","sequence":1,"kind":"send","message_id":"send-gossip","send":"gossip:A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} {"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-cause","expected_locations":["A"],"node":"A","sequence":2,"kind":"vote_accepted","message_id":"receive-vote","caused_by":"send-gossip","source":"A","pre":"GOSSIPING","post":"GOSSIPING"} diff --git a/lean/disaster-recovery/fixtures/rejected.ndjson b/lean/disaster-recovery/fixtures/rejected.ndjson index 8a54beb2d427..559769f20e18 100644 --- a/lean/disaster-recovery/fixtures/rejected.ndjson +++ b/lean/disaster-recovery/fixtures/rejected.ndjson @@ -1,2 +1,3 @@ -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":1,"kind":"gossip_accepted","message_id":"bad-gossip","source":"A","view":1,"seqno":10,"pre":"GOSSIPING","post":"OPEN"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-state","expected_locations":["A"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-state","expected_locations":["A"],"node":"A","sequence":1,"kind":"send","message_id":"send-gossip","send":"gossip:A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} +{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-state","expected_locations":["A"],"node":"A","sequence":2,"kind":"gossip_accepted","message_id":"receive-gossip","caused_by":"send-gossip","source":"A","view":1,"seqno":1,"pre":"GOSSIPING","post":"OPEN"} diff --git a/src/node/recovery_decision_protocol.cpp b/src/node/recovery_decision_protocol.cpp index ff03eaa36ab5..a2984ff1a27f 100644 --- a/src/node/recovery_decision_protocol.cpp +++ b/src/node/recovery_decision_protocol.cpp @@ -182,15 +182,23 @@ namespace ccf } void RecoveryDecisionProtocolSubsystem::emit_trace_send_unsafe( - const std::string& message_id, const std::string& description) + const std::string& message_id, + const std::string& description, + const std::optional& txid) { - emit_trace_event_unsafe({ + recovery_decision_protocol::TraceEvent event{ .kind = "send", .message_id = message_id, .pre = "", .post = "", .send = description, - }); + }; + if (txid.has_value()) + { + event.view = txid->view; + event.seqno = txid->seqno; + } + emit_trace_event_unsafe(std::move(event)); } void RecoveryDecisionProtocolSubsystem::record_trace_effects( @@ -933,7 +941,8 @@ namespace ccf #ifdef CCF_RECOVERY_TRACE emit_trace_send_unsafe( request.trace_message_id.value(), - fmt::format("gossip:{}", target.name)); + fmt::format("gossip:{}", target.name), + request.txid); #endif dispatch_authenticated_message( request_json, diff --git a/src/node/recovery_decision_protocol.h b/src/node/recovery_decision_protocol.h index fd204735553d..1cac209ba929 100644 --- a/src/node/recovery_decision_protocol.h +++ b/src/node/recovery_decision_protocol.h @@ -141,7 +141,9 @@ namespace ccf std::string new_trace_message_id(); std::string new_trace_message_id_unsafe(); void emit_trace_send_unsafe( - const std::string& message_id, const std::string& description); + const std::string& message_id, + const std::string& description, + const std::optional& txid = std::nullopt); #endif }; } From 2dbb492b1f061925cd42a42f131b1dc4e327f686 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Mon, 31 Aug 2026 13:46:24 +0100 Subject: [PATCH 09/14] Validate only captured recovery traces Remove checked-in and synthetic recovery traces so Lean replay is exercised exclusively with NDJSON captured from the C++ implementation and retained as CI artifacts. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 46d56f8a-935f-4551-abe3-84bd4f951865 --- .github/workflows/README.md | 7 +- .github/workflows/ci-verification.yml | 1 - .github/workflows/lean-shallow.yml | 21 -- lean/disaster-recovery/README.md | 29 ++- lean/disaster-recovery/TRACE_FORMAT_V1.md | 9 +- lean/disaster-recovery/TraceTests.lean | 227 ------------------ .../fixtures/accepted-failover.ndjson | 15 -- .../fixtures/accepted-multinode.ndjson | 26 -- .../fixtures/accepted.ndjson | 11 - .../fixtures/rejected-cause.ndjson | 3 - .../fixtures/rejected.ndjson | 3 - lean/disaster-recovery/lakefile.toml | 5 - tests/infra/recovery_trace_test.py | 164 ------------- 13 files changed, 22 insertions(+), 499 deletions(-) delete mode 100644 lean/disaster-recovery/TraceTests.lean delete mode 100644 lean/disaster-recovery/fixtures/accepted-failover.ndjson delete mode 100644 lean/disaster-recovery/fixtures/accepted-multinode.ndjson delete mode 100644 lean/disaster-recovery/fixtures/accepted.ndjson delete mode 100644 lean/disaster-recovery/fixtures/rejected-cause.ndjson delete mode 100644 lean/disaster-recovery/fixtures/rejected.ndjson delete mode 100644 tests/infra/recovery_trace_test.py diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 5b47a8acedbb..c28db7f199ea 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -103,10 +103,9 @@ File: `tla-shallow.yml` # Lean Shallow Verification -Builds and checks the Lean disaster-recovery models, validates trace fixtures -and the causal log merger, and compares the bounded Lean legacy model with -Stateright on relevant pull requests. The SNP jobs in `ci.yml` additionally -validate committed C++ recovery traces. +Builds and checks the Lean disaster-recovery models and compares the bounded +Lean legacy model with Stateright on relevant pull requests. The SNP jobs in +`ci.yml` validate committed C++ recovery traces. File: `lean-shallow.yml` 3rd party dependencies: None diff --git a/.github/workflows/ci-verification.yml b/.github/workflows/ci-verification.yml index 3cda3abeb8a2..2a4ffcbbc073 100644 --- a/.github/workflows/ci-verification.yml +++ b/.github/workflows/ci-verification.yml @@ -296,6 +296,5 @@ jobs: lake build lake exe semantic-checks lake exe canonical-checks - lake exe trace-checks lake exe disaster-recovery check --nodes 3 python3 compare.py --nodes 1 2 3 diff --git a/.github/workflows/lean-shallow.yml b/.github/workflows/lean-shallow.yml index 80764b8b4c9f..0b6befd27adb 100644 --- a/.github/workflows/lean-shallow.yml +++ b/.github/workflows/lean-shallow.yml @@ -12,7 +12,6 @@ on: - "src/node/rpc/self_healing_open_handlers.h" - "tests/e2e_operations.py" - "tests/infra/recovery_trace.py" - - "tests/infra/recovery_trace_test.py" - "CMakeLists.txt" - ".github/workflows/ci.yml" - ".github/workflows/lean-shallow.yml" @@ -55,28 +54,8 @@ jobs: lake build lake exe semantic-checks lake exe canonical-checks - lake exe trace-checks lake exe disaster-recovery check --nodes 3 - - name: Check trace validation - working-directory: lean/disaster-recovery - shell: bash - run: | - set -euo pipefail - lake exe trace-validator fixtures/accepted.ndjson - lake exe trace-validator fixtures/accepted-failover.ndjson - lake exe trace-validator fixtures/accepted-multinode.ndjson - if lake exe trace-validator fixtures/rejected.ndjson; then - echo "Rejected state trace was accepted" - exit 1 - fi - if lake exe trace-validator fixtures/rejected-cause.ndjson; then - echo "Rejected causal trace was accepted" - exit 1 - fi - PYTHONPATH=../../tests python3 -m unittest discover \ - -s ../../tests/infra -p recovery_trace_test.py - - name: Compare Lean and Stateright working-directory: lean/disaster-recovery shell: bash diff --git a/lean/disaster-recovery/README.md b/lean/disaster-recovery/README.md index 4b45af811e8e..ca6e8c42b23e 100644 --- a/lean/disaster-recovery/README.md +++ b/lean/disaster-recovery/README.md @@ -24,13 +24,13 @@ intentionally differs from the legacy model in several places. It has formal phase-refinement and fairness-aware progress results, plus a versioned trace validator designed for future committed C++ instrumentation. -The initial migration is approximately 4,000 lines across 28 new files: +The initial migration is approximately 4,000 lines across 23 new files: | Area | Files | Approximate lines | Purpose | | ----------------------------------------- | ----: | ----------------: | --------------------------------------------------------------- | | Exact legacy Lean model | 4 | 650 | Stateright state, actions, timers, network, predicates, and BFS | | Canonical protocol and proofs | 4 | 1,000 | C++ behavior, phase refinement, quorum, and temporal proofs | -| Trace validation | 11 | 1,040 | NDJSON format, deterministic replay, tests, and fixtures | +| Trace validation | 5 | 760 | NDJSON format, deterministic replay, and CLI | | Equivalence tooling | 2 | 710 | Rust graph exporter and bidirectional Rust/Lean comparison | | Project, documentation, and configuration | 6 | 530 | Lake/Mathlib setup, entry points, and documentation | | Pull-request CI | 1 | 80 | Lean model and bounded equivalence checks | @@ -46,7 +46,6 @@ The initial migration is approximately 4,000 lines across 28 new files: | [`DisasterRecovery/Protocol/Temporal.lean`](DisasterRecovery/Protocol/Temporal.lean) | 230 | Execution streams, fairness, safety, and progress proofs | | [`DisasterRecovery/Protocol/Trace/Format.lean`](DisasterRecovery/Protocol/Trace/Format.lean) | 141 | Versioned NDJSON types and parser | | [`DisasterRecovery/Protocol/Trace/Replay.lean`](DisasterRecovery/Protocol/Trace/Replay.lean) | 452 | Deterministic implementation-trace replay | -| [`TraceTests.lean`](TraceTests.lean) | 227 | Strict replay, causality, sequencing, and effect regressions | | [`compare.py`](compare.py) | 343 | Exhaustive canonical graph comparison | | [`../../tla/disaster-recovery/src/export.rs`](../../tla/disaster-recovery/src/export.rs) | 370 | Canonical export of the actual Stateright model | | [`TRACE_FORMAT_V1.md`](TRACE_FORMAT_V1.md) | 145 | Contract for future committed C++ trace events | @@ -399,6 +398,12 @@ validator. The quorum, failover, and multiple-timeout recovery scenarios call this helper, and both SNP CI jobs build CCF with tracing enabled and provide the Lean validator binary. +The helper writes `*.recovery.ndjson` before validation. SNP CI uploads these +generated files with node logs for debugging and local replay. No +recovery-decision-protocol trace files are checked into the repository. Every +NDJSON trace passed to the validator in CI comes from the running C++ +implementation. + ## Commands From this directory: @@ -407,13 +412,7 @@ From this directory: lake build lake exe semantic-checks lake exe canonical-checks -lake exe trace-checks lake exe disaster-recovery check --nodes 3 -lake exe trace-validator fixtures/accepted.ndjson -lake exe trace-validator fixtures/accepted-failover.ndjson -lake exe trace-validator fixtures/accepted-multinode.ndjson -! lake exe trace-validator fixtures/rejected.ndjson -! lake exe trace-validator fixtures/rejected-cause.ndjson python3 compare.py ``` @@ -446,11 +445,12 @@ Stateright executables without formal semantics for them. The Rust comparison exporter is checked separately from `tla/disaster-recovery/` with `cargo check` and `cargo build`. -`.github/workflows/lean-shallow.yml` runs the Lean build, semantic checks, -trace checks, three-node legacy property check, and the one- and two-node -Rust/Lean comparison on relevant pull requests. The weekly continuous +`.github/workflows/lean-shallow.yml` builds the Lean library and trace validator, +runs semantic and canonical checks, checks the three-node legacy properties, +and compares the one- and two-node Rust/Lean graphs. The weekly continuous verification workflow additionally runs the exhaustive three-node comparison. -The Rust job remains in place until the replacement criteria below are met. +Only the SNP jobs feed real C++ traces to the validator. The Rust job remains in +place until the replacement criteria below are met. ### Current CI evidence @@ -458,8 +458,7 @@ Draft PR [microsoft/CCF#8241](https://github.com/microsoft/CCF/pull/8241) validated commit `c8dd40a7` on both proof and hardware paths: - [Lean shallow verification](https://github.com/microsoft/CCF/actions/runs/33315731130/job/99268715043) - built every Lean target, checked trace fixtures and merger behavior, and - compared the bounded Rust/Lean graphs. + built every Lean target and compared the bounded Rust/Lean graphs. - [Milan SNP](https://github.com/microsoft/CCF/actions/runs/33315731084/job/99268714883) validated committed quorum, failover, and multiple-timeout implementation traces. diff --git a/lean/disaster-recovery/TRACE_FORMAT_V1.md b/lean/disaster-recovery/TRACE_FORMAT_V1.md index 37cad96f0e63..a7d7cc5e7b59 100644 --- a/lean/disaster-recovery/TRACE_FORMAT_V1.md +++ b/lean/disaster-recovery/TRACE_FORMAT_V1.md @@ -115,6 +115,8 @@ Each log record contains `RDP_TRACE ` followed by the event object. logs, topologically orders them by per-node sequence and causal send edges, writes NDJSON, and invokes the Lean validator. The quorum, failover, and multiple-timeout SNP e2e scenarios call this helper. +Each generated `*.recovery.ndjson` file is retained with the SNP job's uploaded +logs, so a failed replay can be reproduced locally. The e2e helper additionally requires scenario-specific terminal evidence before accepting the trace: the expected open kind, at least one completed opener, and @@ -135,7 +137,6 @@ a `complete` or `join_restart` event for every participating node. } ``` -Accepted quorum, failover-with-unavailable-locations, and multi-node examples -are `fixtures/accepted.ndjson`, `fixtures/accepted-failover.ndjson`, and -`fixtures/accepted-multinode.ndjson`. Deliberately rejected state and causal -examples are `fixtures/rejected.ndjson` and `fixtures/rejected-cause.ndjson`. +No recovery-decision-protocol traces are checked into the repository. Every +NDJSON trace passed to the validator in CI is captured from the running C++ +implementation. diff --git a/lean/disaster-recovery/TraceTests.lean b/lean/disaster-recovery/TraceTests.lean deleted file mode 100644 index e719b5c76eeb..000000000000 --- a/lean/disaster-recovery/TraceTests.lean +++ /dev/null @@ -1,227 +0,0 @@ -import DisasterRecovery.Protocol.Trace - -open DisasterRecovery.Protocol -open DisasterRecovery.Protocol.Trace - -private def expect (condition : Bool) (message : String) : IO Unit := - unless condition do throw (IO.userError message) - -private def baseEvent - (locations : List Location) - (node : Location) - (sequence : Nat) - (kind : Kind) : TraceEvent := { - version := contractVersion - instanceId := "trace-tests" - expectedLocations := locations - node - sequence - kind - messageId := none - causedBy := none - source := none - txid := none - pre := none - post := none - openKind := none - send := none -} - -private def startEvent - (locations : List Location) - (node : Location) : TraceEvent := { - baseEvent locations node 0 .start with - pre := some .gossiping - post := some .gossiping -} - -private def sendEvent - (locations : List Location) - (sequence : Nat) - (messageId description : String) - (phase : Phase) : TraceEvent := { - baseEvent locations "A" sequence .send with - messageId := some messageId - pre := some phase - post := some phase - txid := if description.startsWith "gossip:" then - some { view := 1, seqno := 1 } - else - none - send := some description -} - -private def gossipEvent - (locations : List Location) - (sequence : Nat) - (messageId cause : String) - (post : Phase) : TraceEvent := { - baseEvent locations "A" sequence .gossipAccepted with - messageId := some messageId - causedBy := some cause - source := some "A" - txid := some { view := 1, seqno := 1 } - pre := some .gossiping - post := some post -} - -private def voteEvent - (locations : List Location) - (sequence : Nat) - (messageId cause : String) - (post : Phase) : TraceEvent := { - baseEvent locations "A" sequence .voteAccepted with - messageId := some messageId - causedBy := some cause - source := some "A" - pre := some .voting - post := some post -} - -private def timeoutEvent - (locations : List Location) - (sequence : Nat) - (pre post : Phase) : TraceEvent := { - baseEvent locations "A" sequence .timeout with - pre := some pre - post := some post -} - -private def openEvent - (locations : List Location) - (sequence : Nat) - (kind : OpenKind) : TraceEvent := { - baseEvent locations "A" sequence .open with - pre := some .opening - post := some .opening - openKind := some kind -} - -private def completeEvent - (locations : List Location) - (sequence : Nat) : TraceEvent := { - baseEvent locations "A" sequence .complete with - pre := some .open - post := some .open -} - -private def validationSucceeds (events : List TraceEvent) : Bool := - match validate events with - | .ok () => true - | .error _ => false - -private def failedAt (events : List TraceEvent) (expectedPrefix : Nat) : Bool := - match validate events with - | .error failure => failure.prefixLength == expectedPrefix - | .ok () => false - -private def parseFails (value : String) : Bool := - match parseEvent value with - | .error _ => true - | .ok _ => false - -private def quorumTrace : List TraceEvent := - let locations := ["A"] - [ - startEvent locations "A", - sendEvent locations 1 "send-gossip" "gossip:A" .gossiping, - gossipEvent locations 2 "receive-gossip" "send-gossip" .voting, - sendEvent locations 3 "send-vote" "vote:A" .voting, - sendEvent locations 4 "send-voting-gossip" "gossip:A" .voting, - voteEvent locations 5 "receive-vote" "send-vote" .opening, - openEvent locations 6 .quorum, - timeoutEvent locations 7 .opening .opening, - timeoutEvent locations 8 .opening .opening, - timeoutEvent locations 9 .opening .open, - completeEvent locations 10 - ] - -def main : IO UInt32 := do - expect (validationSucceeds quorumTrace) "complete quorum trace was rejected" - - let locations := ["A", "B"] - expect - (failedAt [startEvent locations "A", startEvent locations "B"] 2) - "incomplete multi-node trace was accepted" - expect - (failedAt [startEvent locations "A"] 1) - "incomplete single-node trace was accepted" - expect - (failedAt [startEvent locations "A", startEvent locations "A"] 2) - "duplicate start was accepted" - expect - (failedAt [startEvent ["A", "A"] "A"] 1) - "duplicate expected locations were accepted" - expect - (failedAt [{ startEvent ["A"] "A" with instanceId := "" }] 1) - "empty recovery instance was accepted" - - let single := ["A"] - let start := startEvent single "A" - let gossipSend := sendEvent single 1 "send-gossip" "gossip:A" .gossiping - let missingCause := { - gossipEvent single 2 "receive-gossip" "missing" .voting with - causedBy := none - } - expect - (failedAt [start, gossipSend, missingCause] 3) - "receive without caused_by was accepted" - - let wrongClass := { - voteEvent single 2 "receive-vote" "send-gossip" .voting with - pre := some .gossiping - } - expect - (failedAt [start, gossipSend, wrongClass] 3) - "vote consumed a gossip send" - - let wrongTxid := { - gossipEvent single 2 "receive-gossip" "send-gossip" .voting with - txid := some { view := 9, seqno := 9 } - } - expect - (failedAt [start, gossipSend, wrongTxid] 3) - "gossip received a different TxID than its send" - - let wrongPost := gossipEvent single 2 "receive-gossip" "send-gossip" .open - expect - (failedAt [start, gossipSend, wrongPost] 3) - "invalid gossip post-state was accepted" - - let received := gossipEvent single 2 "receive-gossip" "send-gossip" .voting - let reusedCause := { - voteEvent single 3 "receive-vote" "send-gossip" .voting with - pre := some .voting - } - expect - (failedAt [start, gossipSend, received, reusedCause] 4) - "one send caused multiple receives" - - let badSend := sendEvent single 1 "send-vote" "vote:A" .gossiping - expect - (failedAt [start, badSend] 2) - "Voting send was accepted while Gossiping" - - let abortedTimeout := timeoutEvent single 1 .gossiping .gossiping - expect - (failedAt [start, abortedTimeout] 2) - "aborted empty-gossip timeout was accepted" - - let throughOpen := quorumTrace.take 7 - expect - (failedAt (throughOpen ++ [openEvent single 7 .quorum]) 8) - "one opening transition produced multiple open observations" - expect - (failedAt (quorumTrace.take 6) 6) - "trace with an unobserved opening effect was accepted" - - let rejectedJson := - "{\"version\":\"ccf.recovery_decision_protocol.trace/1\"," - ++ "\"instance\":\"x\",\"expected_locations\":[\"A\"]," - ++ "\"node\":\"A\",\"sequence\":0,\"kind\":\"gossip_rejected\"," - ++ "\"pre\":\"GOSSIPING\",\"post\":\"GOSSIPING\"}" - expect (parseFails rejectedJson) - "unused rejection event remains in the strict v1 format" - - IO.println "all strict trace replay checks passed" - pure 0 diff --git a/lean/disaster-recovery/fixtures/accepted-failover.ndjson b/lean/disaster-recovery/fixtures/accepted-failover.ndjson deleted file mode 100644 index 27e560c8b59f..000000000000 --- a/lean/disaster-recovery/fixtures/accepted-failover.ndjson +++ /dev/null @@ -1,15 +0,0 @@ -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":1,"kind":"send","message_id":"send-gossip-a","send":"gossip:A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":2,"kind":"send","message_id":"send-gossip-b","send":"gossip:B","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":3,"kind":"send","message_id":"send-gossip-c","send":"gossip:C","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":4,"kind":"gossip_accepted","message_id":"gossip-a","caused_by":"send-gossip-a","source":"A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":5,"kind":"timeout","pre":"GOSSIPING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":6,"kind":"send","message_id":"send-vote-a","send":"vote:A","pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":7,"kind":"send","message_id":"send-voting-gossip-a","send":"gossip:A","view":1,"seqno":1,"pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":8,"kind":"send","message_id":"send-voting-gossip-b","send":"gossip:B","view":1,"seqno":1,"pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":9,"kind":"send","message_id":"send-voting-gossip-c","send":"gossip:C","view":1,"seqno":1,"pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":10,"kind":"vote_accepted","message_id":"vote-a","caused_by":"send-vote-a","source":"A","pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":11,"kind":"timeout","pre":"VOTING","post":"OPENING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":12,"kind":"open","open_kind":"FAILOVER","pre":"OPENING","post":"OPENING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":13,"kind":"timeout","pre":"OPENING","post":"OPEN"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"failover","expected_locations":["A","B","C"],"node":"A","sequence":14,"kind":"complete","pre":"OPEN","post":"OPEN"} diff --git a/lean/disaster-recovery/fixtures/accepted-multinode.ndjson b/lean/disaster-recovery/fixtures/accepted-multinode.ndjson deleted file mode 100644 index 906a46e2209b..000000000000 --- a/lean/disaster-recovery/fixtures/accepted-multinode.ndjson +++ /dev/null @@ -1,26 +0,0 @@ -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":1,"kind":"send","message_id":"a-gossip-a","send":"gossip:A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":2,"kind":"send","message_id":"a-gossip-b","send":"gossip:B","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":1,"kind":"send","message_id":"b-gossip-a","send":"gossip:A","view":2,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":2,"kind":"send","message_id":"b-gossip-b","send":"gossip:B","view":2,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":3,"kind":"gossip_accepted","message_id":"a-receive-a","caused_by":"a-gossip-a","source":"A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":4,"kind":"gossip_accepted","message_id":"a-receive-b","caused_by":"b-gossip-a","source":"B","view":2,"seqno":1,"pre":"GOSSIPING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":3,"kind":"gossip_accepted","message_id":"b-receive-a","caused_by":"a-gossip-b","source":"A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":4,"kind":"gossip_accepted","message_id":"b-receive-b","caused_by":"b-gossip-b","source":"B","view":2,"seqno":1,"pre":"GOSSIPING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":5,"kind":"send","message_id":"a-vote-b","send":"vote:B","pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":6,"kind":"send","message_id":"a-voting-gossip-a","send":"gossip:A","view":1,"seqno":1,"pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":7,"kind":"send","message_id":"a-voting-gossip-b","send":"gossip:B","view":1,"seqno":1,"pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":5,"kind":"send","message_id":"b-vote-b","send":"vote:B","pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":6,"kind":"send","message_id":"b-voting-gossip-a","send":"gossip:A","view":2,"seqno":1,"pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":7,"kind":"send","message_id":"b-voting-gossip-b","send":"gossip:B","view":2,"seqno":1,"pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":8,"kind":"vote_accepted","message_id":"b-receive-vote-a","caused_by":"a-vote-b","source":"A","pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":9,"kind":"vote_accepted","message_id":"b-receive-vote-b","caused_by":"b-vote-b","source":"B","pre":"VOTING","post":"OPENING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":10,"kind":"open","open_kind":"QUORUM","pre":"OPENING","post":"OPENING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":11,"kind":"send","message_id":"b-open-a","send":"iamopen:A","pre":"OPENING","post":"OPENING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":8,"kind":"iamopen_accepted","message_id":"a-receive-open","caused_by":"b-open-a","source":"B","pre":"VOTING","post":"JOINING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"A","sequence":9,"kind":"join_restart","pre":"JOINING","post":"JOINING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":12,"kind":"timeout","pre":"OPENING","post":"OPENING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":13,"kind":"timeout","pre":"OPENING","post":"OPENING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":14,"kind":"timeout","pre":"OPENING","post":"OPEN"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"multi","expected_locations":["A","B"],"node":"B","sequence":15,"kind":"complete","pre":"OPEN","post":"OPEN"} diff --git a/lean/disaster-recovery/fixtures/accepted.ndjson b/lean/disaster-recovery/fixtures/accepted.ndjson deleted file mode 100644 index 182111180bfa..000000000000 --- a/lean/disaster-recovery/fixtures/accepted.ndjson +++ /dev/null @@ -1,11 +0,0 @@ -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":1,"kind":"send","message_id":"send-gossip-1","send":"gossip:A","view":1,"seqno":10,"pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":2,"kind":"gossip_accepted","message_id":"recv-gossip-1","caused_by":"send-gossip-1","source":"A","view":1,"seqno":10,"pre":"GOSSIPING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":3,"kind":"send","message_id":"send-vote-1","send":"vote:A","pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":4,"kind":"send","message_id":"send-gossip-2","send":"gossip:A","view":1,"seqno":10,"pre":"VOTING","post":"VOTING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":5,"kind":"vote_accepted","message_id":"recv-vote-1","caused_by":"send-vote-1","source":"A","pre":"VOTING","post":"OPENING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":6,"kind":"open","open_kind":"QUORUM","pre":"OPENING","post":"OPENING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":7,"kind":"timeout","pre":"OPENING","post":"OPENING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":8,"kind":"timeout","pre":"OPENING","post":"OPENING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":9,"kind":"timeout","pre":"OPENING","post":"OPEN"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"fixture","expected_locations":["A"],"node":"A","sequence":10,"kind":"complete","pre":"OPEN","post":"OPEN"} diff --git a/lean/disaster-recovery/fixtures/rejected-cause.ndjson b/lean/disaster-recovery/fixtures/rejected-cause.ndjson deleted file mode 100644 index 41afdf3eb2e5..000000000000 --- a/lean/disaster-recovery/fixtures/rejected-cause.ndjson +++ /dev/null @@ -1,3 +0,0 @@ -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-cause","expected_locations":["A"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-cause","expected_locations":["A"],"node":"A","sequence":1,"kind":"send","message_id":"send-gossip","send":"gossip:A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-cause","expected_locations":["A"],"node":"A","sequence":2,"kind":"vote_accepted","message_id":"receive-vote","caused_by":"send-gossip","source":"A","pre":"GOSSIPING","post":"GOSSIPING"} diff --git a/lean/disaster-recovery/fixtures/rejected.ndjson b/lean/disaster-recovery/fixtures/rejected.ndjson deleted file mode 100644 index 559769f20e18..000000000000 --- a/lean/disaster-recovery/fixtures/rejected.ndjson +++ /dev/null @@ -1,3 +0,0 @@ -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-state","expected_locations":["A"],"node":"A","sequence":0,"kind":"start","pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-state","expected_locations":["A"],"node":"A","sequence":1,"kind":"send","message_id":"send-gossip","send":"gossip:A","view":1,"seqno":1,"pre":"GOSSIPING","post":"GOSSIPING"} -{"version":"ccf.recovery_decision_protocol.trace/1","instance":"bad-state","expected_locations":["A"],"node":"A","sequence":2,"kind":"gossip_accepted","message_id":"receive-gossip","caused_by":"send-gossip","source":"A","view":1,"seqno":1,"pre":"GOSSIPING","post":"OPEN"} diff --git a/lean/disaster-recovery/lakefile.toml b/lean/disaster-recovery/lakefile.toml index 634272dd805a..aa5192985ce0 100644 --- a/lean/disaster-recovery/lakefile.toml +++ b/lean/disaster-recovery/lakefile.toml @@ -5,7 +5,6 @@ defaultTargets = [ "disaster-recovery", "semantic-checks", "canonical-checks", - "trace-checks", "trace-validator", ] @@ -29,10 +28,6 @@ root = "Tests" name = "canonical-checks" root = "CanonicalTests" -[[lean_exe]] -name = "trace-checks" -root = "TraceTests" - [[lean_exe]] name = "trace-validator" root = "TraceMain" diff --git a/tests/infra/recovery_trace_test.py b/tests/infra/recovery_trace_test.py deleted file mode 100644 index 46b52ed00b46..000000000000 --- a/tests/infra/recovery_trace_test.py +++ /dev/null @@ -1,164 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the Apache 2.0 License. - -import json -import pathlib -import tempfile -import unittest -from unittest import mock - -import infra.recovery_trace - -VERSION = "ccf.recovery_decision_protocol.trace/1" -EXPECTED_LOCATIONS = ["A", "B"] - - -def event(node, sequence, kind, **extra): - value = { - "version": VERSION, - "instance": "synthetic", - "expected_locations": EXPECTED_LOCATIONS, - "node": node, - "sequence": sequence, - "kind": kind, - "pre": "GOSSIPING", - "post": "GOSSIPING", - } - value.update(extra) - return value - - -class FakeNode: - def __init__(self, path, name=None): - self.path = path - self.name = name - self.remote = object() if name is not None else None - - def get_logs(self): - return str(self.path), None - - def get_sealing_recovery_location(self): - return {"name": self.name} - - -class FakeNetwork: - def __init__(self, nodes, common_dir): - self.nodes = nodes - self.common_dir = common_dir - - -class RecoveryTraceTest(unittest.TestCase): - def test_extract_linearize_and_validate(self): - with tempfile.TemporaryDirectory() as directory: - root = pathlib.Path(directory) - a_log = root / "a.out" - b_log = root / "b.out" - fixture = ( - pathlib.Path(__file__).resolve().parents[2] - / "lean" - / "disaster-recovery" - / "fixtures" - / "accepted-multinode.ndjson" - ) - with open(fixture, encoding="utf-8") as trace: - events = [json.loads(line) for line in trace if line.strip()] - a_events = [item for item in events if item["node"] == "A"] - b_events = [item for item in events if item["node"] == "B"] - a_log.write_text( - "".join( - f"[info] RDP_TRACE {json.dumps(trace_event)}\n" - for trace_event in a_events - ), - encoding="utf-8", - ) - b_log.write_text( - "".join( - json.dumps({"msg": f"RDP_TRACE {json.dumps(trace_event)}"}) + "\n" - for trace_event in b_events - ), - encoding="utf-8", - ) - network = FakeNetwork( - [FakeNode(b_log), FakeNode(a_log)], - directory, - ) - - extracted = infra.recovery_trace.extract_events(network.nodes) - ordered = infra.recovery_trace.linearize(extracted) - positions = { - item["message_id"]: index - for index, item in enumerate(ordered) - if "message_id" in item - } - for index, item in enumerate(ordered): - if "caused_by" in item: - self.assertLess(positions[item["caused_by"]], index) - - trace_path = infra.recovery_trace.validate_recovery_trace( - network, "synthetic" - ) - self.assertTrue(trace_path.is_file()) - - def test_rejects_non_contiguous_sequence(self): - broken = [ - event("A", 0, "start"), - event("A", 2, "timeout"), - ] - with self.assertRaisesRegex(ValueError, "not contiguous"): - infra.recovery_trace.linearize(broken) - - def test_rejects_unresolved_cause(self): - broken = [ - event("A", 0, "start"), - event( - "A", - 1, - "gossip_accepted", - message_id="receive", - caused_by="missing-send", - source="B", - view=1, - seqno=1, - ), - ] - with self.assertRaisesRegex(ValueError, "no matching send"): - infra.recovery_trace.linearize(broken) - - def test_disabled_validation_preserves_default_tests(self): - with mock.patch.dict( - "os.environ", - {infra.recovery_trace.TRACE_VALIDATOR_ENV: ""}, - clear=False, - ): - self.assertIsNone( - infra.recovery_trace.validate_recovery_trace_if_enabled( - FakeNetwork([], "."), "disabled", "QUORUM" - ) - ) - - def test_waits_for_terminal_scenario_evidence(self): - with tempfile.TemporaryDirectory() as directory: - log_path = pathlib.Path(directory) / "a.out" - events = [ - event("A", 0, "start"), - event("A", 1, "open", open_kind="QUORUM"), - event("A", 2, "complete"), - ] - log_path.write_text( - "".join( - f"RDP_TRACE {json.dumps(trace_event)}\n" for trace_event in events - ), - encoding="utf-8", - ) - network = FakeNetwork( - [FakeNode(log_path, "A")], - directory, - ) - self.assertEqual( - infra.recovery_trace.wait_for_terminal_events(network, "QUORUM", 0.1), - events, - ) - - -if __name__ == "__main__": - unittest.main() From ff554b7a00f994084a562cfa74e8e101594cf39b Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Mon, 31 Aug 2026 18:14:17 +0100 Subject: [PATCH 10/14] Add global recovery semantics and invariants Model retries, in-flight delivery, timeouts, and terminal effects around the canonical protocol. Prove provenance, location consistency, action locality, append-only histories, and well-formedness for all reachable global states. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 46d56f8a-935f-4551-abe3-84bd4f951865 --- lean/disaster-recovery/DisasterRecovery.lean | 2 + .../DisasterRecovery/Protocol/Global.lean | 154 ++++ .../DisasterRecovery/Protocol/Invariants.lean | 860 ++++++++++++++++++ .../DisasterRecovery/Protocol/Model.lean | 2 +- lean/disaster-recovery/README.md | 68 +- 5 files changed, 1069 insertions(+), 17 deletions(-) create mode 100644 lean/disaster-recovery/DisasterRecovery/Protocol/Global.lean create mode 100644 lean/disaster-recovery/DisasterRecovery/Protocol/Invariants.lean diff --git a/lean/disaster-recovery/DisasterRecovery.lean b/lean/disaster-recovery/DisasterRecovery.lean index a414ad18aebe..11da5615d6d3 100644 --- a/lean/disaster-recovery/DisasterRecovery.lean +++ b/lean/disaster-recovery/DisasterRecovery.lean @@ -1,6 +1,8 @@ import DisasterRecovery.Model import DisasterRecovery.Checker import DisasterRecovery.Protocol.Model +import DisasterRecovery.Protocol.Global +import DisasterRecovery.Protocol.Invariants import DisasterRecovery.Protocol.Temporal import DisasterRecovery.Protocol.Refinement import DisasterRecovery.Protocol.Trace diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/Global.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/Global.lean new file mode 100644 index 000000000000..59ccb4376f8f --- /dev/null +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/Global.lean @@ -0,0 +1,154 @@ +import DisasterRecovery.Protocol.Model + +namespace DisasterRecovery.Protocol.Global + +structure Config where + protocol : Protocol.Config + recovered : List (Prod Location TxID) +deriving Repr, BEq + +def Config.Valid (config : Config) : Prop := + config.protocol.isValid = true /\ + config.recovered.map Prod.fst = config.protocol.expectedLocations + +def recoveredTxID (config : Config) (source : Location) : Option TxID := + (config.recovered.find? fun entry => entry.1 == source).map Prod.snd + +inductive Payload where + | gossip (txid : TxID) + | vote + | iAmOpen +deriving Repr, BEq + +structure Envelope where + source : Location + target : Location + payload : Payload + sourceState : NodeState +deriving Repr, BEq + +structure Opening where + node : Location + kind : OpenKind +deriving Repr, BEq + +structure State where + system : SystemState + active : List Location + network : List Envelope := [] + sent : List Envelope := [] + openings : List Opening := [] + restarts : List Location := [] + completed : List Location := [] +deriving Repr, BEq + +inductive Action where + | retry (source : Location) + | deliver (envelope : Envelope) + | timeout (target : Location) +deriving Repr, BEq + +def nodeState (state : State) (node : Location) : Option NodeState := + (state.system.nodes.find? fun entry => entry.1 == node).map Prod.snd + +def messageForEffect + (config : Config) + (source : Location) + (sourceState : NodeState) : Effect -> Option Envelope + | .sendGossip target => do + let txid <- recoveredTxID config source + pure { source, target, payload := .gossip txid, sourceState } + | .sendVote target => + some { source, target, payload := .vote, sourceState } + | .sendIAmOpen target => + some { source, target, payload := .iAmOpen, sourceState } + | _ => none + +def retryMessages + (config : Config) + (source : Location) + (sourceState : NodeState) : List Envelope := + (step config.protocol sourceState .retry).effects.filterMap + (messageForEffect config source sourceState) + +def Envelope.Valid (config : Config) (envelope : Envelope) : Prop := + envelope.sourceState.location = envelope.source /\ + envelope ∈ retryMessages config envelope.source envelope.sourceState + +def eventFor (envelope : Envelope) : Event := + match envelope.payload with + | .gossip txid => .receiveGossip envelope.source txid .accepted + | .vote => .receiveVote envelope.source .accepted + | .iAmOpen => .receiveIAmOpen envelope.source .accepted + +def removeOne [BEq α] (value : α) : List α -> List α + | [] => [] + | head :: tail => + if head == value then tail else head :: removeOne value tail + +def recordEffect (node : Location) (state : State) : Effect -> State + | .opening kind => + { state with openings := { node, kind } :: state.openings } + | .restart _ => + { state with restarts := node :: state.restarts } + | .completed => + { state with completed := node :: state.completed } + | _ => state + +def recordEffects + (node : Location) + (effects : List Effect) + (state : State) : State := + effects.foldl (recordEffect node) state + +def initial (config : Config) (active : List Location) : State := { + system := initialSystem config.protocol + active +} + +def next (config : Config) (state : State) : Action -> Option State + | .retry source => do + guard (state.active.contains source) + let sourceState <- nodeState state source + let messages := retryMessages config source sourceState + guard (!messages.isEmpty) + pure { + state with + network := state.network ++ messages + sent := state.sent ++ messages + } + | .deliver envelope => do + guard (state.network.contains envelope) + guard (state.active.contains envelope.target) + let (system, output) <- + systemStep config.protocol state.system envelope.target + (eventFor envelope) + let delivered := { + state with + system + network := removeOne envelope state.network + } + pure (recordEffects envelope.target output.effects delivered) + | .timeout target => do + guard (state.active.contains target) + let (system, output) <- + systemStep config.protocol state.system target .timeout + guard output.accepted + pure (recordEffects target output.effects { state with system }) + +inductive Reachable (config : Config) : State -> Prop where + | initial + (active : List Location) + (nodup : active.Nodup) + (configured : + forall node, node ∈ active -> + node ∈ config.protocol.expectedLocations) : + Reachable config (Global.initial config active) + | step + {state nextState : State} + {action : Action} + (reachable : Reachable config state) + (transition : next config state action = some nextState) : + Reachable config nextState + +end DisasterRecovery.Protocol.Global diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/Invariants.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/Invariants.lean new file mode 100644 index 000000000000..f957925c3420 --- /dev/null +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/Invariants.lean @@ -0,0 +1,860 @@ +import DisasterRecovery.Protocol.Global +import Mathlib.Tactic + +namespace DisasterRecovery.Protocol.Global + +structure HistoriesActive (state : State) : Prop where + openings : + forall opening, opening ∈ state.openings -> + opening.node ∈ state.active + restarts : + forall node, node ∈ state.restarts -> + node ∈ state.active + completed : + forall node, node ∈ state.completed -> + node ∈ state.active + +structure WellFormed (config : Config) (state : State) : Prop where + nodeKeys : + state.system.nodes.map Prod.fst = + config.protocol.expectedLocations + nodeLocations : + forall entry, entry ∈ state.system.nodes -> + entry.2.location = entry.1 + activeNodup : state.active.Nodup + activeConfigured : + forall node, node ∈ state.active -> + node ∈ config.protocol.expectedLocations + sentValid : + forall envelope, envelope ∈ state.sent -> + envelope.Valid config + sentSourceActive : + forall envelope, envelope ∈ state.sent -> + envelope.source ∈ state.active + networkSent : + forall envelope, envelope ∈ state.network -> + envelope ∈ state.sent + historiesActive : HistoriesActive state + +theorem messageForEffect_source + {config : Config} + {source : Location} + {sourceState : NodeState} + {effect : Effect} + {envelope : Envelope} + (created : + messageForEffect config source sourceState effect = some envelope) : + envelope.source = source /\ + envelope.sourceState = sourceState := by + cases effect with + | sendGossip target => + cases found : recoveredTxID config source with + | none => + simp [messageForEffect, found] at created + | some txid => + simp [messageForEffect, found] at created + rw [←created] + exact ⟨rfl, rfl⟩ + | sendVote target => + simp [messageForEffect] at created + rw [←created] + exact ⟨rfl, rfl⟩ + | sendIAmOpen target => + simp [messageForEffect] at created + rw [←created] + exact ⟨rfl, rfl⟩ + | opening kind => + simp_all [messageForEffect] + | restart chosen => + simp_all [messageForEffect] + | completed => + simp_all [messageForEffect] + | rejected reason => + simp_all [messageForEffect] + +theorem retryMessages_source + {config : Config} + {source : Location} + {sourceState : NodeState} + {envelope : Envelope} + (created : + envelope ∈ retryMessages config source sourceState) : + envelope.source = source /\ + envelope.sourceState = sourceState := by + rw [retryMessages, List.mem_filterMap] at created + rcases created with ⟨effect, _, produced⟩ + exact messageForEffect_source produced + +theorem retryMessages_valid + (config : Config) + (source : Location) + (sourceState : NodeState) + (sourceLocation : sourceState.location = source) : + forall envelope, + envelope ∈ retryMessages config source sourceState -> + envelope.Valid config := by + intro envelope created + rcases retryMessages_source created with + ⟨sourceEq, stateEq⟩ + constructor + · rw [stateEq, sourceEq] + exact sourceLocation + · rw [sourceEq, stateEq] + exact created + +theorem valid_envelope_effect + {config : Config} + {envelope : Envelope} + (valid : envelope.Valid config) : + exists effect, + effect ∈ + (step config.protocol envelope.sourceState .retry).effects /\ + messageForEffect config envelope.source + envelope.sourceState effect = some envelope := by + rcases valid with ⟨_, created⟩ + rw [retryMessages, List.mem_filterMap] at created + exact created + +theorem valid_gossip_uses_recovered_txid + {config : Config} + {envelope : Envelope} + {txid : TxID} + (valid : envelope.Valid config) + (gossip : envelope.payload = .gossip txid) : + recoveredTxID config envelope.source = some txid := by + rcases valid_envelope_effect valid with + ⟨effect, _, created⟩ + cases effect with + | sendGossip target => + cases found : recoveredTxID config envelope.source with + | none => + simp [messageForEffect, found] at created + | some recovered => + simp [messageForEffect, found] at created + rw [←created] at gossip + injection gossip with same + subst recovered + rfl + | sendVote target => + simp [messageForEffect] at created + rw [←created] at gossip + contradiction + | sendIAmOpen target => + simp [messageForEffect] at created + rw [←created] at gossip + contradiction + | opening kind => + simp [messageForEffect] at created + | restart chosen => + simp [messageForEffect] at created + | completed => + simp [messageForEffect] at created + | rejected reason => + simp [messageForEffect] at created + +theorem step_preserves_location + (config : Protocol.Config) + (state : NodeState) + (event : Event) : + (step config state event).state.location = state.location := by + cases event <;> + simp [step, rejected, advance, advanceTimeoutLane] + all_goals repeat first | split | simp_all + +theorem nodeState_location + {state : State} + {node : Location} + {foundState : NodeState} + (locations : + forall entry, entry ∈ state.system.nodes -> + entry.2.location = entry.1) + (found : nodeState state node = some foundState) : + foundState.location = node := by + rw [nodeState, Option.map_eq_some_iff] at found + rcases found with ⟨entry, findEq, stateEq⟩ + have membership : entry ∈ state.system.nodes := + List.mem_of_find?_eq_some findEq + have condition : (entry.1 == node) = true := + List.find?_some + (p := fun entry : Prod Location NodeState => + entry.1 == node) findEq + have keyEq : entry.1 = node := beq_iff_eq.mp condition + rw [←stateEq, locations entry membership, keyEq] + +theorem initial_well_formed + (config : Config) + (active : List Location) + (activeNodup : active.Nodup) + (activeConfigured : + forall node, node ∈ active -> + node ∈ config.protocol.expectedLocations) : + WellFormed config (initial config active) := by + constructor + · simp [Global.initial, initialSystem, Function.comp_def] + · simp [Global.initial, initialSystem, initialNode] + · exact activeNodup + · exact activeConfigured + · simp [Global.initial] + · simp [Global.initial] + · simp [Global.initial] + · constructor <;> simp [Global.initial] + +@[simp] +theorem recordEffects_active + (node : Location) + (effects : List Effect) + (state : State) : + (recordEffects node effects state).active = state.active := by + induction effects generalizing state with + | nil => rfl + | cons effect tail ih => + simp only [recordEffects, List.foldl_cons] + change + (recordEffects node tail (recordEffect node state effect)).active = + state.active + rw [ih] + cases effect <;> rfl + +@[simp] +theorem recordEffects_system + (node : Location) + (effects : List Effect) + (state : State) : + (recordEffects node effects state).system = state.system := by + induction effects generalizing state with + | nil => rfl + | cons effect tail ih => + simp only [recordEffects, List.foldl_cons] + change + (recordEffects node tail (recordEffect node state effect)).system = + state.system + rw [ih] + cases effect <;> rfl + +@[simp] +theorem recordEffects_network + (node : Location) + (effects : List Effect) + (state : State) : + (recordEffects node effects state).network = state.network := by + induction effects generalizing state with + | nil => rfl + | cons effect tail ih => + simp only [recordEffects, List.foldl_cons] + change + (recordEffects node tail (recordEffect node state effect)).network = + state.network + rw [ih] + cases effect <;> rfl + +@[simp] +theorem recordEffects_sent + (node : Location) + (effects : List Effect) + (state : State) : + (recordEffects node effects state).sent = state.sent := by + induction effects generalizing state with + | nil => rfl + | cons effect tail ih => + simp only [recordEffects, List.foldl_cons] + change + (recordEffects node tail (recordEffect node state effect)).sent = + state.sent + rw [ih] + cases effect <;> rfl + +theorem recordEffect_preserves_histories_active + {node : Location} + {state : State} + {effect : Effect} + (wellFormed : HistoriesActive state) + (nodeActive : node ∈ state.active) : + HistoriesActive (recordEffect node state effect) := by + rcases wellFormed with ⟨openings, restarts, completed⟩ + cases effect <;> + constructor <;> + simp_all [recordEffect] + +theorem recordEffects_preserves_histories_active + {node : Location} + {effects : List Effect} + {state : State} + (wellFormed : HistoriesActive state) + (nodeActive : node ∈ state.active) : + HistoriesActive (recordEffects node effects state) := by + induction effects generalizing state with + | nil => exact wellFormed + | cons effect tail ih => + simp only [recordEffects, List.foldl_cons] + apply ih + · exact recordEffect_preserves_histories_active wellFormed nodeActive + · cases effect <;> simpa [recordEffect] using nodeActive + +theorem mem_of_mem_removeOne + [BEq α] + (value member : α) + (values : List α) : + member ∈ removeOne value values -> + member ∈ values := by + induction values with + | nil => simp [removeOne] + | cons head tail ih => + simp only [removeOne] + split + · exact List.mem_cons_of_mem head + · intro membership + rw [List.mem_cons] at membership ⊢ + exact membership.imp_right ih + +theorem mem_openings_recordEffect + {node : Location} + {state : State} + {effect : Effect} + {opening : Opening} + (membership : opening ∈ state.openings) : + opening ∈ (recordEffect node state effect).openings := by + cases effect <;> simp_all [recordEffect] + +theorem mem_restarts_recordEffect + {node : Location} + {state : State} + {effect : Effect} + {restart : Location} + (membership : restart ∈ state.restarts) : + restart ∈ (recordEffect node state effect).restarts := by + cases effect <;> simp_all [recordEffect] + +theorem mem_completed_recordEffect + {node : Location} + {state : State} + {effect : Effect} + {completed : Location} + (membership : completed ∈ state.completed) : + completed ∈ (recordEffect node state effect).completed := by + cases effect <;> simp_all [recordEffect] + +theorem mem_openings_recordEffects + {node : Location} + {state : State} + {effects : List Effect} + {opening : Opening} + (membership : opening ∈ state.openings) : + opening ∈ (recordEffects node effects state).openings := by + induction effects generalizing state with + | nil => exact membership + | cons effect tail ih => + simp only [recordEffects, List.foldl_cons] + exact ih (mem_openings_recordEffect membership) + +theorem mem_restarts_recordEffects + {node : Location} + {state : State} + {effects : List Effect} + {restart : Location} + (membership : restart ∈ state.restarts) : + restart ∈ (recordEffects node effects state).restarts := by + induction effects generalizing state with + | nil => exact membership + | cons effect tail ih => + simp only [recordEffects, List.foldl_cons] + exact ih (mem_restarts_recordEffect membership) + +theorem mem_completed_recordEffects + {node : Location} + {state : State} + {effects : List Effect} + {completed : Location} + (membership : completed ∈ state.completed) : + completed ∈ (recordEffects node effects state).completed := by + induction effects generalizing state with + | nil => exact membership + | cons effect tail ih => + simp only [recordEffects, List.foldl_cons] + exact ih (mem_completed_recordEffect membership) + +theorem replaceNode_keys + (target : Location) + (nextState : NodeState) + (nodes : List (Prod Location NodeState)) : + (replaceNode target nextState nodes).map Prod.fst = + nodes.map Prod.fst := by + induction nodes with + | nil => rfl + | cons entry tail ih => + simp only [replaceNode, List.map_cons] + split + · + rename_i condition + have same : entry.1 = target := beq_iff_eq.mp condition + simp only [List.cons.injEq] + constructor + · exact same.symm + · simpa [replaceNode] using ih + · + simp only [List.cons.injEq, true_and] + simpa [replaceNode] using ih + +theorem replaceNode_locations + (target : Location) + (nextState : NodeState) + (nodes : List (Prod Location NodeState)) + (locations : + forall entry, entry ∈ nodes -> + entry.2.location = entry.1) + (nextLocation : nextState.location = target) : + forall entry, entry ∈ replaceNode target nextState nodes -> + entry.2.location = entry.1 := by + intro entry membership + rw [replaceNode, List.mem_map] at membership + rcases membership with ⟨previous, previousMember, rfl⟩ + split + · exact nextLocation + · exact locations previous previousMember + +theorem findNode_replaceNode_ne + (target other : Location) + (nextState : NodeState) + (nodes : List (Prod Location NodeState)) + (different : other ≠ target) : + ((replaceNode target nextState nodes).find? + fun entry => entry.1 == other).map Prod.snd = + (nodes.find? fun entry => entry.1 == other).map Prod.snd := by + let replace : Prod Location NodeState -> Prod Location NodeState := + fun entry => + if entry.1 == target then (target, nextState) else entry + change + Option.map Prod.snd + (List.find? (fun entry => entry.1 == other) + (nodes.map replace)) = + Option.map Prod.snd + (List.find? (fun entry => entry.1 == other) nodes) + rw [List.find?_map] + have predicate : + ((fun entry : Prod Location NodeState => entry.1 == other) ∘ + replace) = + (fun entry => entry.1 == other) := by + funext entry + by_cases atTarget : entry.1 = target + · simp [replace, atTarget] + · simp [replace, atTarget] + rw [predicate] + cases found : + List.find? (fun entry : Prod Location NodeState => + entry.1 == other) nodes with + | none => simp + | some entry => + have condition : + (entry.1 == other) = true := + List.find?_some + (p := fun entry : Prod Location NodeState => + entry.1 == other) found + have entryOther : entry.1 = other := + beq_iff_eq.mp condition + have notTarget : entry.1 ≠ target := by + simpa [entryOther] using different + simp [replace, notTarget] + +theorem systemStep_node_keys_eq + {config : Protocol.Config} + {before after : SystemState} + {target : Location} + {event : Event} + {output : StepOutput} + (transition : + systemStep config before target event = some (after, output)) : + after.nodes.map Prod.fst = before.nodes.map Prod.fst := by + simp [systemStep, Option.bind_eq_some_iff] at transition + rcases transition with ⟨node, _, stateEq, _⟩ + rw [←stateEq] + exact replaceNode_keys target + (step config node event).state before.nodes + +theorem systemStep_preserves_node_locations + {config : Protocol.Config} + {before after : SystemState} + {target : Location} + {event : Event} + {output : StepOutput} + (locations : + forall entry, entry ∈ before.nodes -> + entry.2.location = entry.1) + (transition : + systemStep config before target event = some (after, output)) : + forall entry, entry ∈ after.nodes -> + entry.2.location = entry.1 := by + simp [systemStep, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨node, ⟨key, found⟩, stateEq, _⟩ + rw [←stateEq] + apply replaceNode_locations + · exact locations + · calc + (step config node event).state.location = + node.location := step_preserves_location config node event + _ = key := + locations (key, node) (List.mem_of_find?_eq_some found) + _ = target := + beq_iff_eq.mp + (List.find?_some + (p := fun entry : Prod Location NodeState => + entry.1 == target) found) + +theorem systemStep_other_node_eq + {config : Protocol.Config} + {before after : SystemState} + {target other : Location} + {event : Event} + {output : StepOutput} + (different : other ≠ target) + (transition : + systemStep config before target event = some (after, output)) : + (after.nodes.find? fun entry => entry.1 == other).map Prod.snd = + (before.nodes.find? fun entry => entry.1 == other).map Prod.snd := by + simp [systemStep, Option.bind_eq_some_iff] at transition + rcases transition with ⟨node, _, stateEq, _⟩ + rw [←stateEq] + exact findNode_replaceNode_ne target other + (step config node event).state before.nodes different + +theorem next_active_eq + {config : Config} + {before after : State} + {action : Action} + (transition : next config before action = some after) : + after.active = before.active := by + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, sourceState, _, _, rfl⟩ + rfl + | deliver envelope => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, _, system, output, _, rfl⟩ + exact recordEffects_active envelope.target output.effects _ + | timeout target => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, system, output, _, _, rfl⟩ + exact recordEffects_active target output.effects _ + +theorem next_node_keys_eq + {config : Config} + {before after : State} + {action : Action} + (transition : next config before action = some after) : + after.system.nodes.map Prod.fst = + before.system.nodes.map Prod.fst := by + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, sourceState, _, _, rfl⟩ + rfl + | deliver envelope => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, _, system, output, systemStep, rfl⟩ + simpa using systemStep_node_keys_eq systemStep + | timeout target => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, system, output, systemStep, _, rfl⟩ + simpa using systemStep_node_keys_eq systemStep + +theorem retry_system_eq + {config : Config} + {before after : State} + {source : Location} + (transition : next config before (.retry source) = some after) : + after.system = before.system := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, sourceState, _, _, rfl⟩ + rfl + +theorem deliver_network_eq + {config : Config} + {before after : State} + {envelope : Envelope} + (transition : next config before (.deliver envelope) = some after) : + after.network = removeOne envelope before.network := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, _, system, output, _, rfl⟩ + exact recordEffects_network envelope.target output.effects _ + +theorem timeout_network_eq + {config : Config} + {before after : State} + {target : Location} + (transition : next config before (.timeout target) = some after) : + after.network = before.network := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, _, _, rfl⟩ + exact recordEffects_network target output.effects _ + +theorem deliver_other_node_eq + {config : Config} + {before after : State} + {envelope : Envelope} + {other : Location} + (different : other ≠ envelope.target) + (transition : next config before (.deliver envelope) = some after) : + nodeState after other = nodeState before other := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, _, system, output, systemStep, stateEq⟩ + rw [←stateEq] + simp only [nodeState, recordEffects_system] + exact systemStep_other_node_eq different systemStep + +theorem timeout_other_node_eq + {config : Config} + {before after : State} + {target other : Location} + (different : other ≠ target) + (transition : next config before (.timeout target) = some after) : + nodeState after other = nodeState before other := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, systemStep, _, stateEq⟩ + rw [←stateEq] + simp only [nodeState, recordEffects_system] + exact systemStep_other_node_eq different systemStep + +theorem next_sent_extends + {config : Config} + {before after : State} + {action : Action} + (transition : next config before action = some after) : + exists added, after.sent = before.sent ++ added := by + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, sourceState, _, _, rfl⟩ + exact ⟨retryMessages config source sourceState, rfl⟩ + | deliver envelope => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, _, system, output, _, rfl⟩ + refine ⟨[], ?_⟩ + simp + | timeout target => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, _, _, rfl⟩ + refine ⟨[], ?_⟩ + simp + +theorem next_openings_monotonic + {config : Config} + {before after : State} + {action : Action} + (transition : next config before action = some after) : + forall opening, opening ∈ before.openings -> + opening ∈ after.openings := by + intro opening membership + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, sourceState, _, _, rfl⟩ + exact membership + | deliver envelope => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, _, system, output, _, rfl⟩ + exact mem_openings_recordEffects membership + | timeout target => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, _, _, rfl⟩ + exact mem_openings_recordEffects membership + +theorem next_restarts_monotonic + {config : Config} + {before after : State} + {action : Action} + (transition : next config before action = some after) : + forall restart, restart ∈ before.restarts -> + restart ∈ after.restarts := by + intro restart membership + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, sourceState, _, _, rfl⟩ + exact membership + | deliver envelope => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, _, system, output, _, rfl⟩ + exact mem_restarts_recordEffects membership + | timeout target => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, _, _, rfl⟩ + exact mem_restarts_recordEffects membership + +theorem next_completed_monotonic + {config : Config} + {before after : State} + {action : Action} + (transition : next config before action = some after) : + forall completed, completed ∈ before.completed -> + completed ∈ after.completed := by + intro completed membership + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, sourceState, _, _, rfl⟩ + exact membership + | deliver envelope => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, _, system, output, _, rfl⟩ + exact mem_completed_recordEffects membership + | timeout target => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, _, _, rfl⟩ + exact mem_completed_recordEffects membership + +theorem retry_preserves_well_formed + {config : Config} + {before after : State} + {source : Location} + (wellFormed : WellFormed config before) + (transition : next config before (.retry source) = some after) : + WellFormed config after := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨sourceActive, sourceState, found, _, stateEq⟩ + have sourceLocation : sourceState.location = source := + nodeState_location wellFormed.nodeLocations found + rw [←stateEq] + constructor + · exact wellFormed.nodeKeys + · exact wellFormed.nodeLocations + · exact wellFormed.activeNodup + · exact wellFormed.activeConfigured + · intro envelope membership + rw [List.mem_append] at membership + rcases membership with membership | membership + · exact wellFormed.sentValid envelope membership + · exact retryMessages_valid config source sourceState + sourceLocation envelope membership + · intro envelope membership + rw [List.mem_append] at membership + rcases membership with membership | membership + · exact wellFormed.sentSourceActive envelope membership + · rw [(retryMessages_source membership).1] + exact sourceActive + · intro envelope membership + rw [List.mem_append] at membership ⊢ + rcases membership with membership | membership + · exact Or.inl (wellFormed.networkSent envelope membership) + · exact Or.inr membership + · constructor + · exact wellFormed.historiesActive.openings + · exact wellFormed.historiesActive.restarts + · exact wellFormed.historiesActive.completed + +theorem deliver_preserves_well_formed + {config : Config} + {before after : State} + {envelope : Envelope} + (wellFormed : WellFormed config before) + (transition : next config before (.deliver envelope) = some after) : + WellFormed config after := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, targetActive, system, output, systemStep, stateEq⟩ + rw [←stateEq] + constructor + · rw [recordEffects_system] + exact (systemStep_node_keys_eq systemStep).trans + wellFormed.nodeKeys + · rw [recordEffects_system] + exact systemStep_preserves_node_locations + wellFormed.nodeLocations systemStep + · simpa using wellFormed.activeNodup + · simpa using wellFormed.activeConfigured + · intro sent membership + rw [recordEffects_sent] at membership + exact wellFormed.sentValid sent membership + · intro sent membership + rw [recordEffects_sent] at membership + rw [recordEffects_active] + exact wellFormed.sentSourceActive sent membership + · intro pending membership + rw [recordEffects_network] at membership + rw [recordEffects_sent] + exact wellFormed.networkSent pending + (mem_of_mem_removeOne envelope pending before.network membership) + · apply recordEffects_preserves_histories_active + · constructor + · exact wellFormed.historiesActive.openings + · exact wellFormed.historiesActive.restarts + · exact wellFormed.historiesActive.completed + · exact targetActive + +theorem timeout_preserves_well_formed + {config : Config} + {before after : State} + {target : Location} + (wellFormed : WellFormed config before) + (transition : next config before (.timeout target) = some after) : + WellFormed config after := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨targetActive, system, output, systemStep, _, stateEq⟩ + rw [←stateEq] + constructor + · rw [recordEffects_system] + exact (systemStep_node_keys_eq systemStep).trans + wellFormed.nodeKeys + · rw [recordEffects_system] + exact systemStep_preserves_node_locations + wellFormed.nodeLocations systemStep + · simpa using wellFormed.activeNodup + · simpa using wellFormed.activeConfigured + · intro sent membership + rw [recordEffects_sent] at membership + exact wellFormed.sentValid sent membership + · intro sent membership + rw [recordEffects_sent] at membership + rw [recordEffects_active] + exact wellFormed.sentSourceActive sent membership + · intro pending membership + rw [recordEffects_network] at membership + rw [recordEffects_sent] + exact wellFormed.networkSent pending membership + · apply recordEffects_preserves_histories_active + · constructor + · exact wellFormed.historiesActive.openings + · exact wellFormed.historiesActive.restarts + · exact wellFormed.historiesActive.completed + · exact targetActive + +theorem next_preserves_well_formed + {config : Config} + {before after : State} + {action : Action} + (wellFormed : WellFormed config before) + (transition : next config before action = some after) : + WellFormed config after := by + cases action with + | retry source => + exact retry_preserves_well_formed wellFormed transition + | deliver envelope => + exact deliver_preserves_well_formed wellFormed transition + | timeout target => + exact timeout_preserves_well_formed wellFormed transition + +theorem reachable_well_formed + {config : Config} + {state : State} + (reachable : Reachable config state) : + WellFormed config state := by + induction reachable with + | initial active nodup configured => + exact initial_well_formed config active nodup configured + | step reachable transition wellFormed => + exact next_preserves_well_formed wellFormed transition + +end DisasterRecovery.Protocol.Global diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/Model.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/Model.lean index a5dd71dce88c..df8226c0efc7 100644 --- a/lean/disaster-recovery/DisasterRecovery/Protocol/Model.lean +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/Model.lean @@ -253,7 +253,7 @@ def step (config : Config) (state : NodeState) : Event -> StepOutput | .joining | .open => [] { state, effects } -private def replaceNode +def replaceNode (target : Location) (next : NodeState) (nodes : List (Prod Location NodeState)) : diff --git a/lean/disaster-recovery/README.md b/lean/disaster-recovery/README.md index ca6e8c42b23e..a325c3880e71 100644 --- a/lean/disaster-recovery/README.md +++ b/lean/disaster-recovery/README.md @@ -21,15 +21,18 @@ legacy predicate valuations. The canonical Lean model is separate because the production C++ protocol intentionally differs from the legacy model in several places. It has formal -phase-refinement and fairness-aware progress results, plus a versioned trace -validator designed for future committed C++ instrumentation. +phase-refinement and fairness-aware progress results. A small global semantics +adds active nodes, in-flight messages, immutable send history, and terminal +effects around that same canonical transition function. The versioned trace +validator also replays committed C++ instrumentation against the canonical +model. -The initial migration is approximately 4,000 lines across 23 new files: +The migration is approximately 5,000 lines across 25 new files: | Area | Files | Approximate lines | Purpose | | ----------------------------------------- | ----: | ----------------: | --------------------------------------------------------------- | | Exact legacy Lean model | 4 | 650 | Stateright state, actions, timers, network, predicates, and BFS | -| Canonical protocol and proofs | 4 | 1,000 | C++ behavior, phase refinement, quorum, and temporal proofs | +| Canonical protocol and proofs | 6 | 1,900 | C++ behavior, global semantics, invariants, and temporal proofs | | Trace validation | 5 | 760 | NDJSON format, deterministic replay, and CLI | | Equivalence tooling | 2 | 710 | Rust graph exporter and bidirectional Rust/Lean comparison | | Project, documentation, and configuration | 6 | 530 | Lake/Mathlib setup, entry points, and documentation | @@ -37,18 +40,20 @@ The initial migration is approximately 4,000 lines across 23 new files: ### Principal files -| File | Lines | Role | -| -------------------------------------------------------------------------------------------- | ----: | -------------------------------------------------------------- | -| [`DisasterRecovery/Model.lean`](DisasterRecovery/Model.lean) | 392 | Exact executable legacy semantics | -| [`DisasterRecovery/Checker.lean`](DisasterRecovery/Checker.lean) | 152 | BFS enumeration, property checking, and canonical graph export | -| [`DisasterRecovery/Protocol/Model.lean`](DisasterRecovery/Protocol/Model.lean) | 286 | Production-oriented C++ protocol model | -| [`DisasterRecovery/Protocol/Refinement.lean`](DisasterRecovery/Protocol/Refinement.lean) | 332 | Canonical-to-legacy formal phase refinement | -| [`DisasterRecovery/Protocol/Temporal.lean`](DisasterRecovery/Protocol/Temporal.lean) | 230 | Execution streams, fairness, safety, and progress proofs | -| [`DisasterRecovery/Protocol/Trace/Format.lean`](DisasterRecovery/Protocol/Trace/Format.lean) | 141 | Versioned NDJSON types and parser | -| [`DisasterRecovery/Protocol/Trace/Replay.lean`](DisasterRecovery/Protocol/Trace/Replay.lean) | 452 | Deterministic implementation-trace replay | -| [`compare.py`](compare.py) | 343 | Exhaustive canonical graph comparison | -| [`../../tla/disaster-recovery/src/export.rs`](../../tla/disaster-recovery/src/export.rs) | 370 | Canonical export of the actual Stateright model | -| [`TRACE_FORMAT_V1.md`](TRACE_FORMAT_V1.md) | 145 | Contract for future committed C++ trace events | +| File | Lines | Role | +| -------------------------------------------------------------------------------------------- | ----: | ------------------------------------------------------------------ | +| [`DisasterRecovery/Model.lean`](DisasterRecovery/Model.lean) | 392 | Exact executable legacy semantics | +| [`DisasterRecovery/Checker.lean`](DisasterRecovery/Checker.lean) | 152 | BFS enumeration, property checking, and canonical graph export | +| [`DisasterRecovery/Protocol/Model.lean`](DisasterRecovery/Protocol/Model.lean) | 286 | Production-oriented C++ protocol model | +| [`DisasterRecovery/Protocol/Refinement.lean`](DisasterRecovery/Protocol/Refinement.lean) | 332 | Canonical-to-legacy formal phase refinement | +| [`DisasterRecovery/Protocol/Temporal.lean`](DisasterRecovery/Protocol/Temporal.lean) | 230 | Execution streams, fairness, safety, and progress proofs | +| [`DisasterRecovery/Protocol/Global.lean`](DisasterRecovery/Protocol/Global.lean) | 154 | Global active-node, network, send-history, and effect semantics | +| [`DisasterRecovery/Protocol/Invariants.lean`](DisasterRecovery/Protocol/Invariants.lean) | 860 | Global provenance, locality, monotonicity, and reachability proofs | +| [`DisasterRecovery/Protocol/Trace/Format.lean`](DisasterRecovery/Protocol/Trace/Format.lean) | 141 | Versioned NDJSON types and parser | +| [`DisasterRecovery/Protocol/Trace/Replay.lean`](DisasterRecovery/Protocol/Trace/Replay.lean) | 452 | Deterministic implementation-trace replay | +| [`compare.py`](compare.py) | 343 | Exhaustive canonical graph comparison | +| [`../../tla/disaster-recovery/src/export.rs`](../../tla/disaster-recovery/src/export.rs) | 370 | Canonical export of the actual Stateright model | +| [`TRACE_FORMAT_V1.md`](TRACE_FORMAT_V1.md) | 145 | Contract for future committed C++ trace events | The migration also updates the existing Stateright CLI and documentation, adds weekly exhaustive verification, and adds @@ -139,6 +144,37 @@ These are theorem-checked for arbitrary configurations and states satisfying their explicit premises. The progress theorem assumes weak fairness; the safety theorems do not. +### Kernel-checked global foundations + +`DisasterRecovery.Protocol.Global` permits only retry, delivery of an in-flight +message, and aligned local timeout actions. Every delivery is therefore tied to +a prior modeled send rather than an arbitrary receive input. Its reachability +relation requires the active locations to be unique and configured. + +`DisasterRecovery.Protocol.Global.WellFormed` records the foundational +invariants needed by the remaining distributed proofs. The principal results +are: + +- `reachable_well_formed`: every globally reachable state has the configured + node keys and matching state locations, unique configured active nodes, valid + sent-message provenance, only previously sent messages in flight, active + senders, and terminal histories attributed to active nodes; +- `valid_envelope_effect` and `valid_gossip_uses_recovered_txid`: every + envelope comes from a retry effect over its captured source state, and every + gossip payload is the recovered TxID configured for that source; +- `retry_system_eq`, `deliver_other_node_eq`, and + `timeout_other_node_eq`: retry does not mutate protocol state, while delivery + and timeout mutate at most their target node; +- `deliver_network_eq`, `timeout_network_eq`, and `next_sent_extends`: delivery + removes exactly one matching envelope, timeout leaves the network unchanged, + and sent-message history is append-only; and +- `next_openings_monotonic`, `next_restarts_monotonic`, and + `next_completed_monotonic`: observed terminal effects are never removed. + +These are global semantic foundations, not yet the planned quorum-path +uniqueness, committed-prefix preservation, or fair global termination +theorems. + ### Kernel-checked refinement properties | Property | Lean theorem(s) | From 82355e63deb1fa973dbcd7b15f7935c29d42b9d3 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Mon, 31 Aug 2026 19:48:36 +0100 Subject: [PATCH 11/14] Prove global recovery safety properties Prove unbounded quorum-opener uniqueness from vote provenance and strict-majority intersection. Prove maximum selection and committed-prefix preservation under explicit full-gossip and durability premises. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 46d56f8a-935f-4551-abe3-84bd4f951865 --- lean/disaster-recovery/DisasterRecovery.lean | 2 + .../DisasterRecovery/Protocol/Committed.lean | 258 +++ .../DisasterRecovery/Protocol/Global.lean | 26 +- .../DisasterRecovery/Protocol/Invariants.lean | 119 +- .../DisasterRecovery/Protocol/Model.lean | 24 +- .../DisasterRecovery/Protocol/Quorum.lean | 1467 +++++++++++++++++ lean/disaster-recovery/README.md | 41 +- 7 files changed, 1870 insertions(+), 67 deletions(-) create mode 100644 lean/disaster-recovery/DisasterRecovery/Protocol/Committed.lean create mode 100644 lean/disaster-recovery/DisasterRecovery/Protocol/Quorum.lean diff --git a/lean/disaster-recovery/DisasterRecovery.lean b/lean/disaster-recovery/DisasterRecovery.lean index 11da5615d6d3..6a7e6aea83b9 100644 --- a/lean/disaster-recovery/DisasterRecovery.lean +++ b/lean/disaster-recovery/DisasterRecovery.lean @@ -3,6 +3,8 @@ import DisasterRecovery.Checker import DisasterRecovery.Protocol.Model import DisasterRecovery.Protocol.Global import DisasterRecovery.Protocol.Invariants +import DisasterRecovery.Protocol.Quorum +import DisasterRecovery.Protocol.Committed import DisasterRecovery.Protocol.Temporal import DisasterRecovery.Protocol.Refinement import DisasterRecovery.Protocol.Trace diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/Committed.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/Committed.lean new file mode 100644 index 000000000000..aeaec563bea1 --- /dev/null +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/Committed.lean @@ -0,0 +1,258 @@ +import DisasterRecovery.Protocol.Quorum +import Mathlib.Tactic + +namespace DisasterRecovery.Protocol + +namespace TxID + +def PrefixOf (left right : TxID) : Prop := + left.view < right.view \/ + (left.view = right.view /\ left.seqno <= right.seqno) + +theorem prefix_refl (txid : TxID) : PrefixOf txid txid := by + simp [PrefixOf] + +theorem prefix_trans + {first second third : TxID} + (firstSecond : PrefixOf first second) + (secondThird : PrefixOf second third) : + PrefixOf first third := by + simp [PrefixOf] at firstSecond secondThird ⊢ + omega + +end TxID + +namespace Global + +theorem prefix_of_score_true + (leftName rightName : Location) + (left right : TxID) + (score : + txScoreGreater leftName left rightName right = true) : + TxID.PrefixOf right left := by + simp [txScoreGreater] at score + simp [TxID.PrefixOf] + omega + +theorem prefix_of_score_false + (leftName rightName : Location) + (left right : TxID) + (score : + txScoreGreater leftName left rightName right = false) : + TxID.PrefixOf left right := by + simp [txScoreGreater] at score + simp [TxID.PrefixOf] + omega + +theorem current_prefix_selectMaximum + (current candidate : Prod Location TxID) : + TxID.PrefixOf current.2 + (selectMaximum current candidate).2 := by + unfold selectMaximum + split + · rename_i score + exact prefix_of_score_true + candidate.1 current.1 candidate.2 current.2 score + · exact TxID.prefix_refl current.2 + +theorem candidate_prefix_selectMaximum + (current candidate : Prod Location TxID) : + TxID.PrefixOf candidate.2 + (selectMaximum current candidate).2 := by + unfold selectMaximum + split + · exact TxID.prefix_refl candidate.2 + · rename_i score + exact prefix_of_score_false + candidate.1 current.1 candidate.2 current.2 + (Bool.eq_false_iff.mpr score) + +theorem foldl_selectMaximum_upper_bound + (current member : Prod Location TxID) + (tail : List (Prod Location TxID)) + (membership : member = current \/ member ∈ tail) : + TxID.PrefixOf member.2 + (tail.foldl selectMaximum current).2 := by + induction tail generalizing current member with + | nil => + simp at membership + subst member + exact TxID.prefix_refl current.2 + | cons candidate rest ih => + simp only [List.foldl_cons] + rcases membership with currentMember | tailMember + · subst member + exact TxID.prefix_trans + (current_prefix_selectMaximum current candidate) + (ih (selectMaximum current candidate) + (selectMaximum current candidate) (Or.inl rfl)) + · rw [List.mem_cons] at tailMember + rcases tailMember with candidateMember | restMember + · subst member + exact TxID.prefix_trans + (candidate_prefix_selectMaximum current candidate) + (ih (selectMaximum current candidate) + (selectMaximum current candidate) (Or.inl rfl)) + · exact ih (selectMaximum current candidate) member + (Or.inr restMember) + +theorem maximumGossip_upper_bound + {gossips : List (Prod Location TxID)} + {selected member : Prod Location TxID} + (maximum : maximumGossip gossips = some selected) + (membership : member ∈ gossips) : + TxID.PrefixOf member.2 selected.2 := by + cases gossips with + | nil => simp at membership + | cons head tail => + simp [maximumGossip] at maximum + rw [←maximum] + apply foldl_selectMaximum_upper_bound head member tail + simpa using membership + +theorem foldl_selectMaximum_mem + (current : Prod Location TxID) + (tail : List (Prod Location TxID)) : + tail.foldl selectMaximum current ∈ current :: tail := by + induction tail generalizing current with + | nil => simp + | cons candidate rest ih => + simp only [List.foldl_cons] + have selected : + selectMaximum current candidate = current \/ + selectMaximum current candidate = candidate := by + unfold selectMaximum + split <;> simp + have member := + ih (selectMaximum current candidate) + rw [List.mem_cons] at member + rcases member with currentMember | restMember + · rw [currentMember] + rcases selected with selected | selected + · simp [selected] + · simp [selected] + · simp [restMember] + +theorem maximumGossip_mem + {gossips : List (Prod Location TxID)} + {selected : Prod Location TxID} + (maximum : maximumGossip gossips = some selected) : + selected ∈ gossips := by + cases gossips with + | nil => simp [maximumGossip] at maximum + | cons head tail => + simp [maximumGossip] at maximum + rw [←maximum] + exact foldl_selectMaximum_mem head tail + +theorem recoveredTxID_of_mem + {config : Config} + {location : Location} + {txid : TxID} + (valid : config.Valid) + (membership : (location, txid) ∈ config.recovered) : + recoveredTxID config location = some txid := by + have keysNodup : (config.recovered.map Prod.fst).Nodup := by + rw [valid.2.2] + exact valid.2.1 + unfold recoveredTxID + cases found : + config.recovered.find? fun entry => entry.1 == location with + | none => + rw [List.find?_eq_none] at found + exact False.elim + (found (location, txid) membership (by simp)) + | some entry => + have foundMember : entry ∈ config.recovered := + List.mem_of_find?_eq_some found + have foundLocation : entry.1 = location := + beq_iff_eq.mp + (List.find?_some + (p := fun entry : Prod Location TxID => + entry.1 == location) found) + have same : + entry = (location, txid) := + eq_of_key_eq keysNodup foundMember membership foundLocation + simp [same] + +def FullGossipSelection + (config : Config) + (state : State) + (opener : Location) : Prop := + exists vote, + vote ∈ state.sent /\ + vote.payload = .vote /\ + vote.target = opener /\ + forall gossip, + gossip ∈ vote.sourceState.gossips <-> + gossip ∈ config.recovered + +def DurableCommit (config : Config) (committed : TxID) : Prop := + exists location txid, + (location, txid) ∈ config.recovered /\ + TxID.PrefixOf committed txid + +theorem full_gossip_selection_preserves_commit + {config : Config} + {state : State} + {opener : Location} + {committed : TxID} + (reachable : Reachable config state) + (full : FullGossipSelection config state opener) + (durable : DurableCommit config committed) : + exists recovered, + recoveredTxID config opener = some recovered /\ + TxID.PrefixOf committed recovered := by + have configValid := reachable_config_valid reachable + have wellFormed := reachable_well_formed reachable + have invariant := reachable_quorum_invariant reachable + rcases full with + ⟨vote, sent, payload, target, complete⟩ + have voteState := + retry_vote_state (wellFormed.sentValid vote sent) payload + rcases invariant.sentVotesSelected vote sent payload with + ⟨selectedTarget, selectedTxID, choice, selected⟩ + have selectedTargetEq : selectedTarget = vote.target := + Option.some.inj (choice.symm.trans voteState.2) + rw [selectedTargetEq, target] at selected + rcases durable with + ⟨durableLocation, durableTxID, durableMember, committedDurable⟩ + have durableGossip : + (durableLocation, durableTxID) ∈ vote.sourceState.gossips := + (complete (durableLocation, durableTxID)).2 durableMember + have durableMaximum := + maximumGossip_upper_bound selected durableGossip + have selectedGossip : + (opener, selectedTxID) ∈ vote.sourceState.gossips := + maximumGossip_mem selected + have selectedRecovered : + (opener, selectedTxID) ∈ config.recovered := + (complete (opener, selectedTxID)).1 selectedGossip + exact + ⟨selectedTxID, + recoveredTxID_of_mem configValid selectedRecovered, + TxID.prefix_trans committedDurable durableMaximum⟩ + +/-- +Quorum opening scopes the result to an actual decision, while the separate +`FullGossipSelection` premise carries the completeness requirement. Quorum +opening alone does not imply complete gossip because voting may follow a +gossip timeout. +-/ +theorem quorum_open_preserves_commit + {config : Config} + {state : State} + {opener : Location} + {committed : TxID} + (reachable : Reachable config state) + (_opened : QuorumOpened state opener) + (full : FullGossipSelection config state opener) + (durable : DurableCommit config committed) : + exists recovered, + recoveredTxID config opener = some recovered /\ + TxID.PrefixOf committed recovered := + full_gossip_selection_preserves_commit reachable full durable + +end Global + +end DisasterRecovery.Protocol diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/Global.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/Global.lean index 59ccb4376f8f..c29b83a1305f 100644 --- a/lean/disaster-recovery/DisasterRecovery/Protocol/Global.lean +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/Global.lean @@ -9,6 +9,7 @@ deriving Repr, BEq def Config.Valid (config : Config) : Prop := config.protocol.isValid = true /\ + config.protocol.expectedLocations.Nodup /\ config.recovered.map Prod.fst = config.protocol.expectedLocations def recoveredTxID (config : Config) (source : Location) : Option TxID := @@ -18,18 +19,19 @@ inductive Payload where | gossip (txid : TxID) | vote | iAmOpen -deriving Repr, BEq +deriving Repr, BEq, ReflBEq, LawfulBEq structure Envelope where source : Location target : Location payload : Payload sourceState : NodeState -deriving Repr, BEq +deriving Repr, BEq, ReflBEq, LawfulBEq structure Opening where node : Location kind : OpenKind + state : NodeState deriving Repr, BEq structure State where @@ -86,9 +88,15 @@ def removeOne [BEq α] (value : α) : List α -> List α | head :: tail => if head == value then tail else head :: removeOne value tail -def recordEffect (node : Location) (state : State) : Effect -> State +def recordEffect + (node : Location) + (nodeState : NodeState) + (state : State) : Effect -> State | .opening kind => - { state with openings := { node, kind } :: state.openings } + { + state with + openings := { node, kind, state := nodeState } :: state.openings + } | .restart _ => { state with restarts := node :: state.restarts } | .completed => @@ -97,9 +105,10 @@ def recordEffect (node : Location) (state : State) : Effect -> State def recordEffects (node : Location) + (nodeState : NodeState) (effects : List Effect) (state : State) : State := - effects.foldl (recordEffect node) state + effects.foldl (recordEffect node nodeState) state def initial (config : Config) (active : List Location) : State := { system := initialSystem config.protocol @@ -128,17 +137,20 @@ def next (config : Config) (state : State) : Action -> Option State system network := removeOne envelope state.network } - pure (recordEffects envelope.target output.effects delivered) + pure + (recordEffects envelope.target output.state output.effects delivered) | .timeout target => do guard (state.active.contains target) let (system, output) <- systemStep config.protocol state.system target .timeout guard output.accepted - pure (recordEffects target output.effects { state with system }) + pure + (recordEffects target output.state output.effects { state with system }) inductive Reachable (config : Config) : State -> Prop where | initial (active : List Location) + (valid : config.Valid) (nodup : active.Nodup) (configured : forall node, node ∈ active -> diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/Invariants.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/Invariants.lean index f957925c3420..3fa11e0bcdd1 100644 --- a/lean/disaster-recovery/DisasterRecovery/Protocol/Invariants.lean +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/Invariants.lean @@ -18,6 +18,7 @@ structure WellFormed (config : Config) (state : State) : Prop where nodeKeys : state.system.nodes.map Prod.fst = config.protocol.expectedLocations + nodeKeysNodup : (state.system.nodes.map Prod.fst).Nodup nodeLocations : forall entry, entry ∈ state.system.nodes -> entry.2.location = entry.1 @@ -184,6 +185,7 @@ theorem nodeState_location theorem initial_well_formed (config : Config) (active : List Location) + (valid : config.Valid) (activeNodup : active.Nodup) (activeConfigured : forall node, node ∈ active -> @@ -191,6 +193,7 @@ theorem initial_well_formed WellFormed config (initial config active) := by constructor · simp [Global.initial, initialSystem, Function.comp_def] + · simpa [Global.initial, initialSystem, Function.comp_def] using valid.2.1 · simp [Global.initial, initialSystem, initialNode] · exact activeNodup · exact activeConfigured @@ -202,15 +205,17 @@ theorem initial_well_formed @[simp] theorem recordEffects_active (node : Location) + (nodeState : NodeState) (effects : List Effect) (state : State) : - (recordEffects node effects state).active = state.active := by + (recordEffects node nodeState effects state).active = state.active := by induction effects generalizing state with | nil => rfl | cons effect tail ih => simp only [recordEffects, List.foldl_cons] change - (recordEffects node tail (recordEffect node state effect)).active = + (recordEffects node nodeState tail + (recordEffect node nodeState state effect)).active = state.active rw [ih] cases effect <;> rfl @@ -218,15 +223,17 @@ theorem recordEffects_active @[simp] theorem recordEffects_system (node : Location) + (nodeState : NodeState) (effects : List Effect) (state : State) : - (recordEffects node effects state).system = state.system := by + (recordEffects node nodeState effects state).system = state.system := by induction effects generalizing state with | nil => rfl | cons effect tail ih => simp only [recordEffects, List.foldl_cons] change - (recordEffects node tail (recordEffect node state effect)).system = + (recordEffects node nodeState tail + (recordEffect node nodeState state effect)).system = state.system rw [ih] cases effect <;> rfl @@ -234,15 +241,17 @@ theorem recordEffects_system @[simp] theorem recordEffects_network (node : Location) + (nodeState : NodeState) (effects : List Effect) (state : State) : - (recordEffects node effects state).network = state.network := by + (recordEffects node nodeState effects state).network = state.network := by induction effects generalizing state with | nil => rfl | cons effect tail ih => simp only [recordEffects, List.foldl_cons] change - (recordEffects node tail (recordEffect node state effect)).network = + (recordEffects node nodeState tail + (recordEffect node nodeState state effect)).network = state.network rw [ih] cases effect <;> rfl @@ -250,26 +259,29 @@ theorem recordEffects_network @[simp] theorem recordEffects_sent (node : Location) + (nodeState : NodeState) (effects : List Effect) (state : State) : - (recordEffects node effects state).sent = state.sent := by + (recordEffects node nodeState effects state).sent = state.sent := by induction effects generalizing state with | nil => rfl | cons effect tail ih => simp only [recordEffects, List.foldl_cons] change - (recordEffects node tail (recordEffect node state effect)).sent = + (recordEffects node nodeState tail + (recordEffect node nodeState state effect)).sent = state.sent rw [ih] cases effect <;> rfl theorem recordEffect_preserves_histories_active {node : Location} + {nodeState : NodeState} {state : State} {effect : Effect} (wellFormed : HistoriesActive state) (nodeActive : node ∈ state.active) : - HistoriesActive (recordEffect node state effect) := by + HistoriesActive (recordEffect node nodeState state effect) := by rcases wellFormed with ⟨openings, restarts, completed⟩ cases effect <;> constructor <;> @@ -277,11 +289,12 @@ theorem recordEffect_preserves_histories_active theorem recordEffects_preserves_histories_active {node : Location} + {nodeState : NodeState} {effects : List Effect} {state : State} (wellFormed : HistoriesActive state) (nodeActive : node ∈ state.active) : - HistoriesActive (recordEffects node effects state) := by + HistoriesActive (recordEffects node nodeState effects state) := by induction effects generalizing state with | nil => exact wellFormed | cons effect tail ih => @@ -308,38 +321,42 @@ theorem mem_of_mem_removeOne theorem mem_openings_recordEffect {node : Location} + {nodeState : NodeState} {state : State} {effect : Effect} {opening : Opening} (membership : opening ∈ state.openings) : - opening ∈ (recordEffect node state effect).openings := by + opening ∈ (recordEffect node nodeState state effect).openings := by cases effect <;> simp_all [recordEffect] theorem mem_restarts_recordEffect {node : Location} + {nodeState : NodeState} {state : State} {effect : Effect} {restart : Location} (membership : restart ∈ state.restarts) : - restart ∈ (recordEffect node state effect).restarts := by + restart ∈ (recordEffect node nodeState state effect).restarts := by cases effect <;> simp_all [recordEffect] theorem mem_completed_recordEffect {node : Location} + {nodeState : NodeState} {state : State} {effect : Effect} {completed : Location} (membership : completed ∈ state.completed) : - completed ∈ (recordEffect node state effect).completed := by + completed ∈ (recordEffect node nodeState state effect).completed := by cases effect <;> simp_all [recordEffect] theorem mem_openings_recordEffects {node : Location} + {nodeState : NodeState} {state : State} {effects : List Effect} {opening : Opening} (membership : opening ∈ state.openings) : - opening ∈ (recordEffects node effects state).openings := by + opening ∈ (recordEffects node nodeState effects state).openings := by induction effects generalizing state with | nil => exact membership | cons effect tail ih => @@ -348,11 +365,12 @@ theorem mem_openings_recordEffects theorem mem_restarts_recordEffects {node : Location} + {nodeState : NodeState} {state : State} {effects : List Effect} {restart : Location} (membership : restart ∈ state.restarts) : - restart ∈ (recordEffects node effects state).restarts := by + restart ∈ (recordEffects node nodeState effects state).restarts := by induction effects generalizing state with | nil => exact membership | cons effect tail ih => @@ -361,11 +379,12 @@ theorem mem_restarts_recordEffects theorem mem_completed_recordEffects {node : Location} + {nodeState : NodeState} {state : State} {effects : List Effect} {completed : Location} (membership : completed ∈ state.completed) : - completed ∈ (recordEffects node effects state).completed := by + completed ∈ (recordEffects node nodeState effects state).completed := by induction effects generalizing state with | nil => exact membership | cons effect tail ih => @@ -530,11 +549,11 @@ theorem next_active_eq | deliver envelope => simp [next, Option.bind_eq_some_iff] at transition rcases transition with ⟨_, _, system, output, _, rfl⟩ - exact recordEffects_active envelope.target output.effects _ + simp | timeout target => simp [next, Option.bind_eq_some_iff] at transition rcases transition with ⟨_, system, output, _, _, rfl⟩ - exact recordEffects_active target output.effects _ + simp theorem next_node_keys_eq {config : Config} @@ -576,7 +595,7 @@ theorem deliver_network_eq simp [next, Option.bind_eq_some_iff] at transition rcases transition with ⟨_, _, system, output, _, rfl⟩ - exact recordEffects_network envelope.target output.effects _ + simp theorem timeout_network_eq {config : Config} @@ -587,7 +606,7 @@ theorem timeout_network_eq simp [next, Option.bind_eq_some_iff] at transition rcases transition with ⟨_, system, output, _, _, rfl⟩ - exact recordEffects_network target output.effects _ + simp theorem deliver_other_node_eq {config : Config} @@ -659,12 +678,14 @@ theorem next_openings_monotonic simp [next, Option.bind_eq_some_iff] at transition rcases transition with ⟨_, _, system, output, _, rfl⟩ - exact mem_openings_recordEffects membership + apply mem_openings_recordEffects + simpa using membership | timeout target => simp [next, Option.bind_eq_some_iff] at transition rcases transition with ⟨_, system, output, _, _, rfl⟩ - exact mem_openings_recordEffects membership + apply mem_openings_recordEffects + simpa using membership theorem next_restarts_monotonic {config : Config} @@ -683,12 +704,14 @@ theorem next_restarts_monotonic simp [next, Option.bind_eq_some_iff] at transition rcases transition with ⟨_, _, system, output, _, rfl⟩ - exact mem_restarts_recordEffects membership + apply mem_restarts_recordEffects + simpa using membership | timeout target => simp [next, Option.bind_eq_some_iff] at transition rcases transition with ⟨_, system, output, _, _, rfl⟩ - exact mem_restarts_recordEffects membership + apply mem_restarts_recordEffects + simpa using membership theorem next_completed_monotonic {config : Config} @@ -707,12 +730,14 @@ theorem next_completed_monotonic simp [next, Option.bind_eq_some_iff] at transition rcases transition with ⟨_, _, system, output, _, rfl⟩ - exact mem_completed_recordEffects membership + apply mem_completed_recordEffects + simpa using membership | timeout target => simp [next, Option.bind_eq_some_iff] at transition rcases transition with ⟨_, system, output, _, _, rfl⟩ - exact mem_completed_recordEffects membership + apply mem_completed_recordEffects + simpa using membership theorem retry_preserves_well_formed {config : Config} @@ -729,6 +754,7 @@ theorem retry_preserves_well_formed rw [←stateEq] constructor · exact wellFormed.nodeKeys + · exact wellFormed.nodeKeysNodup · exact wellFormed.nodeLocations · exact wellFormed.activeNodup · exact wellFormed.activeConfigured @@ -766,10 +792,12 @@ theorem deliver_preserves_well_formed ⟨_, targetActive, system, output, systemStep, stateEq⟩ rw [←stateEq] constructor - · rw [recordEffects_system] + · simp only [recordEffects_system] exact (systemStep_node_keys_eq systemStep).trans wellFormed.nodeKeys - · rw [recordEffects_system] + · rw [recordEffects_system, systemStep_node_keys_eq systemStep] + exact wellFormed.nodeKeysNodup + · simp only [recordEffects_system] exact systemStep_preserves_node_locations wellFormed.nodeLocations systemStep · simpa using wellFormed.activeNodup @@ -788,10 +816,10 @@ theorem deliver_preserves_well_formed (mem_of_mem_removeOne envelope pending before.network membership) · apply recordEffects_preserves_histories_active · constructor - · exact wellFormed.historiesActive.openings - · exact wellFormed.historiesActive.restarts - · exact wellFormed.historiesActive.completed - · exact targetActive + · simpa using wellFormed.historiesActive.openings + · simpa using wellFormed.historiesActive.restarts + · simpa using wellFormed.historiesActive.completed + · simpa using targetActive theorem timeout_preserves_well_formed {config : Config} @@ -805,10 +833,12 @@ theorem timeout_preserves_well_formed ⟨targetActive, system, output, systemStep, _, stateEq⟩ rw [←stateEq] constructor - · rw [recordEffects_system] + · simp only [recordEffects_system] exact (systemStep_node_keys_eq systemStep).trans wellFormed.nodeKeys - · rw [recordEffects_system] + · rw [recordEffects_system, systemStep_node_keys_eq systemStep] + exact wellFormed.nodeKeysNodup + · simp only [recordEffects_system] exact systemStep_preserves_node_locations wellFormed.nodeLocations systemStep · simpa using wellFormed.activeNodup @@ -826,10 +856,10 @@ theorem timeout_preserves_well_formed exact wellFormed.networkSent pending membership · apply recordEffects_preserves_histories_active · constructor - · exact wellFormed.historiesActive.openings - · exact wellFormed.historiesActive.restarts - · exact wellFormed.historiesActive.completed - · exact targetActive + · simpa using wellFormed.historiesActive.openings + · simpa using wellFormed.historiesActive.restarts + · simpa using wellFormed.historiesActive.completed + · simpa using targetActive theorem next_preserves_well_formed {config : Config} @@ -852,9 +882,18 @@ theorem reachable_well_formed (reachable : Reachable config state) : WellFormed config state := by induction reachable with - | initial active nodup configured => - exact initial_well_formed config active nodup configured + | initial active valid nodup configured => + exact initial_well_formed config active valid nodup configured | step reachable transition wellFormed => exact next_preserves_well_formed wellFormed transition +theorem reachable_config_valid + {config : Config} + {state : State} + (reachable : Reachable config state) : + config.Valid := by + induction reachable with + | initial active valid nodup configured => exact valid + | step reachable transition valid => exact valid + end DisasterRecovery.Protocol.Global diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/Model.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/Model.lean index df8226c0efc7..323e79f08c10 100644 --- a/lean/disaster-recovery/DisasterRecovery/Protocol/Model.lean +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/Model.lean @@ -7,7 +7,7 @@ abbrev Location := String structure TxID where view : Nat seqno : Nat -deriving Repr, BEq, Hashable, Inhabited, DecidableEq +deriving Repr, BEq, ReflBEq, LawfulBEq, Hashable, Inhabited, DecidableEq inductive Phase where | gossiping @@ -15,12 +15,12 @@ inductive Phase where | opening | joining | open -deriving Repr, BEq, Hashable, Inhabited, DecidableEq +deriving Repr, BEq, ReflBEq, LawfulBEq, Hashable, Inhabited, DecidableEq inductive OpenKind where | quorum | failover -deriving Repr, BEq, Hashable, Inhabited, DecidableEq +deriving Repr, BEq, ReflBEq, LawfulBEq, Hashable, Inhabited, DecidableEq inductive Validation where | accepted @@ -48,7 +48,7 @@ structure NodeState where chosen : Option Location := none openKind : Option OpenKind := none restartRequested : Bool := false -deriving Repr, BEq, Hashable, Inhabited +deriving Repr, BEq, ReflBEq, LawfulBEq, Hashable, Inhabited inductive Event where | receiveGossip (source : Location) (txid : TxID) (validation : Validation) @@ -102,7 +102,7 @@ def voteQuorum (config : Config) : Nat := def validTimeout (state : NodeState) (timeout : Bool) : Bool := timeout && decide (state.phase = state.timeoutState) -private def txScoreGreater +def txScoreGreater (leftName : Location) (left : TxID) (rightName : Location) @@ -112,14 +112,18 @@ private def txScoreGreater (right.seqno < left.seqno || (right.seqno == left.seqno && rightName < leftName))) +def selectMaximum + (current candidate : Prod Location TxID) : + Prod Location TxID := + if txScoreGreater candidate.1 candidate.2 current.1 current.2 then + candidate + else + current + def maximumGossip : List (Prod Location TxID) -> Option (Prod Location TxID) | [] => none | head :: tail => - some (tail.foldl (fun current candidate => - if txScoreGreater candidate.1 candidate.2 current.1 current.2 then - candidate - else - current) head) + some (tail.foldl selectMaximum head) def insertGossip (source : Location) diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/Quorum.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/Quorum.lean new file mode 100644 index 000000000000..7413f3f435ee --- /dev/null +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/Quorum.lean @@ -0,0 +1,1467 @@ +import DisasterRecovery.Protocol.Invariants +import Mathlib.Tactic + +namespace DisasterRecovery.Protocol.Global + +def SentVote (state : State) (voter target : Location) : Prop := + exists envelope, + envelope ∈ state.sent /\ + envelope.source = voter /\ + envelope.target = target /\ + envelope.payload = .vote + +def NodeVotesNodup (state : State) : Prop := + forall entry, entry ∈ state.system.nodes -> + entry.2.votes.Nodup + +def NodeVotesSent (state : State) : Prop := + forall entry, entry ∈ state.system.nodes -> + forall voter, voter ∈ entry.2.votes -> + SentVote state voter entry.1 + +def SentVotesFunctional (state : State) : Prop := + forall voter first second, + SentVote state voter first -> + SentVote state voter second -> + first = second + +def SentVoteStable (state : State) : Prop := + forall envelope, envelope ∈ state.sent -> + envelope.payload = .vote -> + forall entry, entry ∈ state.system.nodes -> + entry.1 = envelope.source -> + entry.2.phase ≠ .gossiping /\ + (entry.2.phase = .voting -> + entry.2.chosen = some envelope.target) + +def NodeVotingSelection (state : NodeState) : Prop := + exists target txid, + state.chosen = some target /\ + maximumGossip state.gossips = some (target, txid) + +def VotingSelectionsValid (state : State) : Prop := + forall entry, entry ∈ state.system.nodes -> + entry.2.phase = .voting -> + NodeVotingSelection entry.2 + +def SentVotesSelected (state : State) : Prop := + forall envelope, envelope ∈ state.sent -> + envelope.payload = .vote -> + NodeVotingSelection envelope.sourceState + +structure Opening.Valid + (config : Config) + (globalState : State) + (opening : Opening) : Prop where + location : opening.state.location = opening.node + phase : opening.state.phase = .opening + kind : opening.state.openKind = some opening.kind + votesNodup : opening.state.votes.Nodup + quorum : + opening.kind = .quorum -> + voteQuorum config.protocol <= opening.state.votes.length + votesSent : + forall voter, voter ∈ opening.state.votes -> + SentVote globalState voter opening.node + +def OpeningsValid (config : Config) (state : State) : Prop := + forall opening, opening ∈ state.openings -> + opening.Valid config state + +structure QuorumInvariant (config : Config) (state : State) : Prop where + votesNodup : NodeVotesNodup state + votesSent : NodeVotesSent state + sentVoteStable : SentVoteStable state + sentVotesFunctional : SentVotesFunctional state + votingSelections : VotingSelectionsValid state + sentVotesSelected : SentVotesSelected state + openingsValid : OpeningsValid config state + +theorem insertVote_nodup + (source : Location) + {votes : List Location} + (nodup : votes.Nodup) : + (insertVote source votes).Nodup := by + unfold insertVote + split + · exact nodup + · rename_i absent + apply (List.mergeSort_perm _ _).symm.nodup + rw [List.nodup_cons] + exact + ⟨fun member => absent (List.contains_iff_mem.mpr member), nodup⟩ + +theorem mem_insertVote + {member source : Location} + {votes : List Location} + (membership : member ∈ insertVote source votes) : + member ∈ votes \/ member = source := by + unfold insertVote at membership + split at membership + · exact Or.inl membership + · have unsorted := + (List.mergeSort_perm _ _).mem_iff.mp membership + rw [List.mem_cons] at unsorted + exact unsorted.symm + +theorem step_preserves_votes_nodup + (config : Protocol.Config) + (state : NodeState) + (event : Event) + (nodup : state.votes.Nodup) : + (step config state event).state.votes.Nodup := by + cases event <;> + simp [step, rejected, advance, advanceTimeoutLane] + all_goals + repeat first | split | simp_all [insertVote_nodup] + +def acceptedVoteSource : Event -> Option Location + | .receiveVote source .accepted => some source + | _ => none + +theorem step_votes_shape + (config : Protocol.Config) + (state : NodeState) + (event : Event) : + (step config state event).state.votes = state.votes \/ + exists source, + acceptedVoteSource event = some source /\ + (step config state event).state.votes = + insertVote source state.votes := by + cases event + all_goals try cases_type Validation + all_goals + simp [acceptedVoteSource, step, rejected, advance, advanceTimeoutLane] + all_goals repeat first | split | simp_all + +theorem step_vote_origin + (config : Protocol.Config) + (state : NodeState) + (event : Event) + (voter : Location) + (membership : voter ∈ (step config state event).state.votes) : + voter ∈ state.votes \/ + acceptedVoteSource event = some voter := by + rcases step_votes_shape config state event with + unchanged | ⟨source, sourceEq, changed⟩ + · rw [unchanged] at membership + exact Or.inl membership + · rw [changed] at membership + rcases mem_insertVote membership with old | added + · exact Or.inl old + · subst source + exact Or.inr sourceEq + +theorem step_preserves_non_gossiping + (config : Protocol.Config) + (state : NodeState) + (event : Event) + (pastGossip : state.phase ≠ .gossiping) : + (step config state event).state.phase ≠ .gossiping := by + cases event <;> + simp [step, rejected, advance, advanceTimeoutLane] + all_goals repeat first | split | simp_all + +theorem voting_step_preserves_choice + (config : Protocol.Config) + (state : NodeState) + (event : Event) + (pastGossip : state.phase ≠ .gossiping) + (stillVoting : (step config state event).state.phase = .voting) : + state.phase = .voting /\ + (step config state event).state.chosen = state.chosen := by + cases event <;> + simp [step, rejected, advance, advanceTimeoutLane] at stillVoting ⊢ + all_goals repeat first | split at stillVoting | split | simp_all + +theorem step_preserves_voting_selection + (config : Protocol.Config) + (state : NodeState) + (event : Event) + (before : + state.phase = .voting -> + NodeVotingSelection state) + (voting : (step config state event).state.phase = .voting) : + NodeVotingSelection (step config state event).state := by + cases event + all_goals try cases_type Validation + all_goals + simp [NodeVotingSelection, step, rejected, advance, + advanceTimeoutLane, validTimeout] at before voting ⊢ + all_goals + repeat first | split at voting | split | simp_all | aesop + +theorem retry_vote_state + {config : Config} + {envelope : Envelope} + (valid : envelope.Valid config) + (vote : envelope.payload = .vote) : + envelope.sourceState.phase = .voting /\ + envelope.sourceState.chosen = some envelope.target := by + rcases valid_envelope_effect valid with + ⟨effect, member, created⟩ + cases effect with + | sendGossip target => + cases found : recoveredTxID config envelope.source with + | none => + simp [messageForEffect, found] at created + | some txid => + simp [messageForEffect, found] at created + rw [←created] at vote + contradiction + | sendVote target => + simp [messageForEffect] at created + rw [←created] at vote ⊢ + cases phase : envelope.sourceState.phase <;> + simp [step, phase] at member + next => + cases chosen : envelope.sourceState.chosen <;> + simp_all + | sendIAmOpen target => + simp [messageForEffect] at created + rw [←created] at vote + contradiction + | opening kind => + simp [messageForEffect] at created + | restart chosen => + simp [messageForEffect] at created + | completed => + simp [messageForEffect] at created + | rejected reason => + simp [messageForEffect] at created + +theorem opening_effect_state + (config : Protocol.Config) + (state : NodeState) + (event : Event) + (kind : OpenKind) + (opening : .opening kind ∈ (step config state event).effects) : + (step config state event).state.phase = .opening /\ + (step config state event).state.openKind = some kind := by + cases event + all_goals try cases_type Validation + all_goals + simp [step, rejected, advance, advanceTimeoutLane, validTimeout] + at opening ⊢ + all_goals + repeat first | split at opening | split | simp_all | aesop + +theorem quorum_effect_has_threshold + (config : Protocol.Config) + (state : NodeState) + (event : Event) + (opening : + .opening .quorum ∈ (step config state event).effects) : + voteQuorum config <= + (step config state event).state.votes.length := by + cases event + all_goals try cases_type Validation + all_goals + simp [step, rejected, advance, advanceTimeoutLane, validTimeout] + at opening ⊢ + all_goals + repeat first | split at opening | split | simp_all | aesop + +theorem sentVote_mono + {before after : State} + {voter target : Location} + (sent : forall envelope, envelope ∈ before.sent -> + envelope ∈ after.sent) + (vote : SentVote before voter target) : + SentVote after voter target := by + rcases vote with + ⟨envelope, membership, source, destination, payload⟩ + exact + ⟨envelope, sent envelope membership, source, destination, payload⟩ + +theorem opening_valid_of_sent_eq + {config : Config} + {before after : State} + {opening : Opening} + (sentEq : after.sent = before.sent) + (valid : opening.Valid config before) : + opening.Valid config after := by + rcases valid with + ⟨location, phase, kind, nodup, quorum, votesSent⟩ + constructor + · exact location + · exact phase + · exact kind + · exact nodup + · exact quorum + · intro voter membership + apply sentVote_mono + · intro envelope sent + rw [sentEq] + exact sent + · exact votesSent voter membership + +theorem opening_valid_mono + {config : Config} + {before after : State} + {opening : Opening} + (sent : + forall envelope, envelope ∈ before.sent -> + envelope ∈ after.sent) + (valid : opening.Valid config before) : + opening.Valid config after := by + rcases valid with + ⟨location, phase, kind, nodup, quorum, votesSent⟩ + constructor + · exact location + · exact phase + · exact kind + · exact nodup + · exact quorum + · intro voter membership + exact sentVote_mono sent (votesSent voter membership) + +theorem recordEffect_preserves_openings_valid + {config : Config} + {node : Location} + {nodeState : NodeState} + {state : State} + {effect : Effect} + (valid : OpeningsValid config state) + (newValid : + forall kind, + effect = .opening kind -> + Opening.Valid config state + { node, kind, state := nodeState }) : + OpeningsValid config + (recordEffect node nodeState state effect) := by + intro opening membership + cases effect with + | opening kind => + simp [recordEffect] at membership + rcases membership with rfl | old + · apply opening_valid_of_sent_eq + (before := state) + (after := recordEffect node nodeState state (.opening kind)) + rfl + exact newValid kind rfl + · apply opening_valid_of_sent_eq + (before := state) + (after := recordEffect node nodeState state (.opening kind)) + rfl + exact valid opening old + | sendGossip target => + exact valid opening membership + | sendVote target => + exact valid opening membership + | sendIAmOpen target => + exact valid opening membership + | restart target => + apply opening_valid_of_sent_eq + (before := state) + (after := recordEffect node nodeState state (.restart target)) + rfl + exact valid opening membership + | completed => + apply opening_valid_of_sent_eq + (before := state) + (after := recordEffect node nodeState state .completed) + rfl + exact valid opening membership + | rejected reason => + exact valid opening membership + +theorem recordEffects_preserves_openings_valid + {config : Config} + {node : Location} + {nodeState : NodeState} + {state : State} + {effects : List Effect} + (valid : OpeningsValid config state) + (newValid : + forall kind, + .opening kind ∈ effects -> + Opening.Valid config state + { node, kind, state := nodeState }) : + OpeningsValid config + (recordEffects node nodeState effects state) := by + induction effects generalizing state with + | nil => exact valid + | cons effect tail ih => + simp only [recordEffects, List.foldl_cons] + apply ih + · apply recordEffect_preserves_openings_valid valid + intro kind effectEq + subst effect + exact newValid kind (by simp) + · intro kind membership + apply opening_valid_of_sent_eq + (before := state) + (after := recordEffect node nodeState state effect) + (by cases effect <;> rfl) + exact newValid kind (by simp [membership]) + +theorem eventFor_vote_source + {envelope : Envelope} + {voter : Location} + (source : + acceptedVoteSource (eventFor envelope) = some voter) : + envelope.payload = .vote /\ + envelope.source = voter := by + cases payload : envelope.payload <;> + simp_all [eventFor, acceptedVoteSource] + +theorem systemStep_preserves_votes_nodup + {config : Protocol.Config} + {before after : SystemState} + {target : Location} + {event : Event} + {output : StepOutput} + (nodup : + forall entry, entry ∈ before.nodes -> + entry.2.votes.Nodup) + (transition : + systemStep config before target event = some (after, output)) : + forall entry, entry ∈ after.nodes -> + entry.2.votes.Nodup := by + simp [systemStep, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨node, ⟨key, found⟩, stateEq, _⟩ + rw [←stateEq] + intro entry membership + rw [replaceNode, List.mem_map] at membership + rcases membership with ⟨previous, previousMember, rfl⟩ + split + · apply step_preserves_votes_nodup + exact nodup (key, node) (List.mem_of_find?_eq_some found) + · exact nodup previous previousMember + +theorem systemStep_preserves_voting_selections + {config : Protocol.Config} + {before after : SystemState} + {target : Location} + {event : Event} + {output : StepOutput} + (valid : + forall entry, entry ∈ before.nodes -> + entry.2.phase = .voting -> + NodeVotingSelection entry.2) + (transition : + systemStep config before target event = some (after, output)) : + forall entry, entry ∈ after.nodes -> + entry.2.phase = .voting -> + NodeVotingSelection entry.2 := by + simp [systemStep, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨node, ⟨key, found⟩, systemEq, outputEq⟩ + rw [←systemEq] + intro entry membership voting + rw [replaceNode, List.mem_map] at membership + rcases membership with ⟨previous, previousMember, rfl⟩ + split + · rename_i atTarget + apply step_preserves_voting_selection config node event + · exact valid (key, node) + (List.mem_of_find?_eq_some found) + · simpa [atTarget, outputEq] using voting + · rename_i notTarget + exact valid previous previousMember + (by simpa [notTarget] using voting) + +theorem systemStep_output_location + {config : Protocol.Config} + {before after : SystemState} + {target : Location} + {event : Event} + {output : StepOutput} + (locations : + forall entry, entry ∈ before.nodes -> + entry.2.location = entry.1) + (transition : + systemStep config before target event = some (after, output)) : + output.state.location = target := by + simp [systemStep, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨node, ⟨key, found⟩, _, outputEq⟩ + calc + output.state.location = + node.location := by + rw [←outputEq] + exact step_preserves_location config node event + _ = key := + locations (key, node) (List.mem_of_find?_eq_some found) + _ = target := + beq_iff_eq.mp + (List.find?_some + (p := fun entry : Prod Location NodeState => + entry.1 == target) found) + +theorem systemStep_output_mem + {config : Protocol.Config} + {before after : SystemState} + {target : Location} + {event : Event} + {output : StepOutput} + (transition : + systemStep config before target event = some (after, output)) : + (target, output.state) ∈ after.nodes := by + simp [systemStep, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨node, ⟨key, found⟩, systemEq, outputEq⟩ + rw [←systemEq, replaceNode, List.mem_map] + refine ⟨(key, node), List.mem_of_find?_eq_some found, ?_⟩ + have keyEq : key = target := + beq_iff_eq.mp + (List.find?_some + (p := fun entry : Prod Location NodeState => + entry.1 == target) found) + simp [keyEq, outputEq] + +theorem systemStep_opening_effect_state + {config : Protocol.Config} + {before after : SystemState} + {target : Location} + {event : Event} + {output : StepOutput} + {kind : OpenKind} + (transition : + systemStep config before target event = some (after, output)) + (opening : .opening kind ∈ output.effects) : + output.state.phase = .opening /\ + output.state.openKind = some kind := by + simp [systemStep, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨node, _, _, outputEq⟩ + rw [←outputEq] at opening ⊢ + exact opening_effect_state config node event kind opening + +theorem systemStep_quorum_effect_has_threshold + {config : Protocol.Config} + {before after : SystemState} + {target : Location} + {event : Event} + {output : StepOutput} + (transition : + systemStep config before target event = some (after, output)) + (opening : .opening .quorum ∈ output.effects) : + voteQuorum config <= output.state.votes.length := by + simp [systemStep, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨node, _, _, outputEq⟩ + rw [←outputEq] at opening ⊢ + exact quorum_effect_has_threshold config node event opening + +theorem initial_node_votes_nodup + (config : Config) + (active : List Location) : + NodeVotesNodup (initial config active) := by + simp [NodeVotesNodup, Global.initial, initialSystem, initialNode] + +theorem initial_node_votes_sent + (config : Config) + (active : List Location) : + NodeVotesSent (initial config active) := by + simp [NodeVotesSent, Global.initial, initialSystem, initialNode] + +theorem initial_sent_votes_functional + (config : Config) + (active : List Location) : + SentVotesFunctional (initial config active) := by + simp [SentVotesFunctional, SentVote, Global.initial] + +theorem initial_sent_vote_stable + (config : Config) + (active : List Location) : + SentVoteStable (initial config active) := by + simp [SentVoteStable, Global.initial] + +theorem initial_voting_selections + (config : Config) + (active : List Location) : + VotingSelectionsValid (initial config active) := by + simp [VotingSelectionsValid, Global.initial, initialSystem, initialNode] + +theorem initial_sent_votes_selected + (config : Config) + (active : List Location) : + SentVotesSelected (initial config active) := by + simp [SentVotesSelected, Global.initial] + +theorem initial_openings_valid + (config : Config) + (active : List Location) : + OpeningsValid config (initial config active) := by + simp [OpeningsValid, Global.initial] + +theorem systemStep_preserves_node_votes_sent + {config : Protocol.Config} + {before after : SystemState} + {target : Location} + {event : Event} + {output : StepOutput} + {beforeState afterState : State} + (beforeSystem : beforeState.system = before) + (votesSent : NodeVotesSent beforeState) + (carry : + forall voter destination, + SentVote beforeState voter destination -> + SentVote afterState voter destination) + (introduced : + forall voter, + acceptedVoteSource event = some voter -> + SentVote afterState voter target) + (transition : + systemStep config before target event = some (after, output)) : + forall entry, entry ∈ after.nodes -> + forall voter, voter ∈ entry.2.votes -> + SentVote afterState voter entry.1 := by + simp [systemStep, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨node, ⟨key, found⟩, systemEq, outputEq⟩ + rw [←systemEq] + intro entry membership voter vote + rw [replaceNode, List.mem_map] at membership + rcases membership with ⟨previous, previousMember, rfl⟩ + split + · rename_i atTarget + have targetEq : previous.1 = target := + beq_iff_eq.mp atTarget + rcases step_vote_origin config node event voter + (by simpa [outputEq, atTarget] using vote) with + old | added + · have keyEq : key = target := + beq_iff_eq.mp + (List.find?_some + (p := fun entry : Prod Location NodeState => + entry.1 == target) found) + apply carry + rw [←keyEq] + exact votesSent (key, node) + (by + rw [beforeSystem] + exact List.mem_of_find?_eq_some found) + voter old + · exact introduced voter added + · rename_i notTarget + apply carry + exact votesSent previous + (by + rw [beforeSystem] + exact previousMember) + voter (by simpa [notTarget] using vote) + +theorem eq_of_key_eq + {α : Type} + {nodes : List (Prod Location α)} + (nodup : (nodes.map Prod.fst).Nodup) + {first second : Prod Location α} + (firstMember : first ∈ nodes) + (secondMember : second ∈ nodes) + (keyEq : first.1 = second.1) : + first = second := by + induction nodes generalizing first second with + | nil => simp at firstMember + | cons head tail ih => + rw [List.map_cons, List.nodup_cons] at nodup + rcases nodup with ⟨headFresh, tailNodup⟩ + rw [List.mem_cons] at firstMember secondMember + rcases firstMember with rfl | firstTail + · rcases secondMember with rfl | secondTail + · rfl + · exfalso + apply headFresh + rw [List.mem_map] + exact ⟨second, secondTail, keyEq.symm⟩ + · rcases secondMember with rfl | secondTail + · exfalso + apply headFresh + rw [List.mem_map] + exact ⟨first, firstTail, keyEq⟩ + · exact ih tailNodup firstTail secondTail keyEq + +theorem systemStep_preserves_vote_stability + {config : Protocol.Config} + {before after : SystemState} + {target : Location} + {event : Event} + {output : StepOutput} + {envelope : Envelope} + (stable : + forall entry, entry ∈ before.nodes -> + entry.1 = envelope.source -> + entry.2.phase ≠ .gossiping /\ + (entry.2.phase = .voting -> + entry.2.chosen = some envelope.target)) + (transition : + systemStep config before target event = some (after, output)) : + forall entry, entry ∈ after.nodes -> + entry.1 = envelope.source -> + entry.2.phase ≠ .gossiping /\ + (entry.2.phase = .voting -> + entry.2.chosen = some envelope.target) := by + simp [systemStep, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨node, ⟨key, found⟩, systemEq, outputEq⟩ + rw [←systemEq] + intro entry membership sourceEq + rw [replaceNode, List.mem_map] at membership + rcases membership with ⟨previous, previousMember, rfl⟩ + split + · rename_i atTarget + have targetEq : previous.1 = target := + beq_iff_eq.mp atTarget + have targetSource : target = envelope.source := by + simpa [atTarget] using sourceEq + have keyEq : key = target := + beq_iff_eq.mp + (List.find?_some + (p := fun entry : Prod Location NodeState => + entry.1 == target) found) + have beforeStable := + stable (key, node) (List.mem_of_find?_eq_some found) + (keyEq.trans targetSource) + constructor + · exact step_preserves_non_gossiping config node event + beforeStable.1 + · intro voting + rcases voting_step_preserves_choice config node event + beforeStable.1 voting with ⟨beforeVoting, chosenEq⟩ + rw [chosenEq] + exact beforeStable.2 beforeVoting + · rename_i notTarget + exact stable previous previousMember + (by simpa [notTarget] using sourceEq) + +theorem next_preserves_node_votes_nodup + {config : Config} + {before after : State} + {action : Action} + (nodup : NodeVotesNodup before) + (transition : next config before action = some after) : + NodeVotesNodup after := by + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, sourceState, _, _, rfl⟩ + exact nodup + | deliver envelope => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, _, system, output, systemStep, rfl⟩ + intro entry membership + rw [recordEffects_system] at membership + exact systemStep_preserves_votes_nodup nodup systemStep + entry membership + | timeout target => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, systemStep, _, rfl⟩ + intro entry membership + rw [recordEffects_system] at membership + exact systemStep_preserves_votes_nodup nodup systemStep + entry membership + +theorem next_preserves_voting_selections + {config : Config} + {before after : State} + {action : Action} + (valid : VotingSelectionsValid before) + (transition : next config before action = some after) : + VotingSelectionsValid after := by + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, sourceState, _, _, rfl⟩ + exact valid + | deliver envelope => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, _, system, output, systemStep, rfl⟩ + intro entry membership voting + rw [recordEffects_system] at membership + exact systemStep_preserves_voting_selections valid systemStep + entry membership voting + | timeout target => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, systemStep, _, rfl⟩ + intro entry membership voting + rw [recordEffects_system] at membership + exact systemStep_preserves_voting_selections valid systemStep + entry membership voting + +theorem retry_preserves_sent_votes_selected + {config : Config} + {before after : State} + {source : Location} + (wellFormed : WellFormed config before) + (votingSelections : VotingSelectionsValid before) + (selected : SentVotesSelected before) + (transition : next config before (.retry source) = some after) : + SentVotesSelected after := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, sourceState, found, _, stateEq⟩ + have sourceLocation : sourceState.location = source := + nodeState_location wellFormed.nodeLocations found + rw [←stateEq] + intro envelope membership payload + rw [List.mem_append] at membership + rcases membership with old | added + · exact selected envelope old payload + · have valid : envelope.Valid config := + retryMessages_valid config source sourceState sourceLocation + envelope added + have voteState := retry_vote_state valid payload + have identity := retryMessages_source added + rw [identity.2] at voteState + rw [nodeState, Option.map_eq_some_iff] at found + rcases found with ⟨entry, findEq, stateEq⟩ + have sourceSelection := + votingSelections entry (List.mem_of_find?_eq_some findEq) + (by simpa [stateEq] using voteState.1) + simpa [identity.2, stateEq] using sourceSelection + +theorem deliver_preserves_sent_votes_selected + {config : Config} + {before after : State} + {envelope : Envelope} + (selected : SentVotesSelected before) + (transition : next config before (.deliver envelope) = some after) : + SentVotesSelected after := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, _, system, output, _, stateEq⟩ + rw [←stateEq] + intro vote membership payload + rw [recordEffects_sent] at membership + exact selected vote membership payload + +theorem timeout_preserves_sent_votes_selected + {config : Config} + {before after : State} + {target : Location} + (selected : SentVotesSelected before) + (transition : next config before (.timeout target) = some after) : + SentVotesSelected after := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, _, _, stateEq⟩ + rw [←stateEq] + intro vote membership payload + rw [recordEffects_sent] at membership + exact selected vote membership payload + +theorem retry_preserves_node_votes_sent + {config : Config} + {before after : State} + {source : Location} + (votesSent : NodeVotesSent before) + (transition : next config before (.retry source) = some after) : + NodeVotesSent after := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, sourceState, _, _, rfl⟩ + intro entry membership voter vote + apply sentVote_mono (before := before) + · intro envelope sent + exact List.mem_append_left _ sent + · exact votesSent entry membership voter vote + +theorem deliver_preserves_node_votes_sent + {config : Config} + {before after : State} + {envelope : Envelope} + (wellFormed : WellFormed config before) + (votesSent : NodeVotesSent before) + (transition : next config before (.deliver envelope) = some after) : + NodeVotesSent after := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨inNetwork, _, system, output, systemStep, stateEq⟩ + rw [←stateEq] + intro entry membership voter vote + rw [recordEffects_system] at membership + let afterState := + recordEffects envelope.target output.state output.effects + { + before with + system + network := removeOne envelope before.network + } + have carry : + forall oldVoter oldTarget, + SentVote before oldVoter oldTarget -> + SentVote afterState oldVoter oldTarget := by + intro oldVoter oldTarget oldVote + apply sentVote_mono (before := before) (after := afterState) + · intro sent sentMember + simpa [afterState] using sentMember + · exact oldVote + have introducedVote : + forall newVoter, + acceptedVoteSource (eventFor envelope) = some newVoter -> + SentVote afterState newVoter envelope.target := by + intro newVoter introduced + rcases eventFor_vote_source introduced with + ⟨payload, source⟩ + subst newVoter + refine ⟨envelope, ?_, rfl, rfl, payload⟩ + simp [afterState] + exact wellFormed.networkSent envelope + inNetwork + exact systemStep_preserves_node_votes_sent + (before := before.system) + (after := system) + (beforeState := before) + (afterState := afterState) + rfl votesSent carry introducedVote systemStep + entry membership voter vote + +theorem timeout_preserves_node_votes_sent + {config : Config} + {before after : State} + {target : Location} + (votesSent : NodeVotesSent before) + (transition : next config before (.timeout target) = some after) : + NodeVotesSent after := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, systemStep, _, stateEq⟩ + rw [←stateEq] + intro entry membership voter vote + rw [recordEffects_system] at membership + let afterState := + recordEffects target output.state output.effects + { before with system } + have carry : + forall oldVoter oldTarget, + SentVote before oldVoter oldTarget -> + SentVote afterState oldVoter oldTarget := by + intro oldVoter oldTarget oldVote + apply sentVote_mono (before := before) (after := afterState) + · intro sent sentMember + simpa [afterState] using sentMember + · exact oldVote + have introducedVote : + forall newVoter, + acceptedVoteSource Event.timeout = some newVoter -> + SentVote afterState newVoter target := by + intro newVoter introduced + simp [acceptedVoteSource] at introduced + exact systemStep_preserves_node_votes_sent + (before := before.system) + (after := system) + (beforeState := before) + (afterState := afterState) + rfl votesSent carry introducedVote systemStep + entry membership voter vote + +theorem retry_preserves_openings_valid + {config : Config} + {before after : State} + {source : Location} + (valid : OpeningsValid config before) + (transition : next config before (.retry source) = some after) : + OpeningsValid config after := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, sourceState, _, _, stateEq⟩ + rw [←stateEq] + intro opening membership + apply opening_valid_mono + · intro envelope sent + exact List.mem_append_left _ sent + · exact valid opening membership + +theorem deliver_preserves_openings_valid + {config : Config} + {before after : State} + {envelope : Envelope} + (wellFormed : WellFormed config before) + (votesNodup : NodeVotesNodup before) + (votesSent : NodeVotesSent before) + (valid : OpeningsValid config before) + (transition : next config before (.deliver envelope) = some after) : + OpeningsValid config after := by + have afterNodup := + next_preserves_node_votes_nodup votesNodup transition + have afterVotesSent := + deliver_preserves_node_votes_sent wellFormed votesSent transition + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, _, system, output, systemStep, stateEq⟩ + rw [←stateEq] at afterNodup afterVotesSent ⊢ + let delivered : State := { + before with + system + network := removeOne envelope before.network + } + apply recordEffects_preserves_openings_valid + · intro opening membership + apply opening_valid_of_sent_eq + (before := before) (after := delivered) rfl + exact valid opening membership + · intro kind openingEffect + have effectState := + systemStep_opening_effect_state systemStep openingEffect + constructor + · exact systemStep_output_location + wellFormed.nodeLocations systemStep + · exact effectState.1 + · exact effectState.2 + · apply afterNodup (envelope.target, output.state) + rw [recordEffects_system] + exact systemStep_output_mem systemStep + · intro quorumKind + have kindEq : kind = .quorum := by simpa using quorumKind + rw [kindEq] at openingEffect + simpa using + systemStep_quorum_effect_has_threshold systemStep openingEffect + · intro voter vote + have sent := + afterVotesSent (envelope.target, output.state) + (by + rw [recordEffects_system] + exact systemStep_output_mem systemStep) + voter vote + simpa [SentVote, delivered] using sent + +theorem timeout_preserves_openings_valid + {config : Config} + {before after : State} + {target : Location} + (wellFormed : WellFormed config before) + (votesNodup : NodeVotesNodup before) + (votesSent : NodeVotesSent before) + (valid : OpeningsValid config before) + (transition : next config before (.timeout target) = some after) : + OpeningsValid config after := by + have afterNodup := + next_preserves_node_votes_nodup votesNodup transition + have afterVotesSent := + timeout_preserves_node_votes_sent votesSent transition + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, systemStep, _, stateEq⟩ + rw [←stateEq] at afterNodup afterVotesSent ⊢ + let timedOut : State := { before with system } + apply recordEffects_preserves_openings_valid + · intro opening membership + apply opening_valid_of_sent_eq + (before := before) (after := timedOut) rfl + exact valid opening membership + · intro kind openingEffect + have effectState := + systemStep_opening_effect_state systemStep openingEffect + constructor + · exact systemStep_output_location + wellFormed.nodeLocations systemStep + · exact effectState.1 + · exact effectState.2 + · apply afterNodup (target, output.state) + rw [recordEffects_system] + exact systemStep_output_mem systemStep + · intro quorumKind + have kindEq : kind = .quorum := by simpa using quorumKind + rw [kindEq] at openingEffect + simpa using + systemStep_quorum_effect_has_threshold systemStep openingEffect + · intro voter vote + have sent := + afterVotesSent (target, output.state) + (by + rw [recordEffects_system] + exact systemStep_output_mem systemStep) + voter vote + simpa [SentVote, timedOut] using sent + +theorem retry_preserves_sent_vote_stable + {config : Config} + {before after : State} + {source : Location} + (wellFormed : WellFormed config before) + (stable : SentVoteStable before) + (transition : next config before (.retry source) = some after) : + SentVoteStable after := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, sourceState, found, _, stateEq⟩ + have sourceLocation : sourceState.location = source := + nodeState_location wellFormed.nodeLocations found + rw [←stateEq] + intro envelope membership payload entry entryMember keyEq + rw [List.mem_append] at membership + rcases membership with old | added + · exact stable envelope old payload entry entryMember keyEq + · have valid : envelope.Valid config := + retryMessages_valid config source sourceState sourceLocation + envelope added + have voteState := retry_vote_state valid payload + rcases retryMessages_source added with + ⟨sourceEq, stateEq⟩ + rw [nodeState, Option.map_eq_some_iff] at found + rcases found with ⟨foundEntry, findEq, foundStateEq⟩ + have foundMember : foundEntry ∈ before.system.nodes := + List.mem_of_find?_eq_some findEq + have foundKey : foundEntry.1 = source := + beq_iff_eq.mp + (List.find?_some + (p := fun entry : Prod Location NodeState => + entry.1 == source) findEq) + have sameEntry : entry = foundEntry := + eq_of_key_eq wellFormed.nodeKeysNodup entryMember foundMember + ((keyEq.trans sourceEq).trans foundKey.symm) + subst entry + rw [foundStateEq, ←stateEq] + exact ⟨by simp [voteState.1], fun _ => voteState.2⟩ + +theorem deliver_preserves_sent_vote_stable + {config : Config} + {before after : State} + {delivered : Envelope} + (stable : SentVoteStable before) + (transition : next config before (.deliver delivered) = some after) : + SentVoteStable after := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, _, system, output, systemStep, stateEq⟩ + rw [←stateEq] + intro envelope membership payload entry entryMember keyEq + rw [recordEffects_sent] at membership + rw [recordEffects_system] at entryMember + exact systemStep_preserves_vote_stability + (fun previous previousMember source => + stable envelope membership payload previous previousMember source) + systemStep entry entryMember keyEq + +theorem timeout_preserves_sent_vote_stable + {config : Config} + {before after : State} + {target : Location} + (stable : SentVoteStable before) + (transition : next config before (.timeout target) = some after) : + SentVoteStable after := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, systemStep, _, stateEq⟩ + rw [←stateEq] + intro envelope membership payload entry entryMember keyEq + rw [recordEffects_sent] at membership + rw [recordEffects_system] at entryMember + exact systemStep_preserves_vote_stability + (fun previous previousMember source => + stable envelope membership payload previous previousMember source) + systemStep entry entryMember keyEq + +theorem sentVote_stable_at_node + {state : State} + {voter target : Location} + {current : NodeState} + (stable : SentVoteStable state) + (vote : SentVote state voter target) + (found : nodeState state voter = some current) : + current.phase ≠ .gossiping /\ + (current.phase = .voting -> + current.chosen = some target) := by + rcases vote with + ⟨envelope, sent, sourceEq, targetEq, payload⟩ + rw [nodeState, Option.map_eq_some_iff] at found + rcases found with ⟨entry, findEq, stateEq⟩ + have entryMember : entry ∈ state.system.nodes := + List.mem_of_find?_eq_some findEq + have entryKey : entry.1 = voter := + beq_iff_eq.mp + (List.find?_some + (p := fun entry : Prod Location NodeState => + entry.1 == voter) findEq) + have result := + stable envelope sent payload entry entryMember + (entryKey.trans sourceEq.symm) + rw [stateEq] at result + simpa [targetEq] using result + +theorem retry_preserves_sent_votes_functional + {config : Config} + {before after : State} + {source : Location} + (wellFormed : WellFormed config before) + (functional : SentVotesFunctional before) + (stable : SentVoteStable before) + (transition : next config before (.retry source) = some after) : + SentVotesFunctional after := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, sourceState, found, _, stateEq⟩ + have sourceLocation : sourceState.location = source := + nodeState_location wellFormed.nodeLocations found + rw [←stateEq] + have classify : + forall voter target, + SentVote + { + before with + network := before.network ++ + retryMessages config source sourceState + sent := before.sent ++ + retryMessages config source sourceState + } + voter target -> + SentVote before voter target \/ + (voter = source /\ + sourceState.phase = .voting /\ + sourceState.chosen = some target) := by + intro voter target vote + rcases vote with + ⟨envelope, membership, sourceEq, targetEq, payload⟩ + rw [List.mem_append] at membership + rcases membership with old | added + · exact Or.inl + ⟨envelope, old, sourceEq, targetEq, payload⟩ + · have valid : envelope.Valid config := + retryMessages_valid config source sourceState sourceLocation + envelope added + have voteState := retry_vote_state valid payload + have retryIdentity := retryMessages_source added + rw [retryIdentity.2] at voteState + exact Or.inr + ⟨sourceEq.symm.trans retryIdentity.1, + voteState.1, + by simpa [targetEq] using voteState.2⟩ + intro voter first second firstVote secondVote + rcases classify voter first firstVote with + firstOld | ⟨firstSource, firstPhase, firstChoice⟩ + · rcases classify voter second secondVote with + secondOld | ⟨secondSource, secondPhase, secondChoice⟩ + · exact functional voter first second firstOld secondOld + · have oldState := + sentVote_stable_at_node stable firstOld + (by simpa [secondSource] using found) + have oldChoice := oldState.2 secondPhase + rw [oldChoice] at secondChoice + exact Option.some.inj secondChoice + · rcases classify voter second secondVote with + secondOld | ⟨secondSource, secondPhase, secondChoice⟩ + · have oldState := + sentVote_stable_at_node stable secondOld + (by simpa [firstSource] using found) + have oldChoice := oldState.2 firstPhase + rw [oldChoice] at firstChoice + exact (Option.some.inj firstChoice).symm + · rw [firstChoice] at secondChoice + exact Option.some.inj secondChoice + +theorem deliver_preserves_sent_votes_functional + {config : Config} + {before after : State} + {envelope : Envelope} + (functional : SentVotesFunctional before) + (transition : next config before (.deliver envelope) = some after) : + SentVotesFunctional after := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, _, system, output, _, stateEq⟩ + rw [←stateEq] + intro voter first second firstVote secondVote + apply functional voter first second + · simpa [SentVote] using firstVote + · simpa [SentVote] using secondVote + +theorem timeout_preserves_sent_votes_functional + {config : Config} + {before after : State} + {target : Location} + (functional : SentVotesFunctional before) + (transition : next config before (.timeout target) = some after) : + SentVotesFunctional after := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, _, _, stateEq⟩ + rw [←stateEq] + intro voter first second firstVote secondVote + apply functional voter first second + · simpa [SentVote] using firstVote + · simpa [SentVote] using secondVote + +theorem initial_quorum_invariant + (config : Config) + (active : List Location) : + QuorumInvariant config (initial config active) := { + votesNodup := initial_node_votes_nodup config active + votesSent := initial_node_votes_sent config active + sentVoteStable := initial_sent_vote_stable config active + sentVotesFunctional := initial_sent_votes_functional config active + votingSelections := initial_voting_selections config active + sentVotesSelected := initial_sent_votes_selected config active + openingsValid := initial_openings_valid config active +} + +theorem next_preserves_quorum_invariant + {config : Config} + {before after : State} + {action : Action} + (wellFormed : WellFormed config before) + (invariant : QuorumInvariant config before) + (transition : next config before action = some after) : + QuorumInvariant config after := by + cases action with + | retry source => + constructor + · exact next_preserves_node_votes_nodup + invariant.votesNodup transition + · exact retry_preserves_node_votes_sent + invariant.votesSent transition + · exact retry_preserves_sent_vote_stable + wellFormed invariant.sentVoteStable transition + · exact retry_preserves_sent_votes_functional + wellFormed invariant.sentVotesFunctional + invariant.sentVoteStable transition + · exact next_preserves_voting_selections + invariant.votingSelections transition + · exact retry_preserves_sent_votes_selected + wellFormed invariant.votingSelections + invariant.sentVotesSelected transition + · exact retry_preserves_openings_valid + invariant.openingsValid transition + | deliver envelope => + constructor + · exact next_preserves_node_votes_nodup + invariant.votesNodup transition + · exact deliver_preserves_node_votes_sent + wellFormed invariant.votesSent transition + · exact deliver_preserves_sent_vote_stable + invariant.sentVoteStable transition + · exact deliver_preserves_sent_votes_functional + invariant.sentVotesFunctional transition + · exact next_preserves_voting_selections + invariant.votingSelections transition + · exact deliver_preserves_sent_votes_selected + invariant.sentVotesSelected transition + · exact deliver_preserves_openings_valid + wellFormed invariant.votesNodup invariant.votesSent + invariant.openingsValid transition + | timeout target => + constructor + · exact next_preserves_node_votes_nodup + invariant.votesNodup transition + · exact timeout_preserves_node_votes_sent + invariant.votesSent transition + · exact timeout_preserves_sent_vote_stable + invariant.sentVoteStable transition + · exact timeout_preserves_sent_votes_functional + invariant.sentVotesFunctional transition + · exact next_preserves_voting_selections + invariant.votingSelections transition + · exact timeout_preserves_sent_votes_selected + invariant.sentVotesSelected transition + · exact timeout_preserves_openings_valid + wellFormed invariant.votesNodup invariant.votesSent + invariant.openingsValid transition + +theorem reachable_quorum_invariant + {config : Config} + {state : State} + (reachable : Reachable config state) : + QuorumInvariant config state := by + induction reachable with + | initial active valid nodup configured => + exact initial_quorum_invariant config active + | step reachable transition invariant => + exact next_preserves_quorum_invariant + (reachable_well_formed reachable) invariant transition + +theorem quorum_lists_intersect + {α : Type} + [DecidableEq α] + (expected first second : List α) + (firstNodup : first.Nodup) + (secondNodup : second.Nodup) + (firstSubset : + forall value, value ∈ first -> value ∈ expected) + (secondSubset : + forall value, value ∈ second -> value ∈ expected) + (firstQuorum : + expected.length / 2 + 1 <= first.length) + (secondQuorum : + expected.length / 2 + 1 <= second.length) : + exists value, value ∈ first /\ value ∈ second := by + by_contra noShared + push_neg at noShared + have disjoint : Disjoint first.toFinset second.toFinset := + Finset.disjoint_left.mpr (by + intro value firstMember secondMember + exact noShared value + (List.mem_toFinset.mp firstMember) + (List.mem_toFinset.mp secondMember)) + have unionSubset : + first.toFinset ∪ second.toFinset ⊆ expected.toFinset := by + intro value membership + rw [Finset.mem_union] at membership + rw [List.mem_toFinset] + exact membership.elim + (fun member => + firstSubset value (List.mem_toFinset.mp member)) + (fun member => + secondSubset value (List.mem_toFinset.mp member)) + have unionCard := Finset.card_le_card unionSubset + rw [Finset.card_union_of_disjoint disjoint, + List.toFinset_card_of_nodup firstNodup, + List.toFinset_card_of_nodup secondNodup] at unionCard + have expectedCard := List.toFinset_card_le expected + omega + +def QuorumOpened (state : State) (node : Location) : Prop := + exists opening, + opening ∈ state.openings /\ + opening.node = node /\ + opening.kind = .quorum + +theorem opening_vote_configured + {config : Config} + {state : State} + {opening : Opening} + (wellFormed : WellFormed config state) + (valid : opening.Valid config state) + {voter : Location} + (vote : voter ∈ opening.state.votes) : + voter ∈ config.protocol.expectedLocations := by + rcases valid.votesSent voter vote with + ⟨envelope, sent, sourceEq, _, _⟩ + apply wellFormed.activeConfigured voter + simpa [sourceEq] using + wellFormed.sentSourceActive envelope sent + +theorem quorum_opener_unique + {config : Config} + {state : State} + {first second : Location} + (reachable : Reachable config state) + (firstOpened : QuorumOpened state first) + (secondOpened : QuorumOpened state second) : + first = second := by + have wellFormed := reachable_well_formed reachable + have invariant := reachable_quorum_invariant reachable + rcases firstOpened with + ⟨firstOpening, firstMember, firstNode, firstKind⟩ + rcases secondOpened with + ⟨secondOpening, secondMember, secondNode, secondKind⟩ + have firstValid := + invariant.openingsValid firstOpening firstMember + have secondValid := + invariant.openingsValid secondOpening secondMember + rcases quorum_lists_intersect + config.protocol.expectedLocations + firstOpening.state.votes + secondOpening.state.votes + firstValid.votesNodup + secondValid.votesNodup + (fun voter vote => + opening_vote_configured wellFormed firstValid vote) + (fun voter vote => + opening_vote_configured wellFormed secondValid vote) + (by + simpa [voteQuorum] using firstValid.quorum firstKind) + (by + simpa [voteQuorum] using secondValid.quorum secondKind) with + ⟨voter, firstVote, secondVote⟩ + have targetEq := + invariant.sentVotesFunctional voter + firstOpening.node secondOpening.node + (firstValid.votesSent voter firstVote) + (secondValid.votesSent voter secondVote) + exact firstNode.symm.trans (targetEq.trans secondNode) + +end DisasterRecovery.Protocol.Global diff --git a/lean/disaster-recovery/README.md b/lean/disaster-recovery/README.md index a325c3880e71..4021c189024d 100644 --- a/lean/disaster-recovery/README.md +++ b/lean/disaster-recovery/README.md @@ -27,12 +27,12 @@ effects around that same canonical transition function. The versioned trace validator also replays committed C++ instrumentation against the canonical model. -The migration is approximately 5,000 lines across 25 new files: +The migration is approximately 7,000 lines across 27 new files: | Area | Files | Approximate lines | Purpose | | ----------------------------------------- | ----: | ----------------: | --------------------------------------------------------------- | | Exact legacy Lean model | 4 | 650 | Stateright state, actions, timers, network, predicates, and BFS | -| Canonical protocol and proofs | 6 | 1,900 | C++ behavior, global semantics, invariants, and temporal proofs | +| Canonical protocol and proofs | 8 | 3,600 | C++ behavior, global semantics, invariants, and temporal proofs | | Trace validation | 5 | 760 | NDJSON format, deterministic replay, and CLI | | Equivalence tooling | 2 | 710 | Rust graph exporter and bidirectional Rust/Lean comparison | | Project, documentation, and configuration | 6 | 530 | Lake/Mathlib setup, entry points, and documentation | @@ -44,11 +44,13 @@ The migration is approximately 5,000 lines across 25 new files: | -------------------------------------------------------------------------------------------- | ----: | ------------------------------------------------------------------ | | [`DisasterRecovery/Model.lean`](DisasterRecovery/Model.lean) | 392 | Exact executable legacy semantics | | [`DisasterRecovery/Checker.lean`](DisasterRecovery/Checker.lean) | 152 | BFS enumeration, property checking, and canonical graph export | -| [`DisasterRecovery/Protocol/Model.lean`](DisasterRecovery/Protocol/Model.lean) | 286 | Production-oriented C++ protocol model | +| [`DisasterRecovery/Protocol/Model.lean`](DisasterRecovery/Protocol/Model.lean) | 290 | Production-oriented C++ protocol model | | [`DisasterRecovery/Protocol/Refinement.lean`](DisasterRecovery/Protocol/Refinement.lean) | 332 | Canonical-to-legacy formal phase refinement | | [`DisasterRecovery/Protocol/Temporal.lean`](DisasterRecovery/Protocol/Temporal.lean) | 230 | Execution streams, fairness, safety, and progress proofs | -| [`DisasterRecovery/Protocol/Global.lean`](DisasterRecovery/Protocol/Global.lean) | 154 | Global active-node, network, send-history, and effect semantics | -| [`DisasterRecovery/Protocol/Invariants.lean`](DisasterRecovery/Protocol/Invariants.lean) | 860 | Global provenance, locality, monotonicity, and reachability proofs | +| [`DisasterRecovery/Protocol/Global.lean`](DisasterRecovery/Protocol/Global.lean) | 166 | Global active-node, network, send-history, and effect semantics | +| [`DisasterRecovery/Protocol/Invariants.lean`](DisasterRecovery/Protocol/Invariants.lean) | 899 | Global provenance, locality, monotonicity, and reachability proofs | +| [`DisasterRecovery/Protocol/Quorum.lean`](DisasterRecovery/Protocol/Quorum.lean) | 1,467 | Vote-history invariants and unbounded quorum-opener uniqueness | +| [`DisasterRecovery/Protocol/Committed.lean`](DisasterRecovery/Protocol/Committed.lean) | 258 | TxID maximum and committed-prefix preservation proofs | | [`DisasterRecovery/Protocol/Trace/Format.lean`](DisasterRecovery/Protocol/Trace/Format.lean) | 141 | Versioned NDJSON types and parser | | [`DisasterRecovery/Protocol/Trace/Replay.lean`](DisasterRecovery/Protocol/Trace/Replay.lean) | 452 | Deterministic implementation-trace replay | | [`compare.py`](compare.py) | 343 | Exhaustive canonical graph comparison | @@ -152,8 +154,7 @@ a prior modeled send rather than an arbitrary receive input. Its reachability relation requires the active locations to be unique and configured. `DisasterRecovery.Protocol.Global.WellFormed` records the foundational -invariants needed by the remaining distributed proofs. The principal results -are: +invariants used by the distributed proofs. The principal results are: - `reachable_well_formed`: every globally reachable state has the configured node keys and matching state locations, unique configured active nodes, valid @@ -171,9 +172,29 @@ are: - `next_openings_monotonic`, `next_restarts_monotonic`, and `next_completed_monotonic`: observed terminal effects are never removed. -These are global semantic foundations, not yet the planned quorum-path -uniqueness, committed-prefix preservation, or fair global termination -theorems. +`DisasterRecovery.Protocol.Quorum` additionally proves that counted votes are +duplicate-free and backed by prior sends, retry snapshots retain the selected +maximum, each voter has one immutable vote target, and recorded openings retain +their exact vote evidence. `quorum_lists_intersect` proves strict-majority +intersection for `n / 2 + 1`, and `quorum_opener_unique` applies it to any two +quorum openings in an arbitrary reachable execution. This is an unbounded +safety theorem and needs no fairness assumption. + +`DisasterRecovery.Protocol.Committed` defines lexicographic TxID prefix order, +proves that `maximumGossip` is a member and upper bound of every collected +gossip, and proves `full_gossip_selection_preserves_commit`. Its assumptions are +explicit: + +- `DurableCommit` says at least one configured recovered ledger covers the + committed TxID; +- `FullGossipSelection` says a real sent vote selected the opener from a gossip + snapshot containing exactly the configured recovered TxIDs; and +- `quorum_open_preserves_commit` combines that evidence with a recorded quorum + opening. + +Quorum opening alone does **not** imply `FullGossipSelection`: voting may follow +a gossip timeout. The committed-prefix theorem deliberately does not hide or +derive that premise. Fair global termination remains the next unbounded proof. ### Kernel-checked refinement properties From 3294e9c1a715dd4bb4d146197ab42b3d3588064e Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Mon, 31 Aug 2026 22:45:32 +0100 Subject: [PATCH 12/14] Prove fair global recovery progress Define global executions and action-oriented fairness, prove phase-by-phase progress to a completed opener, and prove all active nodes eventually terminate under an explicit broadcast-before-completion ordering premise. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 46d56f8a-935f-4551-abe3-84bd4f951865 --- lean/disaster-recovery/DisasterRecovery.lean | 1 + .../Protocol/GlobalTemporal.lean | 3168 +++++++++++++++++ lean/disaster-recovery/README.md | 73 +- 3 files changed, 3222 insertions(+), 20 deletions(-) create mode 100644 lean/disaster-recovery/DisasterRecovery/Protocol/GlobalTemporal.lean diff --git a/lean/disaster-recovery/DisasterRecovery.lean b/lean/disaster-recovery/DisasterRecovery.lean index 6a7e6aea83b9..2ae613b58f86 100644 --- a/lean/disaster-recovery/DisasterRecovery.lean +++ b/lean/disaster-recovery/DisasterRecovery.lean @@ -6,5 +6,6 @@ import DisasterRecovery.Protocol.Invariants import DisasterRecovery.Protocol.Quorum import DisasterRecovery.Protocol.Committed import DisasterRecovery.Protocol.Temporal +import DisasterRecovery.Protocol.GlobalTemporal import DisasterRecovery.Protocol.Refinement import DisasterRecovery.Protocol.Trace diff --git a/lean/disaster-recovery/DisasterRecovery/Protocol/GlobalTemporal.lean b/lean/disaster-recovery/DisasterRecovery/Protocol/GlobalTemporal.lean new file mode 100644 index 000000000000..c7cac044b5f6 --- /dev/null +++ b/lean/disaster-recovery/DisasterRecovery/Protocol/GlobalTemporal.lean @@ -0,0 +1,3168 @@ +import DisasterRecovery.Protocol.Committed +import DisasterRecovery.Protocol.Temporal +import Mathlib.Tactic + +namespace DisasterRecovery.Protocol.Global + +structure Execution (config : Config) where + states : Nat -> State + actions : Nat -> Action + step_succ : forall n, + next config (states n) (actions n) = some (states (n + 1)) + +def HasPhase (state : State) (node : Location) (phase : Phase) : Prop := + exists nodeState, + Global.nodeState state node = some nodeState /\ + nodeState.phase = phase + +def HasGossip (state : State) (node : Location) : Prop := + exists nodeState, + Global.nodeState state node = some nodeState /\ + nodeState.gossips ≠ [] + +def HasVote (state : State) (node : Location) : Prop := + exists nodeState, + Global.nodeState state node = some nodeState /\ + nodeState.votes ≠ [] + +def LaneAdvanced (state : State) (node : Location) : Prop := + exists nodeState, + Global.nodeState state node = some nodeState /\ + nodeState.timeoutState ≠ .gossiping + +theorem hasPhase_unique + {state : State} + {node : Location} + {first second : Phase} + (firstPhase : HasPhase state node first) + (secondPhase : HasPhase state node second) : + first = second := by + rcases firstPhase with ⟨firstState, firstFound, firstEq⟩ + rcases secondPhase with ⟨secondState, secondFound, secondEq⟩ + rw [firstFound] at secondFound + injection secondFound with stateEq + subst secondState + exact firstEq.symm.trans secondEq + +def Terminal (state : State) (node : Location) : Prop := + node ∈ state.restarts \/ node ∈ state.completed + +def CompletedOpen (state : State) (node : Location) : Prop := + node ∈ state.completed + +def AnnouncementsLive (state : State) : Prop := + forall envelope, envelope ∈ state.sent -> + envelope.payload = .iAmOpen -> + HasPhase state envelope.source .opening \/ + CompletedOpen state envelope.source + +def SentAnnouncementTo (state : State) (target : Location) : Prop := + exists envelope, + envelope ∈ state.sent /\ + envelope.target = target /\ + envelope.payload = .iAmOpen + +def JoiningAnnouncements (state : State) : Prop := + forall entry, entry ∈ state.system.nodes -> + entry.2.phase = .joining -> + SentAnnouncementTo state entry.1 + +def OpenCompleted (state : State) : Prop := + forall entry, entry ∈ state.system.nodes -> + entry.2.phase = .open -> + CompletedOpen state entry.1 + +def AdvancedNodesActive (state : State) : Prop := + forall entry, entry ∈ state.system.nodes -> + entry.2.phase ≠ .gossiping -> + entry.1 ∈ state.active + +def OpenerWitness (state : State) : Prop := + exists node, + HasPhase state node .opening \/ + CompletedOpen state node + +def OnlyOpenerCompletesFrom + {config : Config} + (execution : Execution config) + (start : Nat) + (opener : Location) : Prop := + forall n node, + start <= n -> + CompletedOpen (execution.states n) node -> + node = opener + +def QuorumOnlyCompletions + {config : Config} + (execution : Execution config) : Prop := + forall n node, + CompletedOpen (execution.states n) node -> + QuorumOpened (execution.states n) node + +def SentAnnouncement + (state : State) + (source target : Location) : Prop := + exists envelope, + envelope ∈ state.sent /\ + envelope.source = source /\ + envelope.target = target /\ + envelope.payload = .iAmOpen + +def BroadcastBeforeCompletion + {config : Config} + (execution : Execution config) : Prop := + forall n opener, + CompletedOpen (execution.states n) opener -> + forall target, target ∈ (execution.states n).active -> + target ≠ opener -> + SentAnnouncement (execution.states n) opener target + +def AnnouncementsResolved (state : State) : Prop := + forall envelope, envelope ∈ state.sent -> + envelope.payload = .iAmOpen -> + envelope ∈ state.network \/ + Terminal state envelope.target \/ + HasPhase state envelope.target .opening + +def Enabled (config : Config) (state : State) (action : Action) : Prop := + exists nextState, next config state action = some nextState + +def LaneValid (state : NodeState) : Prop := + (state.phase = .gossiping -> + state.timeoutState = .gossiping) /\ + (state.phase = .voting -> + state.timeoutState = .gossiping \/ + state.timeoutState = .voting) /\ + (state.phase = .opening -> + state.timeoutState = .gossiping \/ + state.timeoutState = .voting \/ + state.timeoutState = .opening) /\ + (state.phase = .gossiping -> + state.chosen = none) + +def NodeLanesValid (state : State) : Prop := + forall entry, entry ∈ state.system.nodes -> + LaneValid entry.2 + +theorem step_preserves_lane + (config : Protocol.Config) + (state : NodeState) + (event : Event) + (valid : LaneValid state) : + LaneValid (step config state event).state := by + cases event + all_goals try cases_type Validation + all_goals + simp [LaneValid, step, rejected, advance, advanceTimeoutLane, + advanceTimeoutState, validTimeout] at valid ⊢ + all_goals repeat first | split | simp_all | aesop + +theorem step_preserves_advanced_lane + (config : Protocol.Config) + (state : NodeState) + (event : Event) + (advanced : state.timeoutState ≠ .gossiping) : + (step config state event).state.timeoutState ≠ .gossiping := by + cases event + all_goals try cases_type Validation + all_goals + simp [step, rejected, advance, validTimeout, advanceTimeoutLane, + advanceTimeoutState] at advanced ⊢ + all_goals repeat first | split | simp_all + +theorem systemStep_preserves_lanes + {config : Protocol.Config} + {before after : SystemState} + {target : Location} + {event : Event} + {output : StepOutput} + (valid : + forall entry, entry ∈ before.nodes -> + LaneValid entry.2) + (transition : + systemStep config before target event = some (after, output)) : + forall entry, entry ∈ after.nodes -> + LaneValid entry.2 := by + simp [systemStep, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨node, ⟨key, found⟩, systemEq, outputEq⟩ + rw [←systemEq] + intro entry membership + rw [replaceNode, List.mem_map] at membership + rcases membership with ⟨previous, previousMember, rfl⟩ + split + · apply step_preserves_lane config node event + exact valid (key, node) (List.mem_of_find?_eq_some found) + · exact valid previous previousMember + +theorem initial_lanes_valid + (config : Config) + (active : List Location) : + NodeLanesValid (initial config active) := by + simp [NodeLanesValid, LaneValid, Global.initial, initialSystem, + initialNode] + +theorem next_preserves_lanes + {config : Config} + {before after : State} + {action : Action} + (valid : NodeLanesValid before) + (transition : next config before action = some after) : + NodeLanesValid after := by + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, sourceState, _, _, rfl⟩ + exact valid + | deliver envelope => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, _, system, output, systemStep, rfl⟩ + intro entry membership + rw [recordEffects_system] at membership + exact systemStep_preserves_lanes valid systemStep + entry membership + | timeout target => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, systemStep, _, rfl⟩ + intro entry membership + rw [recordEffects_system] at membership + exact systemStep_preserves_lanes valid systemStep + entry membership + +theorem reachable_lanes_valid + {config : Config} + {state : State} + (reachable : Reachable config state) : + NodeLanesValid state := by + induction reachable with + | initial active valid nodup configured => + exact initial_lanes_valid config active + | step reachable transition valid => + exact next_preserves_lanes valid transition + +theorem nodeState_eq_of_mem + {state : State} + {node : Location} + {foundState : NodeState} + (keysNodup : (state.system.nodes.map Prod.fst).Nodup) + (membership : (node, foundState) ∈ state.system.nodes) : + Global.nodeState state node = some foundState := by + unfold Global.nodeState + cases found : + state.system.nodes.find? fun entry => entry.1 == node with + | none => + rw [List.find?_eq_none] at found + exact False.elim + (found (node, foundState) membership (by simp)) + | some entry => + have foundMember : entry ∈ state.system.nodes := + List.mem_of_find?_eq_some found + have foundKey : entry.1 = node := + beq_iff_eq.mp + (List.find?_some + (p := fun entry : Prod Location NodeState => + entry.1 == node) found) + have same : entry = (node, foundState) := + eq_of_key_eq keysNodup foundMember membership foundKey + simp [same] + +theorem node_property_of_nodeState + {state : State} + {node : Location} + {foundState : NodeState} + {predicate : NodeState -> Prop} + (property : + forall entry, entry ∈ state.system.nodes -> + predicate entry.2) + (found : Global.nodeState state node = some foundState) : + predicate foundState := by + rw [Global.nodeState, Option.map_eq_some_iff] at found + rcases found with ⟨entry, findEq, stateEq⟩ + rw [←stateEq] + exact property entry (List.mem_of_find?_eq_some findEq) + +theorem deliver_target_state + {config : Config} + {before after : State} + {envelope : Envelope} + (wellFormed : WellFormed config before) + (transition : next config before (.deliver envelope) = some after) : + exists output, + Global.nodeState after envelope.target = some output.state /\ + systemStep config.protocol before.system envelope.target + (eventFor envelope) = some (after.system, output) := by + have afterWellFormed := + next_preserves_well_formed wellFormed transition + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, _, system, output, systemStep, stateEq⟩ + rw [←stateEq] at afterWellFormed ⊢ + refine ⟨output, ?_, ?_⟩ + · apply nodeState_eq_of_mem afterWellFormed.nodeKeysNodup + rw [recordEffects_system] + exact systemStep_output_mem systemStep + · simpa using systemStep + +theorem timeout_target_state + {config : Config} + {before after : State} + {target : Location} + (wellFormed : WellFormed config before) + (transition : next config before (.timeout target) = some after) : + exists output, + Global.nodeState after target = some output.state /\ + output.accepted = true /\ + systemStep config.protocol before.system target .timeout = + some (after.system, output) := by + have afterWellFormed := + next_preserves_well_formed wellFormed transition + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, systemStep, accepted, stateEq⟩ + rw [←stateEq] at afterWellFormed ⊢ + refine ⟨output, ?_, accepted, ?_⟩ + · apply nodeState_eq_of_mem afterWellFormed.nodeKeysNodup + rw [recordEffects_system] + exact systemStep_output_mem systemStep + · simpa using systemStep + +theorem systemStep_output_eq + {config : Protocol.Config} + {global : State} + {after : SystemState} + {target : Location} + {event : Event} + {state : NodeState} + {output : StepOutput} + (found : Global.nodeState global target = some state) + (transition : + systemStep config global.system target event = some (after, output)) : + output = step config state event := by + change + (do + let node <- Global.nodeState global target + let result := step config node event + pure ({ + nodes := replaceNode target result.state global.system.nodes + }, result)) = some (after, output) at transition + rw [found] at transition + simp at transition + exact transition.2.symm + +theorem completed_effect_recorded + {node : Location} + {nodeState : NodeState} + {effects : List Effect} + {state : State} + (completed : .completed ∈ effects) : + node ∈ (recordEffects node nodeState effects state).completed := by + induction effects generalizing state with + | nil => simp at completed + | cons effect tail ih => + simp only [recordEffects, List.foldl_cons] + rw [List.mem_cons] at completed + rcases completed with rfl | inTail + · apply mem_completed_recordEffects + simp [recordEffect] + · exact ih inTail + +theorem restart_effect_recorded + {node chosen : Location} + {nodeState : NodeState} + {effects : List Effect} + {state : State} + (restart : .restart chosen ∈ effects) : + node ∈ (recordEffects node nodeState effects state).restarts := by + induction effects generalizing state with + | nil => simp at restart + | cons effect tail ih => + simp only [recordEffects, List.foldl_cons] + rw [List.mem_cons] at restart + rcases restart with rfl | inTail + · apply mem_restarts_recordEffects + simp [recordEffect] + · exact ih inTail + +theorem mem_removeOne_or_eq + [BEq α] + [LawfulBEq α] + {member removed : α} + {values : List α} + (membership : member ∈ values) : + member ∈ removeOne removed values \/ member = removed := by + induction values with + | nil => simp at membership + | cons head tail ih => + rw [List.mem_cons] at membership + rcases membership with rfl | inTail + · by_cases equal : member = removed + · exact Or.inr equal + · exact Or.inl (by simp [removeOne, equal]) + · simp only [removeOne] + split + · exact Or.inl inTail + · rcases ih inTail with still | equal + · exact Or.inl (by simp [still]) + · exact Or.inr equal + +structure Fair + {config : Config} + (execution : Execution config) : Prop where + retry : + forall start node phase, + node ∈ (execution.states start).active -> + HasPhase (execution.states start) node phase -> + (phase = .gossiping \/ phase = .voting \/ phase = .opening) -> + Enabled config (execution.states start) (.retry node) -> + EventuallyFrom start (fun n => + Not (HasPhase (execution.states n) node phase) \/ + execution.actions n = .retry node) + delivery : + forall start envelope, + envelope ∈ (execution.states start).network -> + EventuallyFrom start (fun n => + execution.actions n = .deliver envelope) + timeout : + forall start node phase, + node ∈ (execution.states start).active -> + HasPhase (execution.states start) node phase -> + (phase = .gossiping \/ phase = .voting \/ phase = .opening) -> + Enabled config (execution.states start) (.timeout node) -> + EventuallyFrom start (fun n => + Not (HasPhase (execution.states n) node phase) \/ + execution.actions n = .timeout node) + openingTimeout : + forall start node, + node ∈ (execution.states start).active -> + HasPhase (execution.states start) node .opening -> + Enabled config (execution.states start) (.timeout node) -> + EventuallyFrom start (fun n => + CompletedOpen (execution.states n) node \/ + (HasPhase (execution.states n) node .opening /\ + execution.actions n = .timeout node)) + +theorem execution_reachable + {config : Config} + (execution : Execution config) + (initial : Reachable config (execution.states 0)) : + forall n, Reachable config (execution.states n) := by + intro n + induction n with + | zero => exact initial + | succ n reachable => + exact Reachable.step reachable (execution.step_succ n) + +theorem execution_active_eq + {config : Config} + (execution : Execution config) : + forall n, (execution.states n).active = (execution.states 0).active := by + intro n + induction n with + | zero => rfl + | succ n activeEq => + exact (next_active_eq (execution.step_succ n)).trans activeEq + +theorem active_at + {config : Config} + (execution : Execution config) + {node : Location} + (active : node ∈ (execution.states 0).active) : + forall n, node ∈ (execution.states n).active := by + intro n + rw [execution_active_eq execution n] + exact active + +theorem recovered_for_configured + {config : Config} + (valid : config.Valid) + {node : Location} + (configured : node ∈ config.protocol.expectedLocations) : + exists txid, recoveredTxID config node = some txid := by + rw [←valid.2.2] at configured + rcases List.mem_map.mp configured with + ⟨entry, membership, keyEq⟩ + rcases entry with ⟨location, txid⟩ + simp at keyEq + subst location + refine ⟨txid, ?_⟩ + apply recoveredTxID_of_mem valid + exact membership + +theorem active_nodeState + {config : Config} + {state : State} + (wellFormed : WellFormed config state) + {node : Location} + (active : node ∈ state.active) : + exists nodeState, + Global.nodeState state node = some nodeState := by + have configured := wellFormed.activeConfigured node active + rw [←wellFormed.nodeKeys] at configured + rcases List.mem_map.mp configured with + ⟨entry, membership, keyEq⟩ + refine ⟨entry.2, ?_⟩ + apply nodeState_eq_of_mem wellFormed.nodeKeysNodup + rcases entry with ⟨location, nodeState⟩ + simp at keyEq + subst location + exact membership + +theorem retryMessages_self_gossip + {config : Config} + {node : Location} + {state : NodeState} + {txid : TxID} + (phase : state.phase = .gossiping) + (configured : node ∈ config.protocol.expectedLocations) + (recovered : recoveredTxID config node = some txid) : + { + source := node + target := node + payload := Payload.gossip txid + sourceState := state + } ∈ retryMessages config node state := by + rw [retryMessages, List.mem_filterMap] + refine ⟨.sendGossip node, ?_, ?_⟩ + · simpa [step, phase] using configured + · simp [messageForEffect, recovered] + +theorem retryMessages_vote + {config : Config} + {node target : Location} + {state : NodeState} + (phase : state.phase = .voting) + (chosen : state.chosen = some target) : + { + source := node + target + payload := Payload.vote + sourceState := state + } ∈ retryMessages config node state := by + rw [retryMessages, List.mem_filterMap] + refine ⟨.sendVote target, ?_, rfl⟩ + simp [step, phase, chosen] + +theorem retry_iamopen_state + {config : Config} + {envelope : Envelope} + (valid : envelope.Valid config) + (announcement : envelope.payload = .iAmOpen) : + envelope.sourceState.phase = .opening := by + rcases valid_envelope_effect valid with + ⟨effect, member, created⟩ + cases effect with + | sendGossip target => + cases found : recoveredTxID config envelope.source with + | none => simp [messageForEffect, found] at created + | some txid => + simp [messageForEffect, found] at created + rw [←created] at announcement + contradiction + | sendVote target => + simp [messageForEffect] at created + rw [←created] at announcement + contradiction + | sendIAmOpen target => + simp [messageForEffect] at created + rw [←created] at announcement ⊢ + cases phase : envelope.sourceState.phase + case opening => rfl + case voting => + cases chosen : envelope.sourceState.chosen <;> + simp [step, phase, chosen] at member + all_goals simp [step, phase] at member + | opening kind => simp [messageForEffect] at created + | restart chosen => simp [messageForEffect] at created + | completed => simp [messageForEffect] at created + | rejected reason => simp [messageForEffect] at created + +def acceptedIAmOpenSource : Event -> Option Location + | .receiveIAmOpen source .accepted => some source + | _ => none + +theorem step_joining_origin + (config : Protocol.Config) + (state : NodeState) + (event : Event) + (joining : (step config state event).state.phase = .joining) : + state.phase = .joining \/ + exists source, acceptedIAmOpenSource event = some source := by + cases event + all_goals try cases_type Validation + all_goals + simp [acceptedIAmOpenSource, step, rejected, advance, + advanceTimeoutLane] at joining ⊢ + all_goals + repeat first | split at joining | split | simp_all | aesop + +theorem step_open_origin + (config : Protocol.Config) + (state : NodeState) + (event : Event) + (opened : (step config state event).state.phase = .open) : + state.phase = .open \/ + .completed ∈ (step config state event).effects := by + cases event + all_goals try cases_type Validation + all_goals + simp [step, rejected, advance, advanceTimeoutLane, validTimeout] + at opened ⊢ + all_goals + repeat first | split at opened | split | simp_all | aesop + +theorem iamopen_delivery_outcome + (config : Protocol.Config) + (state : NodeState) + (source : Location) : + let output := step config state (.receiveIAmOpen source .accepted) + output.state.phase = .opening \/ + output.state.phase = .open \/ + exists chosen, .restart chosen ∈ output.effects := by + cases phase : state.phase <;> + simp [step, phase, rejected, advance, advanceTimeoutLane] + +theorem iamopen_open_predecessor + (config : Protocol.Config) + (state : NodeState) + (source : Location) + (opened : + (step config state (.receiveIAmOpen source .accepted)).state.phase = + .open) : + state.phase = .open := by + cases phase : state.phase <;> + simp [step, phase, rejected, advance, advanceTimeoutLane] at opened + rfl + +theorem eventFor_iamopen_source + {envelope : Envelope} + {source : Location} + (accepted : + acceptedIAmOpenSource (eventFor envelope) = some source) : + envelope.payload = .iAmOpen /\ + envelope.source = source := by + cases payload : envelope.payload <;> + simp_all [eventFor, acceptedIAmOpenSource] + +theorem retry_gossip_enabled + {config : Config} + {state : State} + {node : Location} + (valid : config.Valid) + (wellFormed : WellFormed config state) + (active : node ∈ state.active) + (phase : HasPhase state node .gossiping) : + Enabled config state (.retry node) := by + rcases phase with ⟨nodeState, found, gossiping⟩ + have configured := wellFormed.activeConfigured node active + rcases recovered_for_configured valid configured with + ⟨txid, recovered⟩ + have message := + retryMessages_self_gossip gossiping configured recovered + have messagesNonempty : + retryMessages config node nodeState ≠ [] := by + intro empty + rw [empty] at message + simp at message + refine ⟨{ + state with + network := state.network ++ retryMessages config node nodeState + sent := state.sent ++ retryMessages config node nodeState + }, ?_⟩ + simp [next, active, found, messagesNonempty] + +theorem retry_voting_enabled + {config : Config} + {state : State} + {node target : Location} + {nodeState : NodeState} + (active : node ∈ state.active) + (found : Global.nodeState state node = some nodeState) + (phase : nodeState.phase = .voting) + (chosen : nodeState.chosen = some target) : + Enabled config state (.retry node) := by + have message := retryMessages_vote (config := config) + (node := node) phase chosen + have messagesNonempty : + retryMessages config node nodeState ≠ [] := by + intro empty + rw [empty] at message + simp at message + refine ⟨{ + state with + network := state.network ++ retryMessages config node nodeState + sent := state.sent ++ retryMessages config node nodeState + }, ?_⟩ + simp [next, active, found, messagesNonempty] + +theorem delivery_enabled + {config : Config} + {state : State} + {envelope : Envelope} + (wellFormed : WellFormed config state) + (network : envelope ∈ state.network) + (targetActive : envelope.target ∈ state.active) : + Enabled config state (.deliver envelope) := by + rcases active_nodeState wellFormed targetActive with + ⟨targetState, found⟩ + let output := step config.protocol targetState (eventFor envelope) + let system : SystemState := { + nodes := replaceNode envelope.target output.state state.system.nodes + } + let delivered : State := { + state with + system + network := removeOne envelope state.network + } + have stepResult : + systemStep config.protocol state.system envelope.target + (eventFor envelope) = some (system, output) := by + change + (do + let node <- Global.nodeState state envelope.target + let result := step config.protocol node (eventFor envelope) + pure ({ + nodes := + replaceNode envelope.target result.state state.system.nodes + }, result)) = some (system, output) + rw [found] + rfl + refine + ⟨recordEffects envelope.target output.state output.effects delivered, ?_⟩ + simp [next, network, targetActive, stepResult, output, system, + delivered] + +theorem timeout_enabled_of_accepted + {config : Config} + {state : State} + {node : Location} + {nodeState : NodeState} + (active : node ∈ state.active) + (found : Global.nodeState state node = some nodeState) + (accepted : (step config.protocol nodeState .timeout).accepted = true) : + Enabled config state (.timeout node) := by + let output := step config.protocol nodeState .timeout + let system : SystemState := { + nodes := replaceNode node output.state state.system.nodes + } + have stepResult : + systemStep config.protocol state.system node .timeout = + some (system, output) := by + change + (do + let current <- Global.nodeState state node + let result := step config.protocol current .timeout + pure ({ + nodes := replaceNode node result.state state.system.nodes + }, result)) = some (system, output) + rw [found] + rfl + refine + ⟨recordEffects node output.state output.effects + { state with system }, ?_⟩ + simp [next, active, stepResult, accepted, output, system] + +theorem retry_gossip_enqueued + {config : Config} + {before after : State} + {node : Location} + (valid : config.Valid) + (wellFormed : WellFormed config before) + (phase : HasPhase before node .gossiping) + (transition : next config before (.retry node) = some after) : + exists envelope, + envelope ∈ after.network /\ + envelope.source = node /\ + envelope.target = node /\ + exists txid, envelope.payload = .gossip txid := by + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨active, sourceState, found, _, stateEq⟩ + rcases phase with ⟨phaseState, foundPhase, gossiping⟩ + have configured := + wellFormed.activeConfigured node active + rcases recovered_for_configured valid configured with + ⟨txid, recovered⟩ + rw [found] at foundPhase + injection foundPhase with stateEq' + subst phaseState + let envelope : Envelope := { + source := node + target := node + payload := .gossip txid + sourceState + } + have message : envelope ∈ retryMessages config node sourceState := + retryMessages_self_gossip gossiping configured recovered + rw [←stateEq] + exact + ⟨envelope, List.mem_append_right _ message, rfl, rfl, txid, rfl⟩ + +theorem retry_vote_enqueued + {config : Config} + {before after : State} + {node target : Location} + {nodeState : NodeState} + (found : Global.nodeState before node = some nodeState) + (phase : nodeState.phase = .voting) + (chosen : nodeState.chosen = some target) + (transition : next config before (.retry node) = some after) : + exists envelope, + envelope ∈ after.network /\ + envelope.source = node /\ + envelope.target = target /\ + envelope.payload = .vote := by + have message := retryMessages_vote (config := config) + (node := node) phase chosen + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, actualState, actualFound, _, stateEq⟩ + rw [found] at actualFound + injection actualFound with actualEq + subst actualState + let envelope : Envelope := { + source := node + target + payload := .vote + sourceState := nodeState + } + rw [←stateEq] + exact + ⟨envelope, List.mem_append_right _ message, rfl, rfl, rfl⟩ + +theorem insertGossip_nonempty + (source : Location) + (txid : TxID) + (gossips : List (Prod Location TxID)) : + insertGossip source txid gossips ≠ [] := by + unfold insertGossip + split + · rename_i present + intro empty + subst gossips + simp at present + · intro empty + have lengths := + (List.mergeSort_perm ((source, txid) :: gossips) + (fun left right => left.1 <= right.1)).length_eq + rw [empty] at lengths + simp at lengths + +theorem maximumGossip_some + {gossips : List (Prod Location TxID)} + (nonempty : gossips ≠ []) : + exists selected, maximumGossip gossips = some selected := by + cases gossips with + | nil => contradiction + | cons head tail => + exact ⟨tail.foldl selectMaximum head, rfl⟩ + +theorem gossip_receive_progress + (config : Protocol.Config) + (state : NodeState) + (source : Location) + (txid : TxID) + (valid : LaneValid state) + (phase : state.phase = .gossiping) : + let output := + step config state (.receiveGossip source txid .accepted) + output.state.phase ≠ .gossiping \/ + output.state.gossips ≠ [] := by + have chosen := valid.2.2.2 phase + have nonempty := insertGossip_nonempty source txid state.gossips + obtain ⟨selected, maximum⟩ := maximumGossip_some nonempty + simp [step, phase, chosen, rejected, advance, advanceTimeoutLane, + validTimeout] + repeat first | split | simp_all + +theorem gossip_timeout_progress + (config : Protocol.Config) + (state : NodeState) + (valid : LaneValid state) + (phase : state.phase = .gossiping) + (accepted : (step config state .timeout).accepted = true) : + (step config state .timeout).state.phase = .voting := by + have lane := valid.1 phase + simp [step, phase, lane, rejected, advance, advanceTimeoutLane, + validTimeout] at accepted ⊢ + repeat first | split at accepted | split | simp_all + +theorem gossip_timeout_enabled_local + (config : Protocol.Config) + (state : NodeState) + (valid : LaneValid state) + (phase : state.phase = .gossiping) + (nonempty : state.gossips ≠ []) : + (step config state .timeout).accepted = true := by + have lane := valid.1 phase + obtain ⟨selected, maximum⟩ := maximumGossip_some nonempty + simp [step, phase, lane, advance, validTimeout, advanceTimeoutLane, + maximum] + +theorem gossip_timeout_enabled + {config : Config} + {state : State} + {node : Location} + (active : node ∈ state.active) + (lanes : NodeLanesValid state) + (phase : HasPhase state node .gossiping) + (gossip : HasGossip state node) : + Enabled config state (.timeout node) := by + rcases phase with ⟨phaseState, foundPhase, gossiping⟩ + rcases gossip with ⟨gossipState, foundGossip, nonempty⟩ + rw [foundPhase] at foundGossip + injection foundGossip with stateEq + subst gossipState + have lane := node_property_of_nodeState lanes foundPhase + apply timeout_enabled_of_accepted active foundPhase + exact gossip_timeout_enabled_local config.protocol phaseState lane + gossiping nonempty + +def openingDistance : Phase -> Nat + | .gossiping => 3 + | .voting => 2 + | .opening => 1 + | .joining | .open => 0 + +theorem opening_timeout_local + (config : Protocol.Config) + (state : NodeState) + (valid : LaneValid state) + (phase : state.phase = .opening) : + let output := step config state .timeout + (output.effects = [.completed] /\ output.state.phase = .open) \/ + (output.state.phase = .opening /\ + openingDistance output.state.timeoutState < + openingDistance state.timeoutState) := by + rcases valid.2.2.1 phase with lane | lane | lane + · simp [step, phase, lane, advance, validTimeout, advanceTimeoutLane, + advanceTimeoutState, openingDistance] + · simp [step, phase, lane, advance, validTimeout, advanceTimeoutLane, + advanceTimeoutState, openingDistance] + · simp [step, phase, lane, advance, validTimeout, advanceTimeoutLane, + advanceTimeoutState, openingDistance] + +theorem opening_step_distance_le + (config : Protocol.Config) + (state : NodeState) + (event : Event) + (valid : LaneValid state) + (phase : state.phase = .opening) + (after : (step config state event).state.phase = .opening) : + openingDistance (step config state event).state.timeoutState <= + openingDistance state.timeoutState := by + cases event with + | receiveGossip source txid validation => + cases validation <;> + simp [step, phase, rejected, advance, validTimeout, + advanceTimeoutLane] + repeat first | split | simp_all + | receiveVote source validation => + cases validation <;> + simp [step, phase, rejected, advance, validTimeout, + advanceTimeoutLane] + | receiveIAmOpen source validation => + cases validation <;> simp [step, phase, rejected] + | timeout => + rcases opening_timeout_local config state valid phase with + done | progress + · rw [done.2] at after + contradiction + · exact Nat.le_of_lt progress.2 + | retry => simp [step] + +theorem opening_step_or_completed + (config : Protocol.Config) + (state : NodeState) + (event : Event) + (phase : state.phase = .opening) : + (step config state event).state.phase = .opening \/ + ((step config state event).state.phase = .open /\ + .completed ∈ (step config state event).effects) := by + cases event with + | receiveGossip source txid validation => + cases validation <;> + simp [step, phase, rejected, advance, validTimeout, + advanceTimeoutLane] + repeat first | split | simp_all + | receiveVote source validation => + cases validation <;> + simp [step, phase, rejected, advance, validTimeout, + advanceTimeoutLane] + | receiveIAmOpen source validation => + cases validation <;> simp [step, phase, rejected] + | timeout => + simp [step, phase, rejected, advance, validTimeout, + advanceTimeoutLane] + split <;> simp_all + | retry => simp [step, phase] + +theorem opening_non_timeout + (config : Protocol.Config) + (state : NodeState) + (event : Event) + (phase : state.phase = .opening) + (notTimeout : event ≠ .timeout) : + (step config state event).state.phase = .opening /\ + (step config state event).state.timeoutState = + state.timeoutState := by + cases event with + | receiveGossip source txid validation => + cases validation <;> + simp [step, phase, rejected, advance, validTimeout, + advanceTimeoutLane] + repeat first | split | simp_all + | receiveVote source validation => + cases validation <;> + simp [step, phase, rejected, advance, validTimeout, + advanceTimeoutLane] + | receiveIAmOpen source validation => + cases validation <;> simp [step, phase, rejected] + | timeout => contradiction + | retry => exact ⟨phase, rfl⟩ + +theorem opening_timeout_enabled + {config : Config} + {state : State} + {node : Location} + (active : node ∈ state.active) + (phase : HasPhase state node .opening) : + Enabled config state (.timeout node) := by + rcases phase with ⟨nodeState, found, opening⟩ + apply timeout_enabled_of_accepted active found + simp [step, opening, advance, rejected] + repeat first | split | simp_all + +theorem timeout_opening_step + {config : Config} + {before after : State} + {node : Location} + {beforeState : NodeState} + (wellFormed : WellFormed config before) + (lanes : NodeLanesValid before) + (foundBefore : + Global.nodeState before node = some beforeState) + (opening : beforeState.phase = .opening) + (transition : next config before (.timeout node) = some after) : + CompletedOpen after node \/ + (exists nextState : NodeState, + Global.nodeState after node = some nextState /\ + nextState.phase = .opening /\ + openingDistance nextState.timeoutState < + openingDistance beforeState.timeoutState) := by + have lane : LaneValid beforeState := by + apply node_property_of_nodeState (predicate := LaneValid) + · exact lanes + · exact foundBefore + have timeoutResult : + ((step config.protocol beforeState .timeout).effects = + [.completed] /\ + (step config.protocol beforeState .timeout).state.phase = .open) \/ + ((step config.protocol beforeState .timeout).state.phase = + .opening /\ + openingDistance + (step config.protocol beforeState .timeout).state.timeoutState < + openingDistance beforeState.timeoutState) := + opening_timeout_local config.protocol beforeState lane opening + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, systemStep, _, stateEq⟩ + have outputEq := systemStep_output_eq foundBefore systemStep + rw [←outputEq] at timeoutResult + rw [←stateEq] + rcases timeoutResult with completed | progress + · exact Or.inl (by + rcases completed with ⟨effects, _⟩ + rw [effects] + simp [CompletedOpen, recordEffects, recordEffect]) + · exact Or.inr + ⟨output.state, + (by + apply nodeState_eq_of_mem + · rw [recordEffects_system, + systemStep_node_keys_eq systemStep] + exact wellFormed.nodeKeysNodup + · rw [recordEffects_system] + exact systemStep_output_mem systemStep), + progress.1, + by simpa using progress.2⟩ + +theorem next_opening_progress + {config : Config} + {before after : State} + {node : Location} + {beforeState : NodeState} + (wellFormed : WellFormed config before) + (lanes : NodeLanesValid before) + (foundBefore : + Global.nodeState before node = some beforeState) + (opening : beforeState.phase = .opening) + (transition : next config before action = some after) : + CompletedOpen after node \/ + (exists afterState : NodeState, + Global.nodeState after node = some afterState /\ + afterState.phase = .opening /\ + openingDistance afterState.timeoutState <= + openingDistance beforeState.timeoutState) := by + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, sourceState, _, _, stateEq⟩ + rw [←stateEq] + exact Or.inr + ⟨beforeState, foundBefore, opening, Nat.le_refl _⟩ + | deliver envelope => + by_cases target : node = envelope.target + · subst node + rcases deliver_target_state wellFormed transition with + ⟨output, foundAfter, systemStep⟩ + have outputEq := + systemStep_output_eq foundBefore systemStep + have preserved := + opening_non_timeout config.protocol beforeState + (eventFor envelope) opening + (by + cases payloadEq : envelope.payload <;> + simp [eventFor, payloadEq]) + rw [←outputEq] at preserved + exact Or.inr + ⟨output.state, foundAfter, preserved.1, + by rw [preserved.2]⟩ + · have unchanged := deliver_other_node_eq target transition + rw [foundBefore] at unchanged + exact Or.inr + ⟨beforeState, by simp [unchanged], opening, Nat.le_refl _⟩ + | timeout target => + by_cases same : node = target + · subst node + rcases timeout_opening_step wellFormed lanes foundBefore + opening transition with + completed | ⟨nextState, foundAfter, nextOpening, distance⟩ + · exact Or.inl completed + · exact Or.inr + ⟨nextState, foundAfter, nextOpening, + Nat.le_of_lt distance⟩ + · have unchanged := timeout_other_node_eq same transition + rw [foundBefore] at unchanged + exact Or.inr + ⟨beforeState, by simp [unchanged], opening, Nat.le_refl _⟩ + +theorem insertVote_nonempty + (source : Location) + (votes : List Location) : + insertVote source votes ≠ [] := by + unfold insertVote + split + · rename_i present + intro empty + subst votes + simp at present + · intro empty + have lengths := + (List.mergeSort_perm (source :: votes) + (fun left right => left <= right)).length_eq + rw [empty] at lengths + simp at lengths + +theorem step_preserves_nonempty_votes + (config : Protocol.Config) + (state : NodeState) + (event : Event) + (nonempty : state.votes ≠ []) : + (step config state event).state.votes ≠ [] := by + rcases step_votes_shape config state event with + unchanged | ⟨source, _, changed⟩ + · rw [unchanged] + exact nonempty + · rw [changed] + exact insertVote_nonempty source state.votes + +theorem next_preserves_hasVote + {config : Config} + {before after : State} + {action : Action} + {node : Location} + (wellFormed : WellFormed config before) + (vote : HasVote before node) + (transition : next config before action = some after) : + HasVote after node := by + rcases vote with ⟨beforeState, foundBefore, nonempty⟩ + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, sourceState, _, _, stateEq⟩ + rw [←stateEq] + exact ⟨beforeState, foundBefore, nonempty⟩ + | deliver envelope => + by_cases target : node = envelope.target + · subst node + rcases deliver_target_state wellFormed transition with + ⟨output, foundAfter, systemStep⟩ + have outputEq := + systemStep_output_eq foundBefore systemStep + exact + ⟨output.state, foundAfter, + by + rw [outputEq] + exact step_preserves_nonempty_votes + config.protocol beforeState (eventFor envelope) nonempty⟩ + · have unchanged := deliver_other_node_eq target transition + rw [foundBefore] at unchanged + exact ⟨beforeState, by simp [unchanged], nonempty⟩ + | timeout target => + by_cases same : node = target + · subst node + rcases timeout_target_state wellFormed transition with + ⟨output, foundAfter, _, systemStep⟩ + have outputEq := + systemStep_output_eq foundBefore systemStep + exact + ⟨output.state, foundAfter, + by + rw [outputEq] + exact step_preserves_nonempty_votes + config.protocol beforeState .timeout nonempty⟩ + · have unchanged := timeout_other_node_eq same transition + rw [foundBefore] at unchanged + exact ⟨beforeState, by simp [unchanged], nonempty⟩ + +theorem hasVote_mono + {config : Config} + (execution : Execution config) + (initial : Reachable config (execution.states 0)) + {start finish : Nat} + {node : Location} + (order : start <= finish) + (vote : HasVote (execution.states start) node) : + HasVote (execution.states finish) node := by + induction finish, order using Nat.le_induction with + | base => exact vote + | succ finish order vote => + exact next_preserves_hasVote + (reachable_well_formed + (execution_reachable execution initial finish)) + vote (execution.step_succ finish) + +theorem next_preserves_advanced_lane + {config : Config} + {before after : State} + {action : Action} + {node : Location} + (wellFormed : WellFormed config before) + (advanced : LaneAdvanced before node) + (transition : next config before action = some after) : + LaneAdvanced after node := by + rcases advanced with ⟨beforeState, foundBefore, lane⟩ + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, sourceState, _, _, stateEq⟩ + rw [←stateEq] + exact ⟨beforeState, foundBefore, lane⟩ + | deliver envelope => + by_cases target : node = envelope.target + · subst node + rcases deliver_target_state wellFormed transition with + ⟨output, foundAfter, systemStep⟩ + have outputEq := + systemStep_output_eq foundBefore systemStep + exact + ⟨output.state, foundAfter, + by + rw [outputEq] + exact step_preserves_advanced_lane + config.protocol beforeState (eventFor envelope) lane⟩ + · have unchanged := deliver_other_node_eq target transition + rw [foundBefore] at unchanged + exact ⟨beforeState, by simp [unchanged], lane⟩ + | timeout target => + by_cases same : node = target + · subst node + rcases timeout_target_state wellFormed transition with + ⟨output, foundAfter, _, systemStep⟩ + have outputEq := + systemStep_output_eq foundBefore systemStep + exact + ⟨output.state, foundAfter, + by + rw [outputEq] + exact step_preserves_advanced_lane + config.protocol beforeState .timeout lane⟩ + · have unchanged := timeout_other_node_eq same transition + rw [foundBefore] at unchanged + exact ⟨beforeState, by simp [unchanged], lane⟩ + +theorem advanced_lane_mono + {config : Config} + (execution : Execution config) + (initial : Reachable config (execution.states 0)) + {start finish : Nat} + {node : Location} + (order : start <= finish) + (advanced : LaneAdvanced (execution.states start) node) : + LaneAdvanced (execution.states finish) node := by + induction finish, order using Nat.le_induction with + | base => exact advanced + | succ finish order advanced => + exact next_preserves_advanced_lane + (reachable_well_formed + (execution_reachable execution initial finish)) + advanced (execution.step_succ finish) + +theorem opening_progress_between + {config : Config} + (execution : Execution config) + (initial : Reachable config (execution.states 0)) + {start finish : Nat} + {node : Location} + {startState : NodeState} + (order : start <= finish) + (foundStart : + Global.nodeState (execution.states start) node = some startState) + (openingStart : startState.phase = .opening) + (notCompleted : + Not (CompletedOpen (execution.states finish) node)) : + exists finishState : NodeState, + Global.nodeState (execution.states finish) node = some finishState /\ + finishState.phase = .opening /\ + openingDistance finishState.timeoutState <= + openingDistance startState.timeoutState := by + induction finish, order using Nat.le_induction with + | base => + exact + ⟨startState, foundStart, openingStart, Nat.le_refl _⟩ + | succ finish order ih => + have notCompletedBefore : + Not (CompletedOpen (execution.states finish) node) := by + intro completed + exact notCompleted + (next_completed_monotonic + (execution.step_succ finish) node completed) + rcases ih notCompletedBefore with + ⟨beforeState, foundBefore, openingBefore, distanceBefore⟩ + rcases next_opening_progress + (reachable_well_formed + (execution_reachable execution initial finish)) + (reachable_lanes_valid + (execution_reachable execution initial finish)) + foundBefore openingBefore (execution.step_succ finish) with + completed | + ⟨afterState, foundAfter, openingAfter, distanceAfter⟩ + · contradiction + · exact + ⟨afterState, foundAfter, openingAfter, + Nat.le_trans distanceAfter distanceBefore⟩ + +theorem deliver_gossip_progress + {config : Config} + {before after : State} + {envelope : Envelope} + (wellFormed : WellFormed config before) + (lanes : NodeLanesValid before) + (payload : exists txid, envelope.payload = .gossip txid) + (phase : HasPhase before envelope.target .gossiping) + (transition : next config before (.deliver envelope) = some after) : + Not (HasPhase after envelope.target .gossiping) \/ + HasGossip after envelope.target := by + rcases phase with ⟨beforeState, foundBefore, gossiping⟩ + rcases payload with ⟨txid, payload⟩ + rcases deliver_target_state wellFormed transition with + ⟨output, foundAfter, systemStep⟩ + have outputEq := + systemStep_output_eq foundBefore systemStep + have lane := + node_property_of_nodeState lanes foundBefore + simp [eventFor, payload] at outputEq + have progress := + gossip_receive_progress config.protocol beforeState + envelope.source txid lane gossiping + rw [←outputEq] at progress + rcases progress with left | right + · exact Or.inl (by + intro stillGossiping + rcases stillGossiping with ⟨state, found, phase⟩ + rw [foundAfter] at found + injection found with stateEq + subst state + exact left phase) + · exact Or.inr ⟨output.state, foundAfter, right⟩ + +theorem timeout_gossip_progress + {config : Config} + {before after : State} + {node : Location} + (wellFormed : WellFormed config before) + (lanes : NodeLanesValid before) + (phase : HasPhase before node .gossiping) + (transition : next config before (.timeout node) = some after) : + HasPhase after node .voting := by + rcases phase with ⟨beforeState, foundBefore, gossiping⟩ + rcases timeout_target_state wellFormed transition with + ⟨output, foundAfter, accepted, systemStep⟩ + have outputEq := + systemStep_output_eq foundBefore systemStep + have lane := + node_property_of_nodeState lanes foundBefore + have voting := + gossip_timeout_progress config.protocol beforeState lane + gossiping (by simpa [outputEq] using accepted) + exact + ⟨output.state, foundAfter, by simpa [outputEq] using voting⟩ + +theorem vote_receive_progress + (config : Protocol.Config) + (state : NodeState) + (source : Location) + (phase : state.phase = .voting) : + let output := step config state (.receiveVote source .accepted) + output.state.phase ≠ .voting \/ output.state.votes ≠ [] := by + have nonempty := insertVote_nonempty source state.votes + simp [step, phase, rejected, advance, validTimeout, + advanceTimeoutLane] + repeat first | split | simp_all + +theorem voting_timeout_local + (config : Protocol.Config) + (state : NodeState) + (valid : LaneValid state) + (phase : state.phase = .voting) + (nonempty : state.votes ≠ []) : + let output := step config state .timeout + output.state.phase = .opening \/ + (output.state.phase = .voting /\ + output.state.timeoutState = .voting) := by + rcases valid.2.1 phase with lane | lane + · simp [step, phase, lane, rejected, advance, validTimeout, + advanceTimeoutLane, advanceTimeoutState] + repeat first | split | simp_all + · simp [step, phase, lane, nonempty, rejected, advance, validTimeout, + advanceTimeoutLane, advanceTimeoutState] + +theorem aligned_voting_timeout_opens + (config : Protocol.Config) + (state : NodeState) + (phase : state.phase = .voting) + (lane : state.timeoutState = .voting) + (nonempty : state.votes ≠ []) : + (step config state .timeout).state.phase = .opening := by + simp [step, phase, lane, nonempty, rejected, advance, validTimeout, + advanceTimeoutLane] + +theorem deliver_vote_progress + {config : Config} + {before after : State} + {envelope : Envelope} + (wellFormed : WellFormed config before) + (payload : envelope.payload = .vote) + (phase : HasPhase before envelope.target .voting) + (transition : next config before (.deliver envelope) = some after) : + Not (HasPhase after envelope.target .voting) \/ + HasVote after envelope.target := by + rcases phase with ⟨beforeState, foundBefore, voting⟩ + rcases deliver_target_state wellFormed transition with + ⟨output, foundAfter, systemStep⟩ + have outputEq := + systemStep_output_eq foundBefore systemStep + simp [eventFor, payload] at outputEq + have progress := + vote_receive_progress config.protocol beforeState + envelope.source voting + rw [←outputEq] at progress + rcases progress with left | right + · exact Or.inl (by + intro stillVoting + rcases stillVoting with ⟨state, found, phase⟩ + rw [foundAfter] at found + injection found with stateEq + subst state + exact left phase) + · exact Or.inr ⟨output.state, foundAfter, right⟩ + +theorem deliver_iamopen_resolves + {config : Config} + {before after : State} + {envelope : Envelope} + (wellFormed : WellFormed config before) + (openCompleted : OpenCompleted before) + (payload : envelope.payload = .iAmOpen) + (transition : next config before (.deliver envelope) = some after) : + Terminal after envelope.target \/ + HasPhase after envelope.target .opening := by + have details := transition + simp [next, Option.bind_eq_some_iff] at details + rcases details with + ⟨_, targetActive, system, output, systemStep, stateEq⟩ + rcases active_nodeState wellFormed targetActive with + ⟨beforeState, foundBefore⟩ + have outputEq := systemStep_output_eq foundBefore systemStep + simp [eventFor, payload] at outputEq + have outcome := + iamopen_delivery_outcome config.protocol beforeState envelope.source + rw [←outputEq] at outcome + rw [←stateEq] + rcases outcome with opening | opened | ⟨chosen, restarted⟩ + · exact Or.inr + ⟨output.state, + by + apply nodeState_eq_of_mem + · rw [recordEffects_system, + systemStep_node_keys_eq systemStep] + exact wellFormed.nodeKeysNodup + · rw [recordEffects_system] + exact systemStep_output_mem systemStep, + opening⟩ + · have beforeOpen := + iamopen_open_predecessor config.protocol beforeState + envelope.source (by simpa [outputEq] using opened) + have completedBefore : CompletedOpen before envelope.target := by + rw [Global.nodeState, Option.map_eq_some_iff] at foundBefore + rcases foundBefore with ⟨entry, findEq, stateEq⟩ + have keyEq : entry.1 = envelope.target := + beq_iff_eq.mp + (List.find?_some + (p := fun entry : Prod Location NodeState => + entry.1 == envelope.target) findEq) + rw [←keyEq] + apply openCompleted entry (List.mem_of_find?_eq_some findEq) + simpa [stateEq] using beforeOpen + exact Or.inl (Or.inr + (mem_completed_recordEffects completedBefore)) + · exact Or.inl (Or.inl + (restart_effect_recorded restarted)) + +theorem voting_timeout_enabled + {config : Config} + {state : State} + {node : Location} + (active : node ∈ state.active) + (phase : HasPhase state node .voting) : + Enabled config state (.timeout node) := by + rcases phase with ⟨nodeState, found, voting⟩ + apply timeout_enabled_of_accepted active found + simp [step, voting, advance, rejected] + repeat first | split | simp_all + +theorem timeout_voting_step + {config : Config} + {before after : State} + {node : Location} + (wellFormed : WellFormed config before) + (lanes : NodeLanesValid before) + (phase : HasPhase before node .voting) + (vote : HasVote before node) + (transition : next config before (.timeout node) = some after) : + HasPhase after node .opening \/ + (exists nextState : NodeState, + Global.nodeState after node = some nextState /\ + nextState.phase = .voting /\ + nextState.timeoutState = .voting /\ + nextState.votes ≠ []) := by + rcases phase with ⟨beforeState, foundBefore, voting⟩ + rcases vote with ⟨voteState, foundVote, nonempty⟩ + rw [foundBefore] at foundVote + injection foundVote with stateEq + subst voteState + have lane : LaneValid beforeState := by + apply node_property_of_nodeState (predicate := LaneValid) + · exact lanes + · exact foundBefore + rcases timeout_target_state wellFormed transition with + ⟨output, foundAfter, _, systemStep⟩ + have outputEq := systemStep_output_eq foundBefore systemStep + have progress := + voting_timeout_local config.protocol beforeState lane voting nonempty + rw [←outputEq] at progress + rcases progress with opening | waiting + · exact Or.inl ⟨output.state, foundAfter, opening⟩ + · exact Or.inr + ⟨output.state, foundAfter, waiting.1, waiting.2, + by + rw [outputEq] + exact step_preserves_nonempty_votes + config.protocol beforeState .timeout nonempty⟩ + +theorem aligned_timeout_voting_opens + {config : Config} + {before after : State} + {node : Location} + {nodeState : NodeState} + (wellFormed : WellFormed config before) + (found : Global.nodeState before node = some nodeState) + (phase : nodeState.phase = .voting) + (lane : nodeState.timeoutState = .voting) + (nonempty : nodeState.votes ≠ []) + (transition : next config before (.timeout node) = some after) : + HasPhase after node .opening := by + rcases timeout_target_state wellFormed transition with + ⟨output, foundAfter, _, systemStep⟩ + have outputEq := systemStep_output_eq found systemStep + exact + ⟨output.state, foundAfter, + by + rw [outputEq] + exact aligned_voting_timeout_opens config.protocol nodeState + phase lane nonempty⟩ + +theorem fair_gossip_progress + {config : Config} + (execution : Execution config) + (initial : Reachable config (execution.states 0)) + (fair : Fair execution) + {start : Nat} + {node : Location} + (active : node ∈ (execution.states start).active) + (phase : HasPhase (execution.states start) node .gossiping) : + EventuallyFrom start (fun n => + Not (HasPhase (execution.states n) node .gossiping)) := by + have reachable (n : Nat) := + execution_reachable execution initial n + have configValid := reachable_config_valid (reachable start) + have retryEnabled := + retry_gossip_enabled configValid + (reachable_well_formed (reachable start)) active phase + rcases fair.retry start node .gossiping active phase + (Or.inl rfl) retryEnabled with + ⟨retryAt, startRetry, leftGossip | retryAction⟩ + · exact ⟨retryAt, startRetry, leftGossip⟩ + · by_cases retryPhase : + HasPhase (execution.states retryAt) node .gossiping + · have retryStep : + next config (execution.states retryAt) (.retry node) = + some (execution.states (retryAt + 1)) := by + simpa [retryAction] using execution.step_succ retryAt + rcases retry_gossip_enqueued configValid + (reachable_well_formed (reachable retryAt)) + retryPhase retryStep with + ⟨envelope, pending, sourceEq, targetEq, txid, payload⟩ + rcases fair.delivery (retryAt + 1) envelope pending with + ⟨deliverAt, retryDeliver, deliverAction⟩ + by_cases deliverPhase : + HasPhase (execution.states deliverAt) node .gossiping + · have deliverStep : + next config (execution.states deliverAt) + (.deliver envelope) = + some (execution.states (deliverAt + 1)) := by + simpa [deliverAction] using execution.step_succ deliverAt + have delivered := + deliver_gossip_progress + (reachable_well_formed (reachable deliverAt)) + (reachable_lanes_valid (reachable deliverAt)) + ⟨txid, payload⟩ + (by simpa [targetEq] using deliverPhase) + deliverStep + rcases delivered with leftAfter | hasGossip + · exact + ⟨deliverAt + 1, by omega, by simpa [targetEq] using leftAfter⟩ + · by_cases afterPhase : + HasPhase (execution.states (deliverAt + 1)) node .gossiping + · have timeoutEnabled := + gossip_timeout_enabled + (config := config) + (by + rw [execution_active_eq execution (deliverAt + 1)] + rw [execution_active_eq execution start] at active + exact active) + (reachable_lanes_valid (reachable (deliverAt + 1))) + afterPhase + (by simpa [targetEq] using hasGossip) + rcases fair.timeout (deliverAt + 1) node .gossiping + (by + rw [execution_active_eq execution (deliverAt + 1)] + rw [execution_active_eq execution start] at active + exact active) + afterPhase (Or.inl rfl) timeoutEnabled with + ⟨timeoutAt, deliverTimeout, leftBeforeTimeout | timeoutAction⟩ + · exact ⟨timeoutAt, by omega, leftBeforeTimeout⟩ + · by_cases timeoutPhase : + HasPhase (execution.states timeoutAt) node .gossiping + · have timeoutStep : + next config (execution.states timeoutAt) + (.timeout node) = + some (execution.states (timeoutAt + 1)) := by + simpa [timeoutAction] using + execution.step_succ timeoutAt + have voting := + timeout_gossip_progress + (reachable_well_formed (reachable timeoutAt)) + (reachable_lanes_valid (reachable timeoutAt)) + timeoutPhase timeoutStep + refine ⟨timeoutAt + 1, by omega, ?_⟩ + intro impossible + have phases := hasPhase_unique voting impossible + contradiction + · exact ⟨timeoutAt, by omega, timeoutPhase⟩ + · exact ⟨deliverAt + 1, by omega, afterPhase⟩ + · exact ⟨deliverAt, by omega, deliverPhase⟩ + · exact ⟨retryAt, startRetry, retryPhase⟩ + +theorem next_gossiping_predecessor + {config : Config} + {before after : State} + {action : Action} + {node : Location} + (wellFormed : WellFormed config before) + (transition : next config before action = some after) + (afterGossip : HasPhase after node .gossiping) : + HasPhase before node .gossiping := by + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with ⟨_, sourceState, _, _, stateEq⟩ + rw [←stateEq] at afterGossip + exact afterGossip + | deliver envelope => + by_cases target : node = envelope.target + · subst node + have details := transition + simp [next, Option.bind_eq_some_iff] at details + have targetActive := details.2.1 + rcases active_nodeState wellFormed targetActive with + ⟨beforeState, foundBefore⟩ + rcases deliver_target_state wellFormed transition with + ⟨output, foundAfter, systemStep⟩ + rcases afterGossip with ⟨afterState, found, phase⟩ + rw [foundAfter] at found + injection found with stateEq + subst afterState + have outputEq := + systemStep_output_eq foundBefore systemStep + have beforePhase : beforeState.phase = .gossiping := by + by_contra notGossip + have notAfter := + step_preserves_non_gossiping config.protocol beforeState + (eventFor envelope) notGossip + rw [←outputEq] at notAfter + exact notAfter phase + exact ⟨beforeState, foundBefore, beforePhase⟩ + · rcases afterGossip with ⟨afterState, foundAfter, phase⟩ + have unchanged := + deliver_other_node_eq target transition + rw [foundAfter] at unchanged + cases foundBefore : + Global.nodeState before node with + | none => simp [foundBefore] at unchanged + | some beforeState => + rw [foundBefore] at unchanged + injection unchanged with stateEq + subst beforeState + exact ⟨afterState, foundBefore, phase⟩ + | timeout target => + by_cases same : node = target + · subst node + have details := transition + simp [next, Option.bind_eq_some_iff] at details + have targetActive := details.1 + rcases active_nodeState wellFormed targetActive with + ⟨beforeState, foundBefore⟩ + rcases timeout_target_state wellFormed transition with + ⟨output, foundAfter, _, systemStep⟩ + rcases afterGossip with ⟨afterState, found, phase⟩ + rw [foundAfter] at found + injection found with stateEq + subst afterState + have outputEq := + systemStep_output_eq foundBefore systemStep + have beforePhase : beforeState.phase = .gossiping := by + by_contra notGossip + have notAfter := + step_preserves_non_gossiping config.protocol beforeState + .timeout notGossip + rw [←outputEq] at notAfter + exact notAfter phase + exact ⟨beforeState, foundBefore, beforePhase⟩ + · rcases afterGossip with ⟨afterState, foundAfter, phase⟩ + have unchanged := + timeout_other_node_eq same transition + rw [foundAfter] at unchanged + cases foundBefore : + Global.nodeState before node with + | none => simp [foundBefore] at unchanged + | some beforeState => + rw [foundBefore] at unchanged + injection unchanged with stateEq + subst beforeState + exact ⟨afterState, foundBefore, phase⟩ + +theorem not_gossiping_mono + {config : Config} + (execution : Execution config) + (initial : Reachable config (execution.states 0)) + {start finish : Nat} + {node : Location} + (order : start <= finish) + (notGossip : + Not (HasPhase (execution.states start) node .gossiping)) : + Not (HasPhase (execution.states finish) node .gossiping) := by + induction finish, order using Nat.le_induction with + | base => exact notGossip + | succ finish order notGossip => + intro gossip + exact notGossip + (next_gossiping_predecessor + (reachable_well_formed + (execution_reachable execution initial finish)) + (execution.step_succ finish) gossip) + +theorem eventually_list + {predicate : Nat -> Location -> Prop} + {start : Nat} + (nodes : List Location) + (eventual : + forall node, node ∈ nodes -> + EventuallyFrom start (fun n => predicate n node)) + (monotonic : + forall node first second, + first <= second -> + predicate first node -> + predicate second node) : + EventuallyFrom start (fun n => + forall node, node ∈ nodes -> predicate n node) := by + revert eventual + induction nodes with + | nil => + intro eventual + exact ⟨start, Nat.le_refl start, by simp⟩ + | cons head tail ih => + intro eventual + rcases eventual head (by simp) with + ⟨headAt, startHead, headHolds⟩ + rcases ih + (fun node membership => eventual node (by simp [membership])) with + ⟨tailAt, startTail, tailHolds⟩ + refine + ⟨max headAt tailAt, by omega, ?_⟩ + intro node membership + rw [List.mem_cons] at membership + rcases membership with rfl | inTail + · exact monotonic _ headAt (max headAt tailAt) + (Nat.le_max_left _ _) headHolds + · exact monotonic node tailAt (max headAt tailAt) + (Nat.le_max_right _ _) (tailHolds node inTail) + +theorem fair_all_leave_gossip + {config : Config} + (execution : Execution config) + (initial : Reachable config (execution.states 0)) + (fair : Fair execution) + (start : Nat) : + EventuallyFrom start (fun n => + forall node, node ∈ (execution.states start).active -> + Not (HasPhase (execution.states n) node .gossiping)) := by + apply eventually_list (execution.states start).active + · intro node active + by_cases phase : + HasPhase (execution.states start) node .gossiping + · exact fair_gossip_progress execution initial fair active phase + · exact ⟨start, Nat.le_refl start, phase⟩ + · intro node first second order notGossip + exact not_gossiping_mono execution initial order notGossip + +theorem terminal_mono_step + {config : Config} + {before after : State} + {action : Action} + {node : Location} + (transition : next config before action = some after) + (terminal : Terminal before node) : + Terminal after node := by + rcases terminal with restarted | completed + · exact Or.inl (next_restarts_monotonic transition node restarted) + · exact Or.inr (next_completed_monotonic transition node completed) + +theorem terminal_mono + {config : Config} + (execution : Execution config) + {start finish : Nat} + {node : Location} + (order : start <= finish) + (terminal : Terminal (execution.states start) node) : + Terminal (execution.states finish) node := by + induction finish, order using Nat.le_induction with + | base => exact terminal + | succ finish order terminal => + exact terminal_mono_step (execution.step_succ finish) terminal + +theorem completed_mono + {config : Config} + (execution : Execution config) + {start finish : Nat} + {node : Location} + (order : start <= finish) + (completed : CompletedOpen (execution.states start) node) : + CompletedOpen (execution.states finish) node := by + induction finish, order using Nat.le_induction with + | base => exact completed + | succ finish order completed => + exact next_completed_monotonic + (execution.step_succ finish) node completed + +theorem quorumOpened_mono + {config : Config} + (execution : Execution config) + {start finish : Nat} + {node : Location} + (order : start <= finish) + (opened : QuorumOpened (execution.states start) node) : + QuorumOpened (execution.states finish) node := by + induction finish, order using Nat.le_induction with + | base => exact opened + | succ finish order opened => + rcases opened with + ⟨opening, membership, openingNode, kind⟩ + exact + ⟨opening, + next_openings_monotonic + (execution.step_succ finish) opening membership, + openingNode, + kind⟩ + +theorem fair_opening_completes + {config : Config} + (execution : Execution config) + (initial : Reachable config (execution.states 0)) + (fair : Fair execution) + {start : Nat} + {node : Location} + (active : node ∈ (execution.states start).active) + (phase : HasPhase (execution.states start) node .opening) : + EventuallyFrom start (fun n => + CompletedOpen (execution.states n) node) := by + rcases phase with ⟨startState, foundStart, openingStart⟩ + have auxiliary : + forall distance start state, + openingDistance state.timeoutState = distance -> + node ∈ (execution.states start).active -> + Global.nodeState (execution.states start) node = some state -> + state.phase = .opening -> + EventuallyFrom start (fun n => + CompletedOpen (execution.states n) node) := by + intro distance + induction distance using Nat.strong_induction_on with + | h distance ih => + intro start state distanceEq active found opening + have enabled := + opening_timeout_enabled (config := config) + active ⟨state, found, opening⟩ + rcases fair.openingTimeout start node active + ⟨state, found, opening⟩ enabled with + ⟨timeoutAt, startTimeout, + completed | ⟨stillOpening, timeoutAction⟩⟩ + · exact ⟨timeoutAt, startTimeout, completed⟩ + · by_cases completedBefore : + CompletedOpen (execution.states timeoutAt) node + · exact ⟨timeoutAt, startTimeout, completedBefore⟩ + · rcases opening_progress_between execution initial startTimeout + found opening completedBefore with + ⟨timeoutState, foundTimeout, openingTimeout, + distanceTimeout⟩ + have timeoutStep : + next config (execution.states timeoutAt) + (.timeout node) = + some (execution.states (timeoutAt + 1)) := by + simpa [timeoutAction] using execution.step_succ timeoutAt + rcases timeout_opening_step + (reachable_well_formed + (execution_reachable execution initial timeoutAt)) + (reachable_lanes_valid + (execution_reachable execution initial timeoutAt)) + foundTimeout openingTimeout timeoutStep with + completedAfter | + ⟨nextState, foundNext, openingNext, distanceNext⟩ + · exact ⟨timeoutAt + 1, by omega, completedAfter⟩ + · have nextLess : openingDistance nextState.timeoutState < + distance := by + rw [←distanceEq] + exact Nat.lt_of_lt_of_le distanceNext distanceTimeout + rcases ih (openingDistance nextState.timeoutState) + nextLess (timeoutAt + 1) nextState rfl + (by + rw [execution_active_eq execution (timeoutAt + 1)] + rw [execution_active_eq execution start] at active + exact active) + foundNext openingNext with + ⟨completedAt, nextCompleted, completed⟩ + exact ⟨completedAt, by omega, completed⟩ + exact auxiliary (openingDistance startState.timeoutState) + start startState rfl active foundStart openingStart + +theorem initial_announcements_live + (config : Config) + (active : List Location) : + AnnouncementsLive (initial config active) := by + simp [AnnouncementsLive, Global.initial] + +theorem next_preserves_announcements_live + {config : Config} + {before after : State} + {action : Action} + (wellFormed : WellFormed config before) + (lanes : NodeLanesValid before) + (live : AnnouncementsLive before) + (transition : next config before action = some after) : + AnnouncementsLive after := by + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, sourceState, found, _, stateEq⟩ + have sourceLocation := + nodeState_location wellFormed.nodeLocations found + rw [←stateEq] + intro envelope membership payload + rw [List.mem_append] at membership + rcases membership with old | added + · exact live envelope old payload + · have valid : envelope.Valid config := + retryMessages_valid config source sourceState sourceLocation + envelope added + have opening := retry_iamopen_state valid payload + have identity := retryMessages_source added + rw [identity.2] at opening + exact Or.inl + ⟨sourceState, + by simpa [identity.1] using found, + opening⟩ + | deliver delivered => + intro envelope membership payload + have details := transition + simp [next, Option.bind_eq_some_iff] at details + rcases details with + ⟨_, _, system, output, systemStep, stateEq⟩ + rw [←stateEq, recordEffects_sent] at membership + rcases live envelope membership payload with + opening | completed + · rcases opening with ⟨sourceState, found, phase⟩ + rcases next_opening_progress wellFormed lanes found phase + transition with + completed | ⟨afterState, foundAfter, phaseAfter, _⟩ + · exact Or.inr completed + · exact Or.inl ⟨afterState, foundAfter, phaseAfter⟩ + · exact Or.inr + (next_completed_monotonic transition envelope.source completed) + | timeout target => + intro envelope membership payload + have details := transition + simp [next, Option.bind_eq_some_iff] at details + rcases details with + ⟨_, system, output, systemStep, _, stateEq⟩ + rw [←stateEq, recordEffects_sent] at membership + rcases live envelope membership payload with + opening | completed + · rcases opening with ⟨sourceState, found, phase⟩ + rcases next_opening_progress wellFormed lanes found phase + transition with + completed | ⟨afterState, foundAfter, phaseAfter, _⟩ + · exact Or.inr completed + · exact Or.inl ⟨afterState, foundAfter, phaseAfter⟩ + · exact Or.inr + (next_completed_monotonic transition envelope.source completed) + +theorem reachable_announcements_live + {config : Config} + {state : State} + (reachable : Reachable config state) : + AnnouncementsLive state := by + induction reachable with + | initial active valid nodup configured => + exact initial_announcements_live config active + | step reachable transition live => + exact next_preserves_announcements_live + (reachable_well_formed reachable) + (reachable_lanes_valid reachable) + live transition + +theorem initial_announcements_resolved + (config : Config) + (active : List Location) : + AnnouncementsResolved (initial config active) := by + simp [AnnouncementsResolved, Global.initial] + +theorem next_preserves_announcements_resolved + {config : Config} + {before after : State} + {action : Action} + (wellFormed : WellFormed config before) + (lanes : NodeLanesValid before) + (openCompleted : OpenCompleted before) + (resolved : AnnouncementsResolved before) + (transition : next config before action = some after) : + AnnouncementsResolved after := by + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, sourceState, _, _, stateEq⟩ + rw [←stateEq] + intro envelope membership payload + rw [List.mem_append] at membership + rcases membership with old | added + · rcases resolved envelope old payload with + pending | terminal | opening + · exact Or.inl (List.mem_append_left _ pending) + · exact Or.inr (Or.inl terminal) + · exact Or.inr (Or.inr opening) + · exact Or.inl (List.mem_append_right _ added) + | deliver delivered => + intro envelope membership payload + have details := transition + simp [next, Option.bind_eq_some_iff] at details + rcases details with + ⟨_, _, system, output, systemStep, stateEq⟩ + rw [←stateEq, recordEffects_sent] at membership + rcases resolved envelope membership payload with + pending | terminal | opening + · rcases mem_removeOne_or_eq pending with remains | equal + · exact Or.inl (by + rw [←stateEq, recordEffects_network] + exact remains) + · subst envelope + rcases deliver_iamopen_resolves wellFormed openCompleted payload + transition with + terminal | opening + · exact Or.inr (Or.inl terminal) + · exact Or.inr (Or.inr opening) + · exact Or.inr (Or.inl + (terminal_mono_step transition terminal)) + · rcases opening with ⟨sourceState, found, phase⟩ + rcases next_opening_progress wellFormed lanes found phase + transition with + completed | ⟨afterState, foundAfter, phaseAfter, _⟩ + · exact Or.inr (Or.inl (Or.inr completed)) + · exact Or.inr (Or.inr + ⟨afterState, foundAfter, phaseAfter⟩) + | timeout target => + intro envelope membership payload + have details := transition + simp [next, Option.bind_eq_some_iff] at details + rcases details with + ⟨_, system, output, systemStep, _, stateEq⟩ + rw [←stateEq, recordEffects_sent] at membership + rcases resolved envelope membership payload with + pending | terminal | opening + · exact Or.inl (by + rw [←stateEq, recordEffects_network] + exact pending) + · exact Or.inr (Or.inl + (terminal_mono_step transition terminal)) + · rcases opening with ⟨sourceState, found, phase⟩ + rcases next_opening_progress wellFormed lanes found phase + transition with + completed | ⟨afterState, foundAfter, phaseAfter, _⟩ + · exact Or.inr (Or.inl (Or.inr completed)) + · exact Or.inr (Or.inr + ⟨afterState, foundAfter, phaseAfter⟩) + +theorem systemStep_preserves_joining_announcements + {config : Protocol.Config} + {before after : SystemState} + {target : Location} + {event : Event} + {output : StepOutput} + {beforeState afterState : State} + (beforeSystem : beforeState.system = before) + (valid : JoiningAnnouncements beforeState) + (carry : + forall destination, + SentAnnouncementTo beforeState destination -> + SentAnnouncementTo afterState destination) + (introduced : + (exists source, acceptedIAmOpenSource event = some source) -> + SentAnnouncementTo afterState target) + (transition : + systemStep config before target event = some (after, output)) : + forall entry, entry ∈ after.nodes -> + entry.2.phase = .joining -> + SentAnnouncementTo afterState entry.1 := by + simp [systemStep, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨node, ⟨key, found⟩, systemEq, outputEq⟩ + rw [←systemEq] + intro entry membership joining + rw [replaceNode, List.mem_map] at membership + rcases membership with ⟨previous, previousMember, rfl⟩ + split + · rename_i atTarget + rcases step_joining_origin config node event + (by simpa [atTarget, outputEq] using joining) with + old | received + · have keyEq : key = target := + beq_iff_eq.mp + (List.find?_some + (p := fun entry : Prod Location NodeState => + entry.1 == target) found) + rw [←keyEq] + apply carry + apply valid (key, node) + · rw [beforeSystem] + exact List.mem_of_find?_eq_some found + · exact old + · exact introduced received + · rename_i notTarget + apply carry + apply valid previous + · rw [beforeSystem] + exact previousMember + · simpa [notTarget] using joining + +theorem initial_joining_announcements + (config : Config) + (active : List Location) : + JoiningAnnouncements (initial config active) := by + simp [JoiningAnnouncements, Global.initial, initialSystem, initialNode] + +theorem next_preserves_joining_announcements + {config : Config} + {before after : State} + {action : Action} + (wellFormed : WellFormed config before) + (valid : JoiningAnnouncements before) + (transition : next config before action = some after) : + JoiningAnnouncements after := by + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, sourceState, _, _, stateEq⟩ + rw [←stateEq] + intro entry membership joining + rcases valid entry membership joining with + ⟨envelope, sent, target, payload⟩ + exact + ⟨envelope, List.mem_append_left _ sent, target, payload⟩ + | deliver envelope => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨inNetwork, _, system, output, systemStep, stateEq⟩ + rw [←stateEq] + intro entry membership joining + rw [recordEffects_system] at membership + let afterState := + recordEffects envelope.target output.state output.effects + { + before with + system + network := removeOne envelope before.network + } + have carry : + forall destination, + SentAnnouncementTo before destination -> + SentAnnouncementTo afterState destination := by + intro destination announcement + rcases announcement with + ⟨sentEnvelope, sent, target, payload⟩ + exact + ⟨sentEnvelope, by simpa [afterState] using sent, + target, payload⟩ + have introduced : + (exists source, + acceptedIAmOpenSource (eventFor envelope) = some source) -> + SentAnnouncementTo afterState envelope.target := by + rintro ⟨source, accepted⟩ + rcases eventFor_iamopen_source accepted with + ⟨payload, _⟩ + exact + ⟨envelope, + by + simp [afterState] + exact wellFormed.networkSent envelope inNetwork, + rfl, payload⟩ + exact systemStep_preserves_joining_announcements + (before := before.system) + (after := system) + (beforeState := before) + (afterState := afterState) + rfl valid carry introduced systemStep + entry membership joining + | timeout target => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, systemStep, _, stateEq⟩ + rw [←stateEq] + intro entry membership joining + rw [recordEffects_system] at membership + let afterState := + recordEffects target output.state output.effects + { before with system } + have carry : + forall destination, + SentAnnouncementTo before destination -> + SentAnnouncementTo afterState destination := by + intro destination announcement + rcases announcement with + ⟨sentEnvelope, sent, target, payload⟩ + exact + ⟨sentEnvelope, by simpa [afterState] using sent, + target, payload⟩ + have introduced : + (exists source, + acceptedIAmOpenSource Event.timeout = some source) -> + SentAnnouncementTo afterState target := by + rintro ⟨source, accepted⟩ + simp [acceptedIAmOpenSource] at accepted + exact systemStep_preserves_joining_announcements + (before := before.system) + (after := system) + (beforeState := before) + (afterState := afterState) + rfl valid carry introduced systemStep + entry membership joining + +theorem reachable_joining_announcements + {config : Config} + {state : State} + (reachable : Reachable config state) : + JoiningAnnouncements state := by + induction reachable with + | initial active valid nodup configured => + exact initial_joining_announcements config active + | step reachable transition valid => + exact next_preserves_joining_announcements + (reachable_well_formed reachable) valid transition + +theorem systemStep_preserves_open_completed + {config : Protocol.Config} + {before after : SystemState} + {target : Location} + {event : Event} + {output : StepOutput} + {beforeState afterState : State} + (beforeSystem : beforeState.system = before) + (valid : OpenCompleted beforeState) + (carry : + forall node, + CompletedOpen beforeState node -> + CompletedOpen afterState node) + (introduced : + .completed ∈ output.effects -> + CompletedOpen afterState target) + (transition : + systemStep config before target event = some (after, output)) : + forall entry, entry ∈ after.nodes -> + entry.2.phase = .open -> + CompletedOpen afterState entry.1 := by + simp [systemStep, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨node, ⟨key, found⟩, systemEq, outputEq⟩ + rw [←systemEq] + intro entry membership opened + rw [replaceNode, List.mem_map] at membership + rcases membership with ⟨previous, previousMember, rfl⟩ + split + · rename_i atTarget + rcases step_open_origin config node event + (by simpa [atTarget, outputEq] using opened) with + old | completed + · have keyEq : key = target := + beq_iff_eq.mp + (List.find?_some + (p := fun entry : Prod Location NodeState => + entry.1 == target) found) + rw [←keyEq] + apply carry + apply valid (key, node) + · rw [beforeSystem] + exact List.mem_of_find?_eq_some found + · exact old + · rw [outputEq] at completed + exact introduced completed + · rename_i notTarget + apply carry + apply valid previous + · rw [beforeSystem] + exact previousMember + · simpa [notTarget] using opened + +theorem initial_open_completed + (config : Config) + (active : List Location) : + OpenCompleted (initial config active) := by + simp [OpenCompleted, Global.initial, initialSystem, initialNode] + +theorem next_preserves_open_completed + {config : Config} + {before after : State} + {action : Action} + (valid : OpenCompleted before) + (transition : next config before action = some after) : + OpenCompleted after := by + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, sourceState, _, _, stateEq⟩ + rw [←stateEq] + exact valid + | deliver envelope => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, _, system, output, systemStep, stateEq⟩ + rw [←stateEq] + intro entry membership opened + rw [recordEffects_system] at membership + let afterState := + recordEffects envelope.target output.state output.effects + { + before with + system + network := removeOne envelope before.network + } + have carry : + forall node, + CompletedOpen before node -> + CompletedOpen afterState node := by + intro node completed + apply mem_completed_recordEffects + exact completed + have introduced : + .completed ∈ output.effects -> + CompletedOpen afterState envelope.target := by + intro completed + exact completed_effect_recorded completed + exact systemStep_preserves_open_completed + (before := before.system) + (after := system) + (beforeState := before) + (afterState := afterState) + rfl valid carry introduced systemStep entry membership opened + | timeout target => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, system, output, systemStep, _, stateEq⟩ + rw [←stateEq] + intro entry membership opened + rw [recordEffects_system] at membership + let afterState := + recordEffects target output.state output.effects + { before with system } + have carry : + forall node, + CompletedOpen before node -> + CompletedOpen afterState node := by + intro node completed + apply mem_completed_recordEffects + exact completed + have introduced : + .completed ∈ output.effects -> + CompletedOpen afterState target := by + intro completed + exact completed_effect_recorded completed + exact systemStep_preserves_open_completed + (before := before.system) + (after := system) + (beforeState := before) + (afterState := afterState) + rfl valid carry introduced systemStep entry membership opened + +theorem reachable_open_completed + {config : Config} + {state : State} + (reachable : Reachable config state) : + OpenCompleted state := by + induction reachable with + | initial active valid nodup configured => + exact initial_open_completed config active + | step reachable transition valid => + exact next_preserves_open_completed valid transition + +theorem reachable_announcements_resolved + {config : Config} + {state : State} + (reachable : Reachable config state) : + AnnouncementsResolved state := by + induction reachable with + | initial active valid nodup configured => + exact initial_announcements_resolved config active + | step reachable transition resolved => + exact next_preserves_announcements_resolved + (reachable_well_formed reachable) + (reachable_lanes_valid reachable) + (reachable_open_completed reachable) + resolved transition + +theorem open_node_completed + {state : State} + {node : Location} + {nodeState : NodeState} + (valid : OpenCompleted state) + (found : Global.nodeState state node = some nodeState) + (opened : nodeState.phase = .open) : + CompletedOpen state node := by + rw [Global.nodeState, Option.map_eq_some_iff] at found + rcases found with ⟨entry, findEq, stateEq⟩ + have keyEq : entry.1 = node := + beq_iff_eq.mp + (List.find?_some + (p := fun entry : Prod Location NodeState => + entry.1 == node) findEq) + rw [←keyEq] + apply valid entry (List.mem_of_find?_eq_some findEq) + simpa [stateEq] using opened + +theorem joining_node_announcement + {state : State} + {node : Location} + {nodeState : NodeState} + (valid : JoiningAnnouncements state) + (found : Global.nodeState state node = some nodeState) + (joining : nodeState.phase = .joining) : + SentAnnouncementTo state node := by + rw [Global.nodeState, Option.map_eq_some_iff] at found + rcases found with ⟨entry, findEq, stateEq⟩ + have keyEq : entry.1 = node := + beq_iff_eq.mp + (List.find?_some + (p := fun entry : Prod Location NodeState => + entry.1 == node) findEq) + rw [←keyEq] + apply valid entry (List.mem_of_find?_eq_some findEq) + simpa [stateEq] using joining + +theorem openerWitness_of_later_phase + {config : Config} + {state : State} + {node : Location} + (reachable : Reachable config state) + (active : node ∈ state.active) + (notGossip : Not (HasPhase state node .gossiping)) + (notVoting : Not (HasPhase state node .voting)) : + OpenerWitness state := by + rcases active_nodeState (reachable_well_formed reachable) active with + ⟨nodeState, found⟩ + cases phase : nodeState.phase with + | gossiping => + exact False.elim + (notGossip ⟨nodeState, found, phase⟩) + | voting => + exact False.elim + (notVoting ⟨nodeState, found, phase⟩) + | opening => + exact ⟨node, Or.inl ⟨nodeState, found, phase⟩⟩ + | joining => + rcases joining_node_announcement + (reachable_joining_announcements reachable) + found phase with + ⟨envelope, sent, target, payload⟩ + rcases reachable_announcements_live reachable + envelope sent payload with + opening | completed + · exact ⟨envelope.source, Or.inl opening⟩ + · exact ⟨envelope.source, Or.inr completed⟩ + | «open» => + exact + ⟨node, Or.inr + (open_node_completed + (reachable_open_completed reachable) found phase)⟩ + +theorem openerWitness_after_leave_voting + {config : Config} + (execution : Execution config) + (initial : Reachable config (execution.states 0)) + {start later : Nat} + {node : Location} + (order : start <= later) + (allPastGossip : + forall activeNode, + activeNode ∈ (execution.states start).active -> + Not (HasPhase (execution.states start) + activeNode .gossiping)) + (active : node ∈ (execution.states later).active) + (notVoting : + Not (HasPhase (execution.states later) node .voting)) : + OpenerWitness (execution.states later) := by + have activeStart : node ∈ (execution.states start).active := by + rw [execution_active_eq execution later] at active + rw [execution_active_eq execution start] + exact active + have notGossip := + not_gossiping_mono execution initial order + (allPastGossip node activeStart) + exact openerWitness_of_later_phase + (execution_reachable execution initial later) + active notGossip notVoting + +theorem systemStep_preserves_advanced_active + {config : Protocol.Config} + {before after : SystemState} + {target : Location} + {event : Event} + {output : StepOutput} + {active : List Location} + (valid : + forall entry, entry ∈ before.nodes -> + entry.2.phase ≠ .gossiping -> + entry.1 ∈ active) + (targetActive : target ∈ active) + (transition : + systemStep config before target event = some (after, output)) : + forall entry, entry ∈ after.nodes -> + entry.2.phase ≠ .gossiping -> + entry.1 ∈ active := by + simp [systemStep, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨node, ⟨key, found⟩, systemEq, outputEq⟩ + rw [←systemEq] + intro entry membership advanced + rw [replaceNode, List.mem_map] at membership + rcases membership with ⟨previous, previousMember, rfl⟩ + split + · exact targetActive + · rename_i notTarget + exact valid previous previousMember + (by simpa [notTarget] using advanced) + +theorem initial_advanced_active + (config : Config) + (active : List Location) : + AdvancedNodesActive (initial config active) := by + simp [AdvancedNodesActive, Global.initial, initialSystem, initialNode] + +theorem next_preserves_advanced_active + {config : Config} + {before after : State} + {action : Action} + (valid : AdvancedNodesActive before) + (transition : next config before action = some after) : + AdvancedNodesActive after := by + cases action with + | retry source => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, sourceState, _, _, stateEq⟩ + rw [←stateEq] + exact valid + | deliver envelope => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨_, targetActive, system, output, systemStep, stateEq⟩ + rw [←stateEq] + intro entry membership advanced + rw [recordEffects_system] at membership + simpa using + systemStep_preserves_advanced_active + valid targetActive systemStep entry membership advanced + | timeout target => + simp [next, Option.bind_eq_some_iff] at transition + rcases transition with + ⟨targetActive, system, output, systemStep, _, stateEq⟩ + rw [←stateEq] + intro entry membership advanced + rw [recordEffects_system] at membership + simpa using + systemStep_preserves_advanced_active + valid targetActive systemStep entry membership advanced + +theorem reachable_advanced_active + {config : Config} + {state : State} + (reachable : Reachable config state) : + AdvancedNodesActive state := by + induction reachable with + | initial active valid nodup configured => + exact initial_advanced_active config active + | step reachable transition valid => + exact next_preserves_advanced_active valid transition + +theorem hasPhase_active + {config : Config} + {state : State} + {node : Location} + {phase : Phase} + (reachable : Reachable config state) + (hasPhase : HasPhase state node phase) + (advancedPhase : phase ≠ .gossiping) : + node ∈ state.active := by + rcases hasPhase with ⟨nodeState, found, phaseEq⟩ + rw [Global.nodeState, Option.map_eq_some_iff] at found + rcases found with ⟨entry, findEq, stateEq⟩ + have keyEq : entry.1 = node := + beq_iff_eq.mp + (List.find?_some + (p := fun entry : Prod Location NodeState => + entry.1 == node) findEq) + rw [←keyEq] + apply reachable_advanced_active reachable entry + (List.mem_of_find?_eq_some findEq) + rw [stateEq, phaseEq] + exact advancedPhase + +theorem fair_opener_witness + {config : Config} + (execution : Execution config) + (initial : Reachable config (execution.states 0)) + (fair : Fair execution) + {start : Nat} + (activeNonempty : (execution.states start).active ≠ []) + (allPastGossip : + forall node, node ∈ (execution.states start).active -> + Not (HasPhase (execution.states start) node .gossiping)) : + EventuallyFrom start (fun n => + OpenerWitness (execution.states n)) := by + obtain ⟨voter, voterActive⟩ := + List.exists_mem_of_ne_nil _ activeNonempty + by_cases voting : + HasPhase (execution.states start) voter .voting + · rcases voting with ⟨voterState, foundVoter, voterVoting⟩ + have selectionProperty := + node_property_of_nodeState + (predicate := fun state => + state.phase = .voting -> NodeVotingSelection state) + (reachable_quorum_invariant + (execution_reachable execution initial start)).votingSelections + foundVoter + rcases selectionProperty voterVoting with + ⟨target, txid, chosen, maximum⟩ + have retryEnabled := + retry_voting_enabled (config := config) + voterActive foundVoter voterVoting chosen + rcases fair.retry start voter .voting voterActive + ⟨voterState, foundVoter, voterVoting⟩ + (Or.inr (Or.inl rfl)) retryEnabled with + ⟨retryAt, startRetry, leftVoting | retryAction⟩ + · exact + ⟨retryAt, startRetry, + openerWitness_after_leave_voting execution initial + startRetry allPastGossip + (by + rw [execution_active_eq execution retryAt] + rw [execution_active_eq execution start] at voterActive + exact voterActive) + leftVoting⟩ + · by_cases retryVoting : + HasPhase (execution.states retryAt) voter .voting + · rcases retryVoting with + ⟨retryState, foundRetry, votingRetry⟩ + have retrySelectionProperty := + node_property_of_nodeState + (predicate := fun state => + state.phase = .voting -> NodeVotingSelection state) + (reachable_quorum_invariant + (execution_reachable execution initial retryAt)).votingSelections + foundRetry + rcases retrySelectionProperty votingRetry with + ⟨retryTarget, retryTxID, retryChosen, retryMaximum⟩ + have retryStep : + next config (execution.states retryAt) (.retry voter) = + some (execution.states (retryAt + 1)) := by + simpa [retryAction] using execution.step_succ retryAt + rcases retry_vote_enqueued foundRetry votingRetry retryChosen + retryStep with + ⟨voteEnvelope, pending, voteSource, voteTarget, votePayload⟩ + rcases fair.delivery (retryAt + 1) voteEnvelope pending with + ⟨deliverAt, retryDeliver, deliverAction⟩ + have deliverStep : + next config (execution.states deliverAt) + (.deliver voteEnvelope) = + some (execution.states (deliverAt + 1)) := by + simpa [deliverAction] using execution.step_succ deliverAt + have deliverDetails := deliverStep + simp [next, Option.bind_eq_some_iff] at deliverDetails + have targetActive : voteEnvelope.target ∈ + (execution.states deliverAt).active := + deliverDetails.2.1 + by_cases targetVoting : + HasPhase (execution.states deliverAt) + voteEnvelope.target .voting + · rcases deliver_vote_progress + (reachable_well_formed + (execution_reachable execution initial deliverAt)) + votePayload targetVoting deliverStep with + leftAfter | hasVote + · have activeAfter : voteEnvelope.target ∈ + (execution.states (deliverAt + 1)).active := by + rw [execution_active_eq execution (deliverAt + 1)] + rw [execution_active_eq execution deliverAt] at targetActive + exact targetActive + exact + ⟨deliverAt + 1, by omega, + openerWitness_after_leave_voting execution initial + (by omega) allPastGossip activeAfter leftAfter⟩ + · by_cases votingAfter : + HasPhase (execution.states (deliverAt + 1)) + voteEnvelope.target .voting + · have activeAfter : voteEnvelope.target ∈ + (execution.states (deliverAt + 1)).active := by + rw [execution_active_eq execution (deliverAt + 1)] + rw [execution_active_eq execution deliverAt] at targetActive + exact targetActive + have timeoutEnabled := + voting_timeout_enabled (config := config) + activeAfter votingAfter + rcases fair.timeout (deliverAt + 1) + voteEnvelope.target .voting activeAfter votingAfter + (Or.inr (Or.inl rfl)) timeoutEnabled with + ⟨timeoutAt, deliverTimeout, + leftBeforeTimeout | timeoutAction⟩ + · exact + ⟨timeoutAt, by omega, + openerWitness_after_leave_voting execution initial + (by omega) allPastGossip + (by + rw [execution_active_eq execution timeoutAt] + rw [execution_active_eq execution + (deliverAt + 1)] at activeAfter + exact activeAfter) + leftBeforeTimeout⟩ + · by_cases votingAtTimeout : + HasPhase (execution.states timeoutAt) + voteEnvelope.target .voting + · have voteAtTimeout := + hasVote_mono execution initial deliverTimeout hasVote + have timeoutStep : + next config (execution.states timeoutAt) + (.timeout voteEnvelope.target) = + some (execution.states (timeoutAt + 1)) := by + simpa [timeoutAction] using + execution.step_succ timeoutAt + rcases timeout_voting_step + (reachable_well_formed + (execution_reachable execution initial timeoutAt)) + (reachable_lanes_valid + (execution_reachable execution initial timeoutAt)) + votingAtTimeout voteAtTimeout timeoutStep with + opened | + ⟨waitingState, foundWaiting, waitingPhase, + waitingLane, waitingVotes⟩ + · exact + ⟨timeoutAt + 1, by omega, + ⟨voteEnvelope.target, Or.inl opened⟩⟩ + · have activeWaiting : voteEnvelope.target ∈ + (execution.states (timeoutAt + 1)).active := by + rw [execution_active_eq execution (timeoutAt + 1)] + rw [execution_active_eq execution + (deliverAt + 1)] at activeAfter + exact activeAfter + have secondEnabled := + voting_timeout_enabled (config := config) + activeWaiting + ⟨waitingState, foundWaiting, waitingPhase⟩ + rcases fair.timeout (timeoutAt + 1) + voteEnvelope.target .voting activeWaiting + ⟨waitingState, foundWaiting, waitingPhase⟩ + (Or.inr (Or.inl rfl)) secondEnabled with + ⟨secondAt, firstSecond, + leftBeforeSecond | secondAction⟩ + · exact + ⟨secondAt, by omega, + openerWitness_after_leave_voting execution initial + (by omega) allPastGossip + (by + rw [execution_active_eq execution secondAt] + rw [execution_active_eq execution + (timeoutAt + 1)] at activeWaiting + exact activeWaiting) + leftBeforeSecond⟩ + · by_cases votingAtSecond : + HasPhase (execution.states secondAt) + voteEnvelope.target .voting + · rcases votingAtSecond with + ⟨secondState, foundSecond, secondPhase⟩ + have votesSecond := + hasVote_mono execution initial firstSecond + ⟨waitingState, foundWaiting, waitingVotes⟩ + rcases votesSecond with + ⟨voteState, foundVotes, secondVotes⟩ + rw [foundSecond] at foundVotes + injection foundVotes with voteStateEq + subst voteState + have advancedSecond := + advanced_lane_mono execution initial firstSecond + ⟨waitingState, foundWaiting, by simp [waitingLane]⟩ + rcases advancedSecond with + ⟨laneState, foundLane, advanced⟩ + rw [foundSecond] at foundLane + injection foundLane with laneStateEq + subst laneState + have laneValid : LaneValid secondState := by + apply node_property_of_nodeState + (predicate := LaneValid) + · exact reachable_lanes_valid + (execution_reachable execution initial secondAt) + · exact foundSecond + have secondLane : secondState.timeoutState = + .voting := by + rcases laneValid.2.1 secondPhase with + gossipLane | votingLane + · contradiction + · exact votingLane + have secondStep : + next config (execution.states secondAt) + (.timeout voteEnvelope.target) = + some (execution.states (secondAt + 1)) := by + simpa [secondAction] using + execution.step_succ secondAt + have opened := + aligned_timeout_voting_opens + (reachable_well_formed + (execution_reachable execution initial secondAt)) + foundSecond secondPhase secondLane secondVotes + secondStep + exact + ⟨secondAt + 1, by omega, + ⟨voteEnvelope.target, Or.inl opened⟩⟩ + · exact + ⟨secondAt, by omega, + openerWitness_after_leave_voting execution initial + (by omega) allPastGossip + (by + rw [execution_active_eq execution secondAt] + rw [execution_active_eq execution + (timeoutAt + 1)] at activeWaiting + exact activeWaiting) + votingAtSecond⟩ + · exact + ⟨timeoutAt, by omega, + openerWitness_after_leave_voting execution initial + (by omega) allPastGossip + (by + rw [execution_active_eq execution timeoutAt] + rw [execution_active_eq execution + (deliverAt + 1)] at activeAfter + exact activeAfter) + votingAtTimeout⟩ + · exact + ⟨deliverAt + 1, by omega, + openerWitness_after_leave_voting execution initial + (by omega) allPastGossip + (by + rw [execution_active_eq execution (deliverAt + 1)] + rw [execution_active_eq execution deliverAt] + at targetActive + exact targetActive) + votingAfter⟩ + · exact + ⟨deliverAt, by omega, + openerWitness_after_leave_voting execution initial + (by omega) allPastGossip targetActive targetVoting⟩ + · exact + ⟨retryAt, startRetry, + openerWitness_after_leave_voting execution initial + startRetry allPastGossip + (by + rw [execution_active_eq execution retryAt] + rw [execution_active_eq execution start] at voterActive + exact voterActive) + retryVoting⟩ + · exact + ⟨start, Nat.le_refl start, + openerWitness_after_leave_voting execution initial + (Nat.le_refl start) allPastGossip voterActive voting⟩ + +theorem openerWitness_eventually_completes + {config : Config} + (execution : Execution config) + (initial : Reachable config (execution.states 0)) + (fair : Fair execution) + {start : Nat} + (witness : OpenerWitness (execution.states start)) : + EventuallyFrom start (fun n => + exists node, CompletedOpen (execution.states n) node) := by + rcases witness with ⟨node, opening | completed⟩ + · have active := + hasPhase_active (execution_reachable execution initial start) + opening (by simp) + rcases fair_opening_completes execution initial fair active opening with + ⟨completedAt, order, completed⟩ + exact ⟨completedAt, order, node, completed⟩ + · exact ⟨start, Nat.le_refl start, node, completed⟩ + +theorem fair_some_opener_completes + {config : Config} + (execution : Execution config) + (initial : Reachable config (execution.states 0)) + (fair : Fair execution) + (activeNonempty : (execution.states 0).active ≠ []) : + EventuallyFrom 0 (fun n => + exists node, CompletedOpen (execution.states n) node) := by + rcases fair_all_leave_gossip execution initial fair 0 with + ⟨pastGossipAt, _, allPastGossip⟩ + have nonemptyAt : + (execution.states pastGossipAt).active ≠ [] := by + rw [execution_active_eq execution pastGossipAt] + exact activeNonempty + have allPastAt : + forall node, node ∈ (execution.states pastGossipAt).active -> + Not (HasPhase (execution.states pastGossipAt) + node .gossiping) := by + intro node active + rw [execution_active_eq execution pastGossipAt] at active + exact allPastGossip node active + rcases fair_opener_witness execution initial fair nonemptyAt + allPastAt with + ⟨witnessAt, pastWitness, witness⟩ + rcases openerWitness_eventually_completes execution initial fair + witness with + ⟨completedAt, witnessCompleted, completed⟩ + exact ⟨completedAt, by omega, completed⟩ + +theorem fair_target_terminal_after_completion + {config : Config} + (execution : Execution config) + (initial : Reachable config (execution.states 0)) + (fair : Fair execution) + (broadcast : BroadcastBeforeCompletion execution) + {start : Nat} + {opener target : Location} + (completed : CompletedOpen (execution.states start) opener) + (active : target ∈ (execution.states start).active) : + EventuallyFrom start (fun n => + Terminal (execution.states n) target) := by + by_cases same : target = opener + · subst target + exact ⟨start, Nat.le_refl start, Or.inr completed⟩ + · rcases broadcast start opener completed target active same with + ⟨envelope, sent, sourceEq, targetEq, payload⟩ + rcases reachable_announcements_resolved + (execution_reachable execution initial start) + envelope sent payload with + pending | terminal | opening + · rcases fair.delivery start envelope pending with + ⟨deliverAt, startDelivery, deliverAction⟩ + have deliverStep : + next config (execution.states deliverAt) (.deliver envelope) = + some (execution.states (deliverAt + 1)) := by + simpa [deliverAction] using execution.step_succ deliverAt + rcases deliver_iamopen_resolves + (reachable_well_formed + (execution_reachable execution initial deliverAt)) + (reachable_open_completed + (execution_reachable execution initial deliverAt)) + payload deliverStep with + terminal | targetOpening + · exact + ⟨deliverAt + 1, by omega, by simpa [targetEq] using terminal⟩ + · have openingActive := + hasPhase_active + (execution_reachable execution initial (deliverAt + 1)) + targetOpening (by simp) + rcases fair_opening_completes execution initial fair + openingActive targetOpening with + ⟨completedAt, deliveryCompleted, targetCompleted⟩ + exact + ⟨completedAt, by omega, + by simpa [targetEq] using (Or.inr targetCompleted)⟩ + · exact + ⟨start, Nat.le_refl start, by simpa [targetEq] using terminal⟩ + · have openingActive := + hasPhase_active + (execution_reachable execution initial start) + opening (by simp) + rcases fair_opening_completes execution initial fair + openingActive opening with + ⟨completedAt, startCompleted, targetCompleted⟩ + exact + ⟨completedAt, startCompleted, + by simpa [targetEq] using (Or.inr targetCompleted)⟩ + +theorem fair_all_terminal_after_completion + {config : Config} + (execution : Execution config) + (initial : Reachable config (execution.states 0)) + (fair : Fair execution) + (broadcast : BroadcastBeforeCompletion execution) + {start : Nat} + {opener : Location} + (completed : CompletedOpen (execution.states start) opener) : + EventuallyFrom start (fun n => + forall node, node ∈ (execution.states start).active -> + Terminal (execution.states n) node) := by + apply eventually_list (execution.states start).active + · intro node active + exact fair_target_terminal_after_completion + execution initial fair broadcast completed active + · intro node first second order terminal + exact terminal_mono execution order terminal + +theorem global_progress + {config : Config} + (execution : Execution config) + (initial : Reachable config (execution.states 0)) + (fair : Fair execution) + (broadcast : BroadcastBeforeCompletion execution) + (activeNonempty : (execution.states 0).active ≠ []) : + EventuallyFrom 0 (fun n => + exists node, CompletedOpen (execution.states n) node) /\ + EventuallyFrom 0 (fun n => + forall node, node ∈ (execution.states 0).active -> + Terminal (execution.states n) node) := by + have completed := + fair_some_opener_completes execution initial fair activeNonempty + constructor + · exact completed + · rcases completed with + ⟨completedAt, _, opener, openerCompleted⟩ + rcases fair_all_terminal_after_completion execution initial fair + broadcast openerCompleted with + ⟨terminalAt, completedTerminal, allTerminal⟩ + refine ⟨terminalAt, by omega, ?_⟩ + intro node active + apply allTerminal node + rw [execution_active_eq execution completedAt] + exact active + +theorem single_completion_path_joins_others + {config : Config} + (execution : Execution config) + (initial : Reachable config (execution.states 0)) + (fair : Fair execution) + (broadcast : BroadcastBeforeCompletion execution) + {start : Nat} + {opener : Location} + (completed : CompletedOpen (execution.states start) opener) + (onlyOpener : OnlyOpenerCompletesFrom execution start opener) : + EventuallyFrom start (fun n => + forall node, node ∈ (execution.states start).active -> + node = opener \/ node ∈ (execution.states n).restarts) := by + apply eventually_list (execution.states start).active + · intro node active + by_cases same : node = opener + · exact ⟨start, Nat.le_refl start, Or.inl same⟩ + · rcases fair_target_terminal_after_completion + execution initial fair broadcast completed active with + ⟨terminalAt, startTerminal, terminal⟩ + rcases terminal with restarted | targetCompleted + · exact ⟨terminalAt, startTerminal, Or.inr restarted⟩ + · exact False.elim + (same + (onlyOpener terminalAt node startTerminal targetCompleted)) + · intro node first second order joined + rcases joined with same | restarted + · exact Or.inl same + · exact Or.inr + (by + induction second, order using Nat.le_induction with + | base => exact restarted + | succ second order restarted => + exact next_restarts_monotonic + (execution.step_succ second) node restarted) + +theorem quorum_path_progress + {config : Config} + (execution : Execution config) + (initial : Reachable config (execution.states 0)) + (fair : Fair execution) + (broadcast : BroadcastBeforeCompletion execution) + {start : Nat} + {opener : Location} + (opened : QuorumOpened (execution.states start) opener) + (completed : CompletedOpen (execution.states start) opener) + (quorumOnly : QuorumOnlyCompletions execution) : + QuorumOpened (execution.states start) opener /\ + CompletedOpen (execution.states start) opener /\ + EventuallyFrom start (fun n => + forall node, node ∈ (execution.states start).active -> + node = opener \/ node ∈ (execution.states n).restarts) := by + have onlyOpener : + OnlyOpenerCompletesFrom execution start opener := by + intro n node startN nodeCompleted + exact quorum_opener_unique + (execution_reachable execution initial n) + (quorumOnly n node nodeCompleted) + (quorumOpened_mono execution startN opened) + exact + ⟨opened, completed, + single_completion_path_joins_others + execution initial fair broadcast completed onlyOpener⟩ + +end DisasterRecovery.Protocol.Global diff --git a/lean/disaster-recovery/README.md b/lean/disaster-recovery/README.md index 4021c189024d..54d9ab746580 100644 --- a/lean/disaster-recovery/README.md +++ b/lean/disaster-recovery/README.md @@ -27,12 +27,12 @@ effects around that same canonical transition function. The versioned trace validator also replays committed C++ instrumentation against the canonical model. -The migration is approximately 7,000 lines across 27 new files: +The migration is approximately 10,000 lines across 28 new files: | Area | Files | Approximate lines | Purpose | | ----------------------------------------- | ----: | ----------------: | --------------------------------------------------------------- | | Exact legacy Lean model | 4 | 650 | Stateright state, actions, timers, network, predicates, and BFS | -| Canonical protocol and proofs | 8 | 3,600 | C++ behavior, global semantics, invariants, and temporal proofs | +| Canonical protocol and proofs | 9 | 6,800 | C++ behavior, global semantics, invariants, and temporal proofs | | Trace validation | 5 | 760 | NDJSON format, deterministic replay, and CLI | | Equivalence tooling | 2 | 710 | Rust graph exporter and bidirectional Rust/Lean comparison | | Project, documentation, and configuration | 6 | 530 | Lake/Mathlib setup, entry points, and documentation | @@ -40,22 +40,23 @@ The migration is approximately 7,000 lines across 27 new files: ### Principal files -| File | Lines | Role | -| -------------------------------------------------------------------------------------------- | ----: | ------------------------------------------------------------------ | -| [`DisasterRecovery/Model.lean`](DisasterRecovery/Model.lean) | 392 | Exact executable legacy semantics | -| [`DisasterRecovery/Checker.lean`](DisasterRecovery/Checker.lean) | 152 | BFS enumeration, property checking, and canonical graph export | -| [`DisasterRecovery/Protocol/Model.lean`](DisasterRecovery/Protocol/Model.lean) | 290 | Production-oriented C++ protocol model | -| [`DisasterRecovery/Protocol/Refinement.lean`](DisasterRecovery/Protocol/Refinement.lean) | 332 | Canonical-to-legacy formal phase refinement | -| [`DisasterRecovery/Protocol/Temporal.lean`](DisasterRecovery/Protocol/Temporal.lean) | 230 | Execution streams, fairness, safety, and progress proofs | -| [`DisasterRecovery/Protocol/Global.lean`](DisasterRecovery/Protocol/Global.lean) | 166 | Global active-node, network, send-history, and effect semantics | -| [`DisasterRecovery/Protocol/Invariants.lean`](DisasterRecovery/Protocol/Invariants.lean) | 899 | Global provenance, locality, monotonicity, and reachability proofs | -| [`DisasterRecovery/Protocol/Quorum.lean`](DisasterRecovery/Protocol/Quorum.lean) | 1,467 | Vote-history invariants and unbounded quorum-opener uniqueness | -| [`DisasterRecovery/Protocol/Committed.lean`](DisasterRecovery/Protocol/Committed.lean) | 258 | TxID maximum and committed-prefix preservation proofs | -| [`DisasterRecovery/Protocol/Trace/Format.lean`](DisasterRecovery/Protocol/Trace/Format.lean) | 141 | Versioned NDJSON types and parser | -| [`DisasterRecovery/Protocol/Trace/Replay.lean`](DisasterRecovery/Protocol/Trace/Replay.lean) | 452 | Deterministic implementation-trace replay | -| [`compare.py`](compare.py) | 343 | Exhaustive canonical graph comparison | -| [`../../tla/disaster-recovery/src/export.rs`](../../tla/disaster-recovery/src/export.rs) | 370 | Canonical export of the actual Stateright model | -| [`TRACE_FORMAT_V1.md`](TRACE_FORMAT_V1.md) | 145 | Contract for future committed C++ trace events | +| File | Lines | Role | +| ------------------------------------------------------------------------------------------------ | ----: | ------------------------------------------------------------------ | +| [`DisasterRecovery/Model.lean`](DisasterRecovery/Model.lean) | 392 | Exact executable legacy semantics | +| [`DisasterRecovery/Checker.lean`](DisasterRecovery/Checker.lean) | 152 | BFS enumeration, property checking, and canonical graph export | +| [`DisasterRecovery/Protocol/Model.lean`](DisasterRecovery/Protocol/Model.lean) | 290 | Production-oriented C++ protocol model | +| [`DisasterRecovery/Protocol/Refinement.lean`](DisasterRecovery/Protocol/Refinement.lean) | 332 | Canonical-to-legacy formal phase refinement | +| [`DisasterRecovery/Protocol/Temporal.lean`](DisasterRecovery/Protocol/Temporal.lean) | 230 | Execution streams, fairness, safety, and progress proofs | +| [`DisasterRecovery/Protocol/Global.lean`](DisasterRecovery/Protocol/Global.lean) | 166 | Global active-node, network, send-history, and effect semantics | +| [`DisasterRecovery/Protocol/Invariants.lean`](DisasterRecovery/Protocol/Invariants.lean) | 899 | Global provenance, locality, monotonicity, and reachability proofs | +| [`DisasterRecovery/Protocol/Quorum.lean`](DisasterRecovery/Protocol/Quorum.lean) | 1,467 | Vote-history invariants and unbounded quorum-opener uniqueness | +| [`DisasterRecovery/Protocol/Committed.lean`](DisasterRecovery/Protocol/Committed.lean) | 258 | TxID maximum and committed-prefix preservation proofs | +| [`DisasterRecovery/Protocol/GlobalTemporal.lean`](DisasterRecovery/Protocol/GlobalTemporal.lean) | 3,168 | Global fairness, phase progress, and termination proofs | +| [`DisasterRecovery/Protocol/Trace/Format.lean`](DisasterRecovery/Protocol/Trace/Format.lean) | 141 | Versioned NDJSON types and parser | +| [`DisasterRecovery/Protocol/Trace/Replay.lean`](DisasterRecovery/Protocol/Trace/Replay.lean) | 452 | Deterministic implementation-trace replay | +| [`compare.py`](compare.py) | 343 | Exhaustive canonical graph comparison | +| [`../../tla/disaster-recovery/src/export.rs`](../../tla/disaster-recovery/src/export.rs) | 370 | Canonical export of the actual Stateright model | +| [`TRACE_FORMAT_V1.md`](TRACE_FORMAT_V1.md) | 145 | Contract for future committed C++ trace events | The migration also updates the existing Stateright CLI and documentation, adds weekly exhaustive verification, and adds @@ -194,7 +195,37 @@ explicit: Quorum opening alone does **not** imply `FullGossipSelection`: voting may follow a gossip timeout. The committed-prefix theorem deliberately does not hide or -derive that premise. Fair global termination remains the next unbounded proof. +derive that premise. + +`DisasterRecovery.Protocol.GlobalTemporal` defines infinite executions over +`Global.next` and action-oriented fairness for enabled retries, reliable +delivery, and enabled timeouts. It proves: + +- every active node eventually leaves Gossiping; +- some vote target eventually reaches Opening; +- every Opening node eventually completes, using a well-founded timeout-lane + measure; +- `fair_some_opener_completes`: a nonempty active execution eventually has a + completed opener; and +- `global_progress`: eventually some opener completes and eventually every + active node is terminal. + +The all-node half of `global_progress` has a separate +`BroadcastBeforeCompletion` premise: before an opener completes, its +`IAmOpen` messages must have been sent to every other active participant. This +is deliberately not called weak fairness. Ordinary weak fairness does not +order two actions that are enabled only for a finite interval, so it cannot +guarantee a retry before the Opening timeout completes. The proof derives +follower termination from those actual sent messages, the +`AnnouncementsResolved` reachable invariant, reliable delivery, and replay of +the `IAmOpen` transition. + +`quorum_path_progress` adds `QuorumOnlyCompletions` (no failover completion), +uses `quorum_opener_unique` to derive the unique completed opener, and proves +that every other active participant restarts/joins. + +These are conditional temporal theorems. They do not construct a concrete +scheduler satisfying the fairness and broadcast-ordering premises. ### Kernel-checked refinement properties @@ -376,7 +407,9 @@ phase to Open. `fair_aligned_opening_progress` proves that an execution whose initial state is `AlignedOpening` eventually reaches phase Open, assuming weak fairness for timeout firing while `AlignedOpening` is enabled. This is not a global termination theorem; reaching aligned Opening still requires separate -message-delivery and timeout progress assumptions. +message-delivery and timeout progress assumptions. The separate +`Protocol/GlobalTemporal.lean` theorem supplies the distributed result under +the stronger, explicit assumptions described above. ## Legacy compatibility From a4824c544b43beada2095db9933679dbd3f8ef04 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Tue, 1 Sep 2026 06:49:25 +0100 Subject: [PATCH 13/14] Reject Lean sorryAx in CI Compile every project target with warnings as errors and scan every DisasterRecovery declaration transitively for sorryAx. Run the scan explicitly in Lean shallow CI. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 46d56f8a-935f-4551-abe3-84bd4f951865 --- .github/workflows/lean-shallow.yml | 1 + lean/disaster-recovery/AxiomChecks.lean | 21 +++++++++++++++++++++ lean/disaster-recovery/README.md | 16 ++++++++++------ lean/disaster-recovery/lakefile.toml | 1 + 4 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 lean/disaster-recovery/AxiomChecks.lean diff --git a/.github/workflows/lean-shallow.yml b/.github/workflows/lean-shallow.yml index 0b6befd27adb..58ba38e8abef 100644 --- a/.github/workflows/lean-shallow.yml +++ b/.github/workflows/lean-shallow.yml @@ -52,6 +52,7 @@ jobs: run: | set -euo pipefail lake build + lake env lean -DwarningAsError=true AxiomChecks.lean lake exe semantic-checks lake exe canonical-checks lake exe disaster-recovery check --nodes 3 diff --git a/lean/disaster-recovery/AxiomChecks.lean b/lean/disaster-recovery/AxiomChecks.lean new file mode 100644 index 000000000000..a962639129b0 --- /dev/null +++ b/lean/disaster-recovery/AxiomChecks.lean @@ -0,0 +1,21 @@ +import DisasterRecovery +import Lean.Elab.Command +import Lean.Util.CollectAxioms + +open Lean Elab Command + +elab "#assert_no_project_sorries" : command => do + let env <- getEnv + let mut offenders : Array Name := #[] + for (name, _) in env.constants.toList do + if name.toString.startsWith "DisasterRecovery" then + let axioms <- liftCoreM <| Lean.collectAxioms name + if axioms.contains (Name.mkSimple "sorryAx") then + offenders := offenders.push name + unless offenders.isEmpty do + throwError "declarations contain sorryAx: {offenders}" + +#assert_no_project_sorries + +def main : IO Unit := + pure () diff --git a/lean/disaster-recovery/README.md b/lean/disaster-recovery/README.md index 54d9ab746580..f65790c923b1 100644 --- a/lean/disaster-recovery/README.md +++ b/lean/disaster-recovery/README.md @@ -27,7 +27,7 @@ effects around that same canonical transition function. The versioned trace validator also replays committed C++ instrumentation against the canonical model. -The migration is approximately 10,000 lines across 28 new files: +The migration is approximately 10,000 lines across 29 new files: | Area | Files | Approximate lines | Purpose | | ----------------------------------------- | ----: | ----------------: | --------------------------------------------------------------- | @@ -35,7 +35,7 @@ The migration is approximately 10,000 lines across 28 new files: | Canonical protocol and proofs | 9 | 6,800 | C++ behavior, global semantics, invariants, and temporal proofs | | Trace validation | 5 | 760 | NDJSON format, deterministic replay, and CLI | | Equivalence tooling | 2 | 710 | Rust graph exporter and bidirectional Rust/Lean comparison | -| Project, documentation, and configuration | 6 | 530 | Lake/Mathlib setup, entry points, and documentation | +| Project, documentation, and configuration | 7 | 550 | Lake/Mathlib setup, entry points, and documentation | | Pull-request CI | 1 | 80 | Lean model and bounded equivalence checks | ### Principal files @@ -54,6 +54,7 @@ The migration is approximately 10,000 lines across 28 new files: | [`DisasterRecovery/Protocol/GlobalTemporal.lean`](DisasterRecovery/Protocol/GlobalTemporal.lean) | 3,168 | Global fairness, phase progress, and termination proofs | | [`DisasterRecovery/Protocol/Trace/Format.lean`](DisasterRecovery/Protocol/Trace/Format.lean) | 141 | Versioned NDJSON types and parser | | [`DisasterRecovery/Protocol/Trace/Replay.lean`](DisasterRecovery/Protocol/Trace/Replay.lean) | 452 | Deterministic implementation-trace replay | +| [`AxiomChecks.lean`](AxiomChecks.lean) | 21 | Project-wide transitive `sorryAx` rejection | | [`compare.py`](compare.py) | 343 | Exhaustive canonical graph comparison | | [`../../tla/disaster-recovery/src/export.rs`](../../tla/disaster-recovery/src/export.rs) | 370 | Canonical export of the actual Stateright model | | [`TRACE_FORMAT_V1.md`](TRACE_FORMAT_V1.md) | 145 | Contract for future committed C++ trace events | @@ -500,6 +501,7 @@ From this directory: ```console lake build +lake env lean -DwarningAsError=true AxiomChecks.lean lake exe semantic-checks lake exe canonical-checks lake exe disaster-recovery check --nodes 3 @@ -536,11 +538,13 @@ The Rust comparison exporter is checked separately from `tla/disaster-recovery/` with `cargo check` and `cargo build`. `.github/workflows/lean-shallow.yml` builds the Lean library and trace validator, -runs semantic and canonical checks, checks the three-node legacy properties, -and compares the one- and two-node Rust/Lean graphs. The weekly continuous +runs the `sorryAx` scan, runs semantic and canonical checks, checks the +three-node legacy properties, and compares the one- and two-node Rust/Lean +graphs. Lake also compiles every project target with warnings as errors, so a +direct `sorry` warning fails before the transitive scan. The weekly continuous verification workflow additionally runs the exhaustive three-node comparison. -Only the SNP jobs feed real C++ traces to the validator. The Rust job remains in -place until the replacement criteria below are met. +Only the SNP jobs feed real C++ traces to the validator. The Rust job remains +in place until the replacement criteria below are met. ### Current CI evidence diff --git a/lean/disaster-recovery/lakefile.toml b/lean/disaster-recovery/lakefile.toml index aa5192985ce0..c6c673493db2 100644 --- a/lean/disaster-recovery/lakefile.toml +++ b/lean/disaster-recovery/lakefile.toml @@ -1,5 +1,6 @@ name = "disaster_recovery" version = "0.1.0" +moreLeanArgs = ["-DwarningAsError=true"] defaultTargets = [ "DisasterRecovery", "disaster-recovery", From 1877a195942fb3d825b349fa5ad095b0381dda3d Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Tue, 1 Sep 2026 10:10:56 +0100 Subject: [PATCH 14/14] Defer recovery restart until commit Request host restart from post-commit hooks instead of from the JOINING transaction. Preserve trace-event-before-restart ordering in trace-enabled builds. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 46d56f8a-935f-4551-abe3-84bd4f951865 --- CHANGELOG.md | 8 ++++++++ python/pyproject.toml | 2 +- src/node/recovery_decision_protocol.cpp | 22 ++++++++++++++++------ src/node/recovery_decision_protocol.h | 1 + 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 126fdff584c8..e0fc72c44e0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [7.0.14] + +[7.0.14]: https://github.com/microsoft/CCF/releases/tag/ccf-7.0.14 + +### Fixed + +- Recovery-decision-protocol nodes now request host restart only after the `JOINING` state transaction commits, preventing restart for an aborted transaction. (#8241) + ## [7.0.13] [7.0.13]: https://github.com/microsoft/CCF/releases/tag/ccf-7.0.13 diff --git a/python/pyproject.toml b/python/pyproject.toml index 18462482f034..7529d0383b9b 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "ccf" -version = "7.0.13" +version = "7.0.14" authors = [ { name="CCF Team", email="CCF-Sec@microsoft.com" }, ] diff --git a/src/node/recovery_decision_protocol.cpp b/src/node/recovery_decision_protocol.cpp index a2984ff1a27f..f693fad3d159 100644 --- a/src/node/recovery_decision_protocol.cpp +++ b/src/node/recovery_decision_protocol.cpp @@ -64,6 +64,11 @@ namespace ccf node_state(node_state_) {} + void RecoveryDecisionProtocolSubsystem::restart_after_commit() + { + RINGBUFFER_WRITE_MESSAGE(AdminMessage::restart, node_state->to_host); + } + #ifdef CCF_RECOVERY_TRACE void RecoveryDecisionProtocolSubsystem::initialise_trace(ccf::kv::Tx& tx) { @@ -106,8 +111,7 @@ namespace ccf emit_trace_event(event.value()); if (event->kind == "join_restart") { - RINGBUFFER_WRITE_MESSAGE( - AdminMessage::restart, node_state->to_host); + restart_after_commit(); } } } @@ -375,6 +379,16 @@ namespace ccf start_message_retry_timers(); start_failover_timers(); } + else if ( + w.has_value() && + w.value() == recovery_decision_protocol::StateMachine::JOINING) + { +#ifndef CCF_RECOVERY_TRACE + restart_after_commit(); +#else + // The trace-event commit hook emits join_restart before restarting. +#endif + } })); } @@ -530,10 +544,6 @@ namespace ccf auto service_cert = ccf::crypto::cert_der_to_pem(node_config->service_cert_der); LOG_INFO_FMT("{}", service_cert.str()); - -#ifndef CCF_RECOVERY_TRACE - RINGBUFFER_WRITE_MESSAGE(AdminMessage::restart, node_state->to_host); -#endif } case recovery_decision_protocol::StateMachine::OPENING: { diff --git a/src/node/recovery_decision_protocol.h b/src/node/recovery_decision_protocol.h index 1cac209ba929..cc9ec2cd6591 100644 --- a/src/node/recovery_decision_protocol.h +++ b/src/node/recovery_decision_protocol.h @@ -106,6 +106,7 @@ namespace ccf // Stop periodic tasks void stop_timers(); + void restart_after_commit(); // Steady state operations recovery_decision_protocol::RequestNodeInfo& get_node_info(