feat(dataset): admit four P2SH redeem-script failures with complete bodies

README.md

@@ -55,12 +55,12 @@ It checks JSONL structure, types, ordering, uniqueness, header hash, PoW and dec
 It enforces the rule/reject-string mapping, required context and rule-specific predicates.
 Validation reports the first error in each record, with its file and line number, then continues to the next record.
 Available block files must parse completely and match their transaction merkle roots and applicable witness commitments.
-CI checks output-value overflow, forward transaction spends, excessive sigop cost, transaction reuse from the canonical parent and coinbase overpayment directly.
+CI checks output-value overflow, forward transaction spends, excessive sigop cost, transaction reuse from the canonical parent, coinbase overpayment and P2SH redeem-script failure directly.
 
 For coinbase overpayment, CI compares the coinbase output sum with the subsidy at the record height plus fees calculated from authenticated previous output values.
 Coinbase-only bodies have zero fees and need no previous transactions.
 
-Sigops, missing-parent, parent-transaction reuse and fee accounting checks use a verified cache in `.cache/prevouts/`, restored between GitHub Actions runs.
+Sigops, missing-parent, parent-transaction reuse, fee accounting and P2SH checks use a verified cache in `.cache/prevouts/`, restored between GitHub Actions runs.
 Missing entries are fetched from public Esplora-compatible APIs when `--fetch-prevouts` is supplied.
 API failures, missing evidence and corrupt cache entries fail validation.
 After filling the cache, omit the flag for an offline run; `--prevouts-dir` selects another cache and `--api-url` selects an API base.

ci/block_evidence.py

@@ -1,7 +1,8 @@
 """Read committed transactions for narrow, offline evidence checks.
 
-This is not a consensus validator: no script execution, UTXO lookup or
-historical chain reconstruction takes place here.
+This is not a consensus validator: no UTXO lookup or historical chain
+reconstruction takes place here, and the only script execution is the
+library's evaluation of one named input for the P2SH rule.
 Transaction IDs and the merkle root bind the checked non-witness data to the
 Bitcoin header. Parsing consumes the entire file so truncation cannot satisfy
 a rule's requirement for a complete block body.
@@ -16,7 +17,8 @@
 from collections.abc import Mapping, Sequence
 from typing import TypeVar
 
-from bitcoin.core import COIN, CBlock, CoreMainParams, CTransaction, Hash as sha256d, MoneyRange, b2lx, lx
+from bitcoin.core import COIN, CBlock, CoreMainParams, CTransaction, Hash as sha256d, MoneyRange, ValidationError, b2lx, lx
+from bitcoin.core.scripteval import SCRIPT_VERIFY_P2SH, VerifyScript, VerifySignature
 from bitcoin.core.script import (
     CScript, CScriptInvalidError, CScriptOp, OP_1, OP_16,
     OP_CHECKSIG, OP_CHECKSIGVERIFY, OP_CHECKMULTISIG, OP_CHECKMULTISIGVERIFY,
@@ -115,6 +117,33 @@ def reuses_parent_transaction(block: CBlock, parent_txids: Sequence[str], txid:
             and any(not tx.is_coinbase() and tx.GetTxid() == target for tx in block.vtx[1:]))
 
 
+def spending_input(transactions: Sequence[CTransaction], txid: bytes, vout: int) -> tuple[CTransaction, int]:
+    """Return the one non-coinbase input among these transactions that spends the outpoint."""
+    spends = [(tx, index) for tx in transactions if not tx.is_coinbase()
+              for index, txin in enumerate(tx.vin) if txin.prevout.hash == txid and txin.prevout.n == vout]
+    if len(spends) != 1:
+        raise ValueError(f"outpoint {b2lx(txid)}:{vout} must be spent by exactly one input")
+    return spends[0]
+
+
+def p2sh_spend_fails(tx: CTransaction, index: int, previous: CTransaction) -> bool:
+    """Require the input to pass without P2SH and fail once the redeem script is executed.
+
+    VerifySignature binds the input to the previous transaction's output and
+    evaluates it with no flags; the caller authenticates that transaction by txid.
+    """
+    try:
+        VerifySignature(previous, tx, index)
+    except ValidationError as error:
+        raise ValueError(f"input fails even without P2SH evaluation: {error}") from error
+    script_pubkey = previous.vout[tx.vin[index].prevout.n].scriptPubKey
+    try:
+        VerifyScript(tx.vin[index].scriptSig, script_pubkey, tx, index, flags=(SCRIPT_VERIFY_P2SH,))
+    except ValidationError:
+        return True
+    return False
+
+
 def establishes_rule(block: CBlock, rule: str) -> bool:
     """Recognize only failures provable from these committed transactions.
 

ci/sanity-check.py

@@ -22,7 +22,8 @@
 from bitcoin.core.serialize import uint256_from_compact
 
 from block_evidence import (
-    MAX_BLOCK_SIGOPS_COST, coinbase_amounts, confirmed_at_or_after, establishes_rule, omitted_prevouts,
+    MAX_BLOCK_SIGOPS_COST, coinbase_amounts, confirmed_at_or_after, establishes_rule, omitted_prevouts, p2sh_spend_fails,
+    spending_input,
     read_block, reuses_parent_transaction, sigop_cost, verify_witness_commitment,
 )
 from prevouts import (
@@ -34,7 +35,7 @@
 REQUIRED = {"height", "hash", "header", "prev_hash", "nTime", "core_reject_reason", "rule"}
 CONTEXT_FIELDS = {
     "expected_nbits", "parent_mtp", "coinbase_height", "coinbase_scriptsig_hex",
-    "pool", "pool_basis", "parent_kind", "missing_prevout", "parent_txid",
+    "pool", "pool_basis", "parent_kind", "missing_prevout", "parent_txid", "failing_prevout",
 }
 OUTPOINT = re.compile(r"[0-9a-f]{64}:(?:0|[1-9][0-9]*)")
 OBSERVATION_REQUIRED = {"channel", "source", "provenance"}
@@ -44,13 +45,14 @@
 PARENT_KINDS = {"canonical", "stale", "invalid"}
 POOL_BASES = {"tag", "reported", "address"}
 POW_LIMIT = 0xFFFF << (8 * (0x1D - 3))
+BIP16_TIME = 1333238400  # 1 April 2012, when 2012 nodes began executing P2SH redeem scripts
 
 # Evidence paths: local = header/context only; body = complete block file;
 # sigops = body plus previous transactions; missing_parent = body plus API
 # evidence for the recorded outpoint; parent_txid_reuse = body plus an
 # authenticated canonical parent txid list; cb_amount = body plus canonical
-# parent and fee prevouts.
-# Rule names and reject strings must
+# parent and fee prevouts; p2sh = body plus the spent output of the named
+# input. Rule names and reject strings must
 # match docs/schema.md.
 RULES = {
     "bad-txns-vout-toolarge": ("bad-txns-vout-toolarge", (), "body"),
@@ -60,6 +62,7 @@
     "missing_unconfirmed_parent": ("bad-txns-inputs-missingorspent", ("missing_prevout",), "missing_parent"),
     "already_confirmed_in_parent": ("bad-txns-inputs-missingorspent",
         ("parent_txid", "parent_kind", "coinbase_height", "coinbase_scriptsig_hex"), "parent_txid_reuse"),
+    "p2sh_redeem_script_failure": ("block-script-verify-flag-failed", ("failing_prevout",), "p2sh"),
     "bip34_v2_coinbase_height_mismatch": (
         "bad-cb-height", ("coinbase_height", "coinbase_scriptsig_hex"), "local"),
     "bip34_coinbase_height_mismatch": (
@@ -178,9 +181,9 @@ def check_context(record: dict[str, Any]) -> None:
         raise ValueError("pool_basis requires pool")
     if "parent_txid" in details:
         hex_value(details, "parent_txid", 32)
-    if "missing_prevout" in details and not (
-            isinstance(details["missing_prevout"], str) and OUTPOINT.fullmatch(details["missing_prevout"])):
-        raise ValueError("missing_prevout must be txid:vout in lowercase hex")
+    for name in ("missing_prevout", "failing_prevout"):
+        if name in details and not (isinstance(details[name], str) and OUTPOINT.fullmatch(details[name])):
+            raise ValueError(f"{name} must be txid:vout in lowercase hex")
     if "parent_kind" in details and details["parent_kind"] not in tuple(PARENT_KINDS):
         raise ValueError(f"parent_kind must be one of {sorted(PARENT_KINDS)}")
     required = set(RULES[record["rule"]][1])
@@ -289,6 +292,8 @@ def check_local_evidence(record: dict[str, Any], header: CBlockHeader) -> None:
             raise ValueError("scriptSig has the correct BIP34 height prefix")
         if rule == "bip34_coinbase_height_missing" and script_height(script) is not None:
             raise ValueError("missing-height rule requires no decodable height prefix")
+    if rule == "p2sh_redeem_script_failure" and record["nTime"] < BIP16_TIME:
+        raise ValueError("P2SH rule requires nTime at or after BIP16 activation")
     if rule == "coinbase_scriptsig_length_above_100" and len(script) <= 100:
         raise ValueError("coinbase scriptSig must exceed 100 bytes")
 
@@ -329,6 +334,11 @@ def check_failure_evidence(record: dict[str, Any], block: CBlock | None, prevout
         if fetch_prevouts:
             print(f"{record['height']}: coinbase {amounts['coinbase']}, subsidy {amounts['subsidy']}, "
                   f"fees {amounts['fees']}, excess {amounts['excess']} sat", flush=True)
+    if mode == "p2sh":
+        txid, vout = record["context"]["failing_prevout"].split(":")
+        tx, index = spending_input(block.vtx, lx(txid), int(vout))
+        if not p2sh_spend_fails(tx, index, load_transaction(txid, prevouts_dir, fetch_prevouts, apis)):
+            raise ValueError("named input does not fail P2SH evaluation")
     if mode == "missing_parent":
         txid, vout = record["context"]["missing_prevout"].split(":")
         if (lx(txid), int(vout)) not in omitted_prevouts(block.vtx):

ci/test_block_evidence.py

@@ -6,10 +6,20 @@
 from bitcoin.core.script import CScript, CScriptWitness, OP_TRUE
 
 from block_evidence import (
-    MAX_MONEY, coinbase_amounts, confirmed_at_or_after, establishes_rule, omitted_prevouts, read_block, read_transaction,
-    reuses_parent_transaction, sha256d, sigop_count, witness_sigops,
+    MAX_MONEY, coinbase_amounts, confirmed_at_or_after, establishes_rule, omitted_prevouts, p2sh_spend_fails, read_block,
+    read_transaction, reuses_parent_transaction, sha256d, sigop_count, witness_sigops,
 )
 
+# The 123-byte spend included by 89 blocks in April to July 2012, and the transaction that funded it.
+P2SH_SPEND = bytes.fromhex(
+    "01000000019dc23528f5a5f376da3f3f4efd45be8c5b551abdb8093940e0b313de459a53b00100000026255121029c7187ecea7f09146820075c3a8d"
+    "e5d33ffbc293b63228ea1667c8d3796aff3f51aeffffffff0130570500000000001976a9147288ca9e213c54cbb2094f00bcf33bfbce691dbb88ac00000000")
+P2SH_FUNDING = bytes.fromhex(
+    "0100000001f6ea284ec7521f8a7d094a6cf4e6873098b90f90725ffd372b343189d7a4089c000000006c4930460221009d1055704950ab3b695c7215"
+    "0169e5a41ccd7757b15185dc298e85112ca7fea1022100e979474bae5d7cb44e5149af8b146eab1cb237646094c99eae07579981b2eeae0121025801"
+    "704c59321b645109931691c996a0ae797cf155a5ece34bdc065e6b5437a1ffffffff02fc0a0300000000001976a9145a3acbc7bbcc97c5ff16f5909c"
+    "9d7d3fadb293a888ac801a06000000000017a914e8c300c87986efa84c37c0519929019ef86eb5b48700000000")
+
 
 def transaction(prev_hash=bytes(32), vout=0xffffffff, amount=1, witness=False):
     """Serialize a one-input, one-output transaction and its stripped form."""
@@ -82,6 +92,15 @@ def test_invalid_block_serialization(self):
             with self.subTest(case=case), self.assertRaises(ValueError):
                 read_block(wire)
 
+    def test_p2sh_spend_passes_legacy_and_fails_p2sh(self):
+        """The 2012 spend fails only once the redeem script runs; a spend that fails regardless is not evidence."""
+        spend, funding = read_transaction(P2SH_SPEND), read_transaction(P2SH_FUNDING)
+        self.assertEqual(spend.vin[0].prevout.hash, funding.GetTxid())
+        self.assertTrue(p2sh_spend_fails(spend, 0, funding))
+        wrong = CTransaction([CTxIn(spend.vin[0].prevout, CScript([b"\x51"]))], spend.vout)
+        with self.subTest(case="fails without P2SH"), self.assertRaisesRegex(ValueError, "even without P2SH"):
+            p2sh_spend_fails(wrong, 0, funding)
+
     def test_parent_transaction_reuse_excludes_coinbases_and_absent_transactions(self):
         """The named witness must be a non-coinbase transaction present in both blocks."""
         coinbase = transaction()

ci/test_sanity_check.py

@@ -219,6 +219,18 @@ def test_parent_reuse_requires_canonical_parent_and_matching_witness(self):
         (cache / f"height-{self.record['height'] - 1}.hash").write_text(parent)
         self.assertTrue(any("missing cached parent header" in p for p in self.validate(prevouts_dir=cache)))
 
+    def test_p2sh_failure_requires_named_spend_after_activation(self):
+        """Admit a P2SH body from cached evidence; reject an unspent outpoint, a pre-BIP16 time and a missing cache."""
+        self.record = self.for_rule("p2sh_redeem_script_failure")
+        self.copy_body(self.record)
+        self.assertEqual(self.validate(), [])
+        with self.subTest(case="outpoint not spent"), patch.dict(self.record["context"], {"failing_prevout": "00" * 32 + ":0"}):
+            self.assertTrue(any("exactly one input" in p for p in self.validate()))
+        with self.subTest(case="before activation"), patch.object(CHECK, "BIP16_TIME", 2 ** 31):
+            self.assertTrue(any("BIP16 activation" in p for p in self.validate()))
+        with self.subTest(case="missing cache"):
+            self.assertTrue(any("missing cached" in p for p in self.validate(prevouts_dir=self.root / "empty")))
+
     def test_body_matches_claimed_evidence(self):
         """Bind the named failure and supplied coinbase scriptSig to the available body."""
         self.record = self.for_rule("bad-txns-vout-toolarge")