feat(dataset): list reported but unproved blocks in their own file

README.md

@@ -14,6 +14,7 @@ This dataset covers blocks that fail those rules, including failures that can be
 - [`docs/notes.md`](docs/notes.md): replay behaviour and incident notes.
 - `blocks/{height}-{hash}.bin`: full block, when available.
 - `proofs/{height}-{hash}.json`: for a P2SH record without a body, the failing transaction and the block's ordered transaction IDs.
+- [`data/reported-blocks.jsonl`](data/reported-blocks.jsonl): blocks reported as invalid whose failure is not established.
 
 Merge-mined recoveries generally provide a header and coinbase rather than a full Bitcoin block.
 
@@ -27,6 +28,8 @@ Include `context` fields needed to establish the failure: BIP34 coinbase height
 Omit unknown optional fields.
 When the pool is known, give `pool` with `pool_basis`: `tag` for a coinbase tag, `address` for a payout address listed in [mining-pools](https://github.com/bitcoin-data/mining-pools), or `reported` when only a contemporaneous report names the pool.
 
+A block whose failure is only reported goes in [`data/reported-blocks.jsonl`](data/reported-blocks.jsonl) with its sources, until the evidence turns up.
+
 Include all available `observations`, with a source and provenance URL for each.
 Distinct child-chain blocks and independent observers remain separate observations.
 Use `merge_mining` for child-chain commitments, `p2p` for direct Bitcoin network reception, and `scrape` for website or API archives where direct reception is not established.

ci/sanity-check.py

@@ -33,7 +33,9 @@
 DATA_PATH = Path("data/invalid-blocks.jsonl")
 BLOCKS_DIR = Path("blocks")
 PROOFS_DIR = Path("proofs")
+REPORTED_PATH = Path("data/reported-blocks.jsonl")
 REQUIRED = {"height", "hash", "header", "prev_hash", "nTime", "core_reject_reason", "rule"}
+REPORTED_REQUIRED = {"height", "hash", "reported_failure", "sources"}
 CONTEXT_FIELDS = {
     "expected_nbits", "parent_mtp", "coinbase_height", "coinbase_scriptsig_hex",
     "pool", "pool_basis", "parent_kind", "missing_prevout", "parent_txid", "failing_prevout",
@@ -160,6 +162,54 @@ def string(record: dict[str, Any], name: str) -> str:
     return value
 
 
+def checked_header(record: dict[str, Any]) -> CBlockHeader:
+    """Decode the 80-byte header and require it to hash to `hash` under a valid target."""
+    header = CBlockHeader.deserialize(hex_value(record, "header", 80))
+    calculated = b2lx(header.GetHash())
+    if record["hash"] != calculated:
+        raise ValueError("header hash mismatch")
+    target = target_from_bits(header.nBits)
+    if not target or int(calculated, 16) > target:
+        raise ValueError("header does not satisfy a valid PoW target")
+    return header
+
+
+def http_url(value: Any) -> bool:
+    """True for a string that parses as an HTTP(S) URL with a host."""
+    url = urlparse(value) if isinstance(value, str) else None
+    return url is not None and url.scheme in ("http", "https") and bool(url.hostname)
+
+
+def check_reported(path: Path | str, established: Set[str]) -> tuple[list[str], int]:
+    """Check the reported-blocks ledger: identity, sources, ordering and no overlap with admitted records."""
+    problems = []
+    seen = set()
+    last_key = None
+    for where, record in read_jsonl(Path(path), problems):
+        try:
+            check_fields(record, REPORTED_REQUIRED, REPORTED_REQUIRED | {"header"})
+            height = integer(record, "height", 1)
+            hex_value(record, "hash", 32)
+            string(record, "reported_failure")
+            sources = record["sources"]
+            if not isinstance(sources, list) or not sources or not all(map(http_url, sources)):
+                raise ValueError("sources must be a nonempty array of HTTP(S) URLs")
+            if "header" in record:
+                checked_header(record)
+            key = (height, record["hash"])
+            if last_key is not None and key < last_key:
+                raise ValueError("records must be ordered by height then hash")
+            last_key = key
+            if record["hash"] in seen:
+                raise ValueError(f"duplicate block hash {record['hash']}")
+            seen.add(record["hash"])
+            if record["hash"] in established:
+                raise ValueError("reported block is already an admitted record")
+        except ValueError as exc:
+            problems.append(f"{where}: {exc}")
+    return problems, len(seen)
+
+
 def check_context(record: dict[str, Any]) -> None:
     """Validate context encoding and the fields required by the rule registry."""
     details = record.get("context", {})
@@ -218,8 +268,7 @@ def check_observations(record: dict[str, Any]) -> int:
             hex_value(observation, "child_block_hash", 32)
         if "child_header" in observation:
             hex_value(observation, "child_header", 80)
-        url = urlparse(observation["provenance"])
-        if url.scheme not in ("http", "https") or not url.hostname:
+        if not http_url(observation["provenance"]):
             raise ValueError("provenance must be an HTTP(S) URL")
         # A chain can commit the same Bitcoin parent at multiple child heights;
         # independent recorders can also observe the same block. Reject only
@@ -374,7 +423,7 @@ def check_failure_evidence(record: dict[str, Any], block: CBlock | None, prevout
 def check_dataset(path: Path | str = DATA_PATH, blocks_dir: Path | str = BLOCKS_DIR,
                   prevouts_dir: Path | str = PREVOUTS_DIR, fetch_prevouts: bool = False,
                   apis: Sequence[str] = DEFAULT_APIS,
-                  proofs_dir: Path | str = PROOFS_DIR) -> tuple[list[str], tuple[int, int, int, int, int]]:
+                  proofs_dir: Path | str = PROOFS_DIR) -> tuple[list[str], set[str], tuple[int, int, int, int]]:
     problems = []
     seen = set()
     remaining = {"block": {path.name: path for path in Path(blocks_dir).glob("*.bin")},
@@ -409,13 +458,7 @@ def check_dataset(path: Path | str = DATA_PATH, blocks_dir: Path | str = BLOCKS_
             if block_hash in seen:
                 raise ValueError(f"duplicate block hash {block_hash}")
             seen.add(block_hash)
-            parsed_header = CBlockHeader.deserialize(header)
-            calculated = b2lx(parsed_header.GetHash())
-            if block_hash != calculated:
-                raise ValueError("header hash mismatch")
-            target = target_from_bits(parsed_header.nBits)
-            if not target or int(calculated, 16) > target:
-                raise ValueError("header does not satisfy a valid PoW target")
+            parsed_header = checked_header(record)
             if record["prev_hash"] != b2lx(parsed_header.hashPrevBlock):
                 raise ValueError("prev_hash mismatch with header")
             if timestamp != parsed_header.nTime:
@@ -447,8 +490,8 @@ def check_dataset(path: Path | str = DATA_PATH, blocks_dir: Path | str = BLOCKS_
     for kind, paths in remaining.items():
         for path in sorted(paths.values()):
             problems.append(f"{path}: orphan {kind} file; name must match a dataset record")
-    return problems, (len(seen), context_count, observation_count,
-                      found["block"] - len(remaining["block"]), found["proof"] - len(remaining["proof"]))
+    return problems, seen, (context_count, observation_count,
+                            found["block"] - len(remaining["block"]), found["proof"] - len(remaining["proof"]))
 
 
 def main() -> int:
@@ -458,14 +501,17 @@ def main() -> int:
     parser.add_argument("--prevouts-dir", type=Path, default=PREVOUTS_DIR, help="verified transaction cache directory")
     parser.add_argument("--api-url", action="append", help="Esplora API base URL; repeat for fallback providers")
     args = parser.parse_args()
-    problems, counts = check_dataset(prevouts_dir=args.prevouts_dir, fetch_prevouts=args.fetch_prevouts,
-                                    apis=args.api_url or DEFAULT_APIS)
+    problems, admitted, counts = check_dataset(prevouts_dir=args.prevouts_dir, fetch_prevouts=args.fetch_prevouts,
+                                              apis=args.api_url or DEFAULT_APIS)
+    reported_problems, reported = check_reported(REPORTED_PATH, admitted)
+    problems += reported_problems
     if problems:
         print("sanity-check failed:")
         print("\n".join(problems))
         return 1
     print("sanity-check successful")
-    print(f"  {counts[0]} blocks, {counts[1]} contexts, {counts[2]} observations, {counts[3]} block files, {counts[4]} proof files")
+    print(f"  {len(admitted)} blocks, {counts[0]} contexts, {counts[1]} observations, {counts[2]} block files, {counts[3]} proof files")
+    print(f"  {reported} reported blocks not admitted")
     return 0
 
 

ci/test_sanity_check.py

@@ -138,7 +138,7 @@ def test_sigops_requires_previous_transactions(self):
         self.copy_evidence(self.record)
         path = self.root / "sigops.jsonl"
         path.write_text(json.dumps(self.record) + "\n")
-        problems, _ = CHECK.check_dataset(path, self.root / "blocks", self.root / "empty-cache")
+        problems = CHECK.check_dataset(path, self.root / "blocks", self.root / "empty-cache")[0]
         self.assertTrue(any("missing cached previous transaction" in p for p in problems))
 
     def test_missing_parent_requires_recorded_outpoint_and_cached_evidence(self):
@@ -258,6 +258,28 @@ def test_p2sh_proof_stands_in_for_a_missing_body(self):
             (self.root / "proofs" / f"{body['height']}-{body['hash']}.json").write_text(json.dumps(proof))
             self.assertTrue(any("not both" in p for p in self.validate([self.record, body])))
 
+    def test_reported_ledger(self):
+        """The real ledger passes; an admitted hash, a header that does not hash to its record, empty sources, disorder and a duplicate are rejected."""
+        rows = [json.loads(line) for line in CHECK.REPORTED_PATH.read_text().splitlines()]
+        established = {r["hash"] for r in self.records}
+        path = self.root / "reported.jsonl"
+
+        def check(content):
+            path.write_text("".join(json.dumps(row) + "\n" for row in content))
+            return CHECK.check_reported(path, established)[0]
+
+        self.assertEqual(check(rows), [])
+        cases = (
+            ("admitted hash", [dict(rows[0], hash=self.record["hash"])], "already an admitted record"),
+            ("wrong header", [dict(rows[0], header=self.record["header"])], "header hash mismatch"),
+            ("no sources", [dict(rows[0], sources=[])], "nonempty array"),
+            ("unsorted", [rows[1], rows[0]], "ordered by height"),
+            ("duplicate", [rows[0], rows[0]], "duplicate block hash"),
+        )
+        for case, content, error in cases:
+            with self.subTest(case=case):
+                self.assertTrue(any(error in p for p in check(content)))
+
     def test_body_matches_claimed_evidence(self):
         """Bind the named failure and supplied coinbase scriptSig to the available body."""
         self.record = self.for_rule("bad-txns-vout-toolarge")

data/reported-blocks.jsonl

@@ -0,0 +1,10 @@
+{"height":197701,"hash":"00000000000003414be07bc5e700b1b2168ceb7259b4b9de4b583c435ebc03f2","reported_failure":"coinbase pays more than the subsidy plus fees","sources":["https://buildingbitcoin.org/bitcoin-dev/log-2012-09-09.html","https://web.archive.org/web/20120918194744id_/http://eligius.st:80/~wizkid057/newstats/blocks.php"]}
+{"height":197705,"hash":"0000000000000607179812c7bc8751fb834436bdb79f1ab6239d34b4f8cd2dda","reported_failure":"coinbase pays more than the subsidy plus fees","sources":["https://buildingbitcoin.org/bitcoin-dev/log-2012-09-09.html","https://web.archive.org/web/20120918194744id_/http://eligius.st:80/~wizkid057/newstats/blocks.php"]}
+{"height":197883,"hash":"000000000000011e6d02f6063d4a797ce42fe1f97ecf3dd640b8145f12c2304d","header":"020000005a7371583e7793a9d8e719a6586fb9c092ccd1b4935a42b765000000000000006243836741520333a1083c2dddf0904cf36ce705e0175df25aa7d13fb40f5041b2954b50383a061a3b092f03","reported_failure":"coinbase pays more than the subsidy plus fees","sources":["https://buildingbitcoin.org/bitcoin-dev/log-2012-09-09.html","https://web.archive.org/web/20120918194744id_/http://eligius.st:80/~wizkid057/newstats/blocks.php"]}
+{"height":212048,"hash":"0000000000000202a4ba1a09870c43b6c7f316a62b755cb82bffaea347c271b3","reported_failure":"unspecified: a node reported InvalidChainFound","sources":["https://buildingbitcoin.org/bitcoin-dev/log-2012-12-13.html#l-334","https://pastebin.com/raw/LZxst5vD","https://bitcointalk.org/index.php?topic=1403436.msg14244002#msg14244002","https://web.archive.org/web/20220518172624/https://pastebin.com/LZxst5vD"]}
+{"height":363997,"hash":"000000000000000003ae1223f4926ec86100885cfe1484dc52fd67e042a19b12","reported_failure":"bad-version: version 2 after BIP66 enforcement","sources":["https://gnusha.org/pi/bitcoindev/48d3940ab1a2bd53c6e056ce7fbcd361@cock.lu/","https://www.mail-archive.com/bitcoin-dev@lists.linuxfoundation.org/msg04789.html"]}
+{"height":364261,"hash":"00000000000000000b6adf92bc192b3c21210f456ab21b5e46951665c74cfab2","reported_failure":"bad-version: version 2 after BIP66 enforcement","sources":["https://gnusha.org/pi/bitcoindev/48d3940ab1a2bd53c6e056ce7fbcd361@cock.lu/","https://www.mail-archive.com/bitcoin-dev@lists.linuxfoundation.org/msg04789.html"]}
+{"height":367195,"hash":"0000000000000000116322b5f25826787b01f7a70fb322837b68dff8216cefc4","reported_failure":"bad-version: version 2 after BIP66 enforcement","sources":["https://gnusha.org/pi/bitcoindev/48d3940ab1a2bd53c6e056ce7fbcd361@cock.lu/","https://www.mail-archive.com/bitcoin-dev@lists.linuxfoundation.org/msg04789.html"]}
+{"height":386682,"hash":"0000000000000000083cbdbb25c1607527c8f3fdb16f0d048c4439a73b501cb6","reported_failure":"bad-version: version 2 after BIP66 enforcement","sources":["https://gnusha.org/pi/bitcoindev/48d3940ab1a2bd53c6e056ce7fbcd361@cock.lu/","https://www.mail-archive.com/bitcoin-dev@lists.linuxfoundation.org/msg04789.html"]}
+{"height":387396,"hash":"00000000000000000afc9fbe7cfe8a6b50502d509ba626beb2e2d6c15d1d3ee3","reported_failure":"bad-version: version 2 after BIP66 enforcement","sources":["https://gnusha.org/pi/bitcoindev/48d3940ab1a2bd53c6e056ce7fbcd361@cock.lu/","https://www.mail-archive.com/bitcoin-dev@lists.linuxfoundation.org/msg04789.html"]}
+{"height":450529,"hash":"000000000000000000cf208f521de0424677f7a87f2f278a1042f38d159565f5","header":"0000002088d8c92a3cf343c66e3586efda20c57892687c26026c4d01000000000000000073bad4c202ec5306f8f4525f0a148bb03379275448b505a018b5bc4cc3feefcd9e928d5847cc021832ba9b71","reported_failure":"bad-blk-length: 1000023 bytes reported","sources":["https://bitco.in/forum/threads/buir-2017-01-29-statement-regarding-excessive-block-by-bitcoin-unlimited-software-29-jan-2017.1790/","https://web.archive.org/web/20170129223228id_/https://live.blockcypher.com/btc/block/000000000000000000cf208f521de0424677f7a87f2f278a1042f38d159565f5/","https://web.archive.org/web/20170713202211id_/https://live.blockcypher.com/btc/block/000000000000000000cf208f521de0424677f7a87f2f278a1042f38d159565f5/"]}

docs/notes.md

@@ -20,6 +20,15 @@ The records below attribute a pool another way, and `pool_basis` says which.
 474294 and 477115 are attributed to 1Hash from the [BitcoinTalk thread](https://bitcointalk.org/index.php?topic=2041607.0) of July 2017 that discussed both blocks; their coinbases carry `/NYA/` and no pool tag.
 226845, 226895 and 226912 are attributed to mmpool, Chris Double's [Bitparking merged-mining pool](https://bitcointalk.org/index.php?topic=57148.0) at mmpool.bitparking.com, from his [20 March 2013 post](https://bitcointalk.org/index.php?topic=57148.msg1646921#msg1646921) in that thread reporting three invalidated blocks that day, and the [bitcoin-dev log](https://buildingbitcoin.org/bitcoin-dev/log-2013-03-20.html) of the same day, where he reports the `block height mismatch in coinbase` rejection and names 226845; the later 367047 carries the `mmpool` tag itself.
 
+## Reported blocks
+
+`data/reported-blocks.jsonl` keeps ten blocks that were reported as invalid but cannot be admitted.
+Three Eligius blocks of September 2012 were reported as coinbase overpayments alongside the admitted 197438; 197883's header survives but its other transaction and fee do not, and 197701 and 197705 are known only by hash.
+P2Pool's 212048 is known from a December 2012 `InvalidChainFound` report and a node-history dump, without a header or a rejection reason.
+Five version-2 hashes from the BIP66 and BIP65 windows come from a March 2017 bitcoin-dev message and still lack headers.
+Bitcoin Unlimited's 450529 has a recovered header and a reported size of 1000023 bytes, but no body bytes to establish it.
+Issues labelled [`reported`](https://github.com/bitcoin-data/invalid-blocks/issues?q=label%3Areported) record what has already been searched for each incident and are the place to bring the missing header or body.
+
 ## Incident notes
 
 ### 507514, 509557, 515319 and 534339 - AntPool parent-transaction reuse (2018)

docs/schema.md

@@ -19,6 +19,7 @@ Hex strings are lowercase without `0x`.
 Bitcoin hashes use RPC/display byte order with leading zeros; `header` is the 160-character wire serialization.
 Full blocks, when available, are `blocks/{height}-{hash}.bin`.
 A P2SH record without a body carries `proofs/{height}-{hash}.json` instead.
+Blocks whose failure was reported but never established are listed separately in `data/reported-blocks.jsonl`, described at the end of this document; they are not part of the dataset's admitted records.
 
 ## Required fields
 
@@ -259,3 +260,19 @@ This evaluates one input with a library interpreter; it is not Bitcoin Core's in
 A record without a body may carry `proofs/{height}-{hash}.json`: an object with `transaction`, the hex of the failing transaction, and `txids`, the block's complete ordered transaction IDs.
 The list must reproduce the header's merkle root and contain the transaction's txid, which binds the transaction to the header without its other bodies; the input is then checked exactly as for a body.
 A record has a body or a proof file, not both, and a proof file for any other rule is an error.
+
+## Reported blocks
+
+`data/reported-blocks.jsonl` lists blocks that a contemporary source reported as invalid but whose failure the dataset cannot establish, usually because the header or body is lost.
+They are not admitted records: nothing in the rule tables applies to them, and a body or proof file for one of them is an error until it becomes a record.
+One object per line, sorted by `(height, hash)`, with the same encoding rules as the main file.
+
+| Field | Type | Meaning |
+| --- | --- | --- |
+| `height` | integer | Reported height. |
+| `hash` | string | Reported block hash, the unique key; it must not appear in `data/invalid-blocks.jsonl`. |
+| `header` | string | 80-byte header in hex, when recovered; it must hash to `hash` and meet its own target. |
+| `reported_failure` | string | What the report claims, in a few words. |
+| `sources` | array | HTTP(S) URLs of the reports. |
+
+CI checks these fields, the header when present, the ordering and the overlap; it does not verify the reports.