.*?
\s*(.*?)
', body, re.S) + sections.append({ + "id": token["id"], + "html": token["html"], + "heights": [int(h) for h in re.findall(r"\d+", token["name"].split(" - ")[0])], + "excerpt": rewrite_links(paragraph.group(1), BLOCK_ROOT) if paragraph else "", + }) + return body, converter.toc, sections + + +def link_heading_heights(body: str, root: str, records: list[dict[str, Any]]) -> str: + """Link the heights in each incident heading to their block pages, where a height names a single record.""" + heights = Counter(r["height"] for r in records) + single = {r["height"]: r["hash"] for r in records if heights[r["height"]] == 1} + + def link(height: re.Match[str]) -> str: + block_hash = single.get(int(height.group(0))) + return f'{height.group(0)}' if block_hash else height.group(0) + + def heading(match: re.Match[str]) -> str: + heights, dash, title = match.group(2).partition(" - ") + return f'{re.sub(r"\d+", link, heights)}{dash}{title}
' + + return HEADING.sub(heading, body) + + +def note_links(records: list[dict[str, Any]], incidents: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + """Map each record hash to the incident note naming its height, or naming a block with the same failing_prevout.""" + by_height = {height: incident for incident in incidents for height in incident["heights"]} + links = {r["hash"]: by_height[r["height"]] for r in records if r["height"] in by_height} + spends = {r["hash"]: r.get("context", {}).get("failing_prevout") for r in records} + by_spend = {spends[block_hash]: incident for block_hash, incident in links.items() if spends[block_hash]} + return {block_hash: by_spend[spend] for block_hash, spend in spends.items() if spend in by_spend} | links + + +# Evidence kinds, in index order, with their index card labels; a kind's badge reads as the name with spaces. +EVIDENCE_CARDS = { + "full-block": "with full block", + "spend-proof": "with P2SH spend proof", + "coinbase-proof": "with coinbase proof", + "header-only": "header only", +} + + +def stem(record: dict[str, Any]) -> str: + return f"{record['height']}-{record['hash']}" + + +def evidence_on_file(record: dict[str, Any]) -> tuple[str, dict[str, Any] | None]: + """Name the kind of evidence on file for a record, with its proof file when it has one.""" + if (BLOCKS_DIR / f"{stem(record)}.bin").exists(): + return "full-block", None + path = PROOFS_DIR / f"{stem(record)}.json" + if not path.exists(): + return "header-only", None + proof = json.loads(path.read_text()) + coinbase = CTransaction.deserialize(bytes.fromhex(proof["transaction"])).is_coinbase() + return ("coinbase-proof" if coinbase else "spend-proof"), proof + + +def badge(kind: str, href: str = "", label: str = "") -> str: + """Badge styled as kind; its text is label, or the kind with spaces for hyphens.""" + text = esc(label) if label else kind.replace("-", " ") + if href: + return f'{text}' + return f'{text}' + + +def site_url(path: str) -> str: + return f"{SITE_URL}/{path.removesuffix('index.html')}" + + +def page(path: str, title: str, description: str, body: str) -> str: + root = root_of(path) + + def nav(href: str, label: str) -> str: + current = ' aria-current="page"' if path == f"{href}index.html" else "" + return f'{label}' + + return f""" + + + + +
+
+
+{body}
+
+
+
+
+
+"""
+
+
+def index_page(
+ records: list[dict[str, Any]],
+ reported: list[dict[str, Any]],
+ on_file: dict[str, tuple[str, dict[str, Any] | None]],
+) -> tuple[str, str]:
+ path = "index.html"
+ counts = Counter(kind for kind, _ in on_file.values())
+ counts[""] = len(records)
+ cards = "\n".join(
+ f''
+ for kind, label in {"": "invalid blocks", **EVIDENCE_CARDS}.items()
+ )
+ chips = "".join(
+ f''
+ for rule, count in Counter(r["rule"] for r in records).most_common()
+ )
+ attributes = {"height": ' class="num" aria-sort="ascending"', "observations": ' class="num"'}
+ headers = "".join(
+ f'Bitcoin invalid blocks
+Headers and blocks with valid proof of work that fail a named consensus rule.
+
+
+A stale block follows the rules but ends up outside the active chain; these blocks break them.
+They were observed on the Bitcoin network or recovered from other sources, including chains that merge-mine with Bitcoin and archived block explorers.
+The data is in data/invalid-blocks.jsonl, and contributions are welcome in the repository.
+{cards}
+reported, not established{len(reported)}
+
+Blocks
+
+
+
+{len(records)} of {len(records)}
+{chips}
+
+
+
+
"""
+ description = f"{len(records)} Bitcoin blocks with valid proof of work that fail a named consensus rule, with headers, full blocks, evidence and observations."
+ return path, page(path, "Bitcoin invalid blocks", description, body)
+
+
+def context_rows(context: dict[str, Any]) -> list[tuple[str, str]]:
+ """List a record's context for display: the pool fields share a row, and parent_kind shows with the previous block."""
+ rows = []
+ if "pool" in context:
+ source = f' (source)' if "pool_provenance" in context else ""
+ rows.append(("pool", f'{esc(context["pool"])} by {esc(context["pool_basis"])}{source}'))
+ for key, value in context.items():
+ if key in ("pool", "pool_basis", "pool_provenance", "parent_kind"):
+ continue
+ shown = f'{esc(str(value))}'
+ if key == "parent_mtp":
+ shown += f' {utc(value)}'
+ rows.append((key, shown))
+ if key == "coinbase_scriptsig_hex":
+ text = "".join(chr(b) if 32 <= b < 127 else "·" for b in bytes.fromhex(value))
+ rows.append(("scriptSig as text", f'{esc(text)}'))
+ return rows
+
+
+def observations_panel(observations: list[dict[str, Any]]) -> str:
+ if not observations:
+ return ""
+ rows = []
+ for obs in observations:
+ child = esc(obs.get("child_chain", ""))
+ if "child_height" in obs:
+ child += f' {obs["child_height"]}'
+ seen = utc(obs["first_seen"]) if "first_seen" in obs else ""
+ url = esc(obs["provenance"])
+ rows.append(
+ f'Observations ({len(observations)})
+Where the block was seen or recovered. Observations record acquisition; they do not establish the failure.
+| channel | source | child chain | first seen | provenance |
|---|
Incident note: {incident['html']}
+{incident['excerpt']}
""" + rule_is_reject = record["rule"] == record["core_reject_reason"] + rule = badge("rule", f'{root}#{esc(record["rule"])}', record["rule"]) + if rule_is_reject: + rule_rows = [("rule and Core reject", rule)] + else: + rule_rows = [("rule", rule), ("Core reject", f"{esc(record['core_reject_reason'])}")]
+ verdict = dl(rule_rows + [("Core check", check), ("CI checks", ci_checks)])
+
+ files = []
+ if kind == "full-block":
+ size = (BLOCKS_DIR / f"{name}.bin").stat().st_size
+ files.append(("full block", f'{name}.bin {size:,} bytes'))
+ elif proof:
+ proved = "coinbase" if kind == "coinbase-proof" else "failing transaction"
+ placement = "its merkle branch" if "merkle_branch" in proof else "the block's ordered txids"
+ files.append(("proof file", f'{name}.json {proved} and {placement}'))
+ files.append(("record", f'data/invalid-blocks.jsonl, line {position + 1}'))
+
+ parent = next((r for r in records if r["hash"] == record["prev_hash"]), None)
+ previous = f'{record["prev_hash"]}'
+ if parent:
+ previous = f'{parent["hash"]} (invalid, in this dataset)'
+ elif context.get("parent_kind") == "canonical":
+ previous += f' (canonical, mempool.space)'
+ elif "parent_kind" in context:
+ previous += f' ({esc(context["parent_kind"])})'
+ header_rows = [
+ ("version", f'0x{header.nVersion & 0xffffffff:08x} ({header.nVersion})'),
+ ("previous block", previous),
+ *(("child in dataset", f'{r["hash"]} height {r["height"]}')
+ for r in records if r["prev_hash"] == record["hash"]),
+ ("merkle root", f'{b2lx(header.hashMerkleRoot)}'),
+ ("time", f'{utc(header.nTime)} ({header.nTime})'),
+ ("bits", f'0x{header.nBits:08x} difficulty {header.difficulty:,.0f}'),
+ ("nonce", str(header.nNonce)),
+ ("header hex", f'{record["header"]}'),
+ ]
+ shown_context = context_rows(context)
+ context_html = f'Context
{dl(shown_context)}
+{f'← {before["height"]}' if before else ""}
+{f'{after["height"]} →' if after else ""}
+
"""
+
+ body = f"""Invalid block {record['height']}
+{record['hash']}
+
+
+{observations_panel(record.get("observations", []))}
+{pager}"""
+ title = f"Invalid Bitcoin block {record['height']} {record['hash']}"
+ by_pool = f", mined by {context['pool']}" if "pool" in context else ""
+ reject = "" if rule_is_reject else f" ({record['core_reject_reason']})"
+ description = (
+ f"Bitcoin block {record['hash']} at height {record['height']}{by_pool}, header time {utc(record['nTime'], '%Y-%m-%d')}, "
+ f"fails {record['rule']}{reject}. Header, evidence and observations."
+ )
+ return path, page(path, title, description, body)
+
+
+def notes_page(body: str, contents: str, records: list[dict[str, Any]]) -> tuple[str, str]:
+ path = "notes/index.html"
+ root = root_of(path)
+ body = link_heading_heights(rewrite_links(body, root), root, records)
+ html_body = f"""
+
+Why it is invalid
{verdict}{note_html}Header
{dl(header_rows)}
+
+Evidence on file {badge(kind, f"{root}#{kind}")}
{dl(files)}Notes
+Rendered from docs/notes.md.
Contents
{contents}Reported blocks
+Blocks a contemporary source reported as invalid, not admitted because the header or block needed to establish the failure is missing.
+Issues labelled reported record what has been searched for each case and are the place to bring the missing data.
+The notes describe each group.
| height | hash | recovered | reported failure | sources |
|---|