From 83a7efb1299078f8f444536f802a9565b5194c15 Mon Sep 17 00:00:00 2001 From: Matt Zamora Date: Sun, 27 Sep 2026 23:02:45 -0400 Subject: [PATCH 01/13] fix(pdf): recover heading structure across lines (#524) Ordered, fail-closed passes re-read detected headings across the reading order. pdf_text records each line's letter case pattern; pdf_anchors uses it to bound an agency's reach. Pins re-derived for the new parser revision. --- .../0014-leveled-heading-tree-scope.md | 9 +- examples/hr8752_pdf_diff.html | 86 +-- src/deltatrack/parsers/pdf_anchors.py | 60 +- .../parsers/pdf_heading_exceptions.py | 51 ++ src/deltatrack/parsers/pdf_heading_passes.py | 518 ++++++++++++++++++ src/deltatrack/parsers/pdf_text.py | 59 +- tests/data/pdf/anchors_golden/118-s-4795.json | 64 +-- .../118-s-4795.pre-agency-anchors.json | 32 +- .../data/pdf/anchors_golden/major_vocab.json | 26 +- tests/data/pdf_canonical_baseline.json | 92 ++-- tests/data/round1_pairing_sentinel.json | 54 +- tests/test_pdf_anchor_golden.py | 67 +-- tests/test_pdf_heading_passes.py | 437 +++++++++++++++ tests/test_pdf_round1_revocation.py | 19 +- tests/test_pdf_size_detection.py | 26 +- tests/test_pipeline_parity.py | 8 +- tests/test_structure_vocabulary_gate.py | 3 + web/webapp/sample/example.html | 86 +-- 18 files changed, 1418 insertions(+), 279 deletions(-) create mode 100644 src/deltatrack/parsers/pdf_heading_exceptions.py create mode 100644 src/deltatrack/parsers/pdf_heading_passes.py create mode 100644 tests/test_pdf_heading_passes.py diff --git a/docs/decisions/0014-leveled-heading-tree-scope.md b/docs/decisions/0014-leveled-heading-tree-scope.md index 28da39e0..eda399d7 100644 --- a/docs/decisions/0014-leveled-heading-tree-scope.md +++ b/docs/decisions/0014-leveled-heading-tree-scope.md @@ -85,9 +85,9 @@ not an equality it would have to fudge: | bill (v1→v2) | XML | PDF | gap | attributed cause | |--------------|----:|----:|----:|------------------| -| 118-hr-8752 | 39 | 37 | −2 | clean; XML sees 2 bare subsections of added SEC.s the PDF folds into the section block (#188) | +| 118-hr-8752 | 39 | 38 | −1 | clean; XML sees 2 bare subsections of added SEC.s the PDF folds into the section block (#188); the PDF sees the added SPENDING REDUCTION ACCOUNT heading as its own block ([0022](0022-pdf-heading-convergence.md)) | | 118-hr-8774 | 31 | 33 | +2 | PDF over-segments a few blocks | -| 117-hr-4502 | 1415 | 1500 | +85 | PDF over-segments a large added block; XML +304 from subsection nodes (#188) | +| 117-hr-4502 | 1415 | 1456 | +41 | PDF over-segments a large added block; XML +304 from subsection nodes (#188) | | 115-hr-5895 | 300 | 336 | +36 | division-collapse + segmentation; XML +54 from subsection nodes (#188) | Snapshot 2026-06-29; the 117-hr-4502 and 115-hr-5895 PDF rows re-measured @@ -100,7 +100,10 @@ the #96 PDF recalibration), and the XML↔PDF gap narrows sharply (4502 +389→+ 5895 +90→+36) — much of what was attributed to PDF over-segmentation was subsection granularity the XML tree previously lacked. 118-hr-8752's exact parity gives way to a −2 by design: XML emits bare subsections the PDF cannot detect -(the all-subsections scope decision recorded on #188). The live numbers and the bands that gate them are in +(the all-subsections scope decision recorded on #188). Two PDF rows re-measured +2026-09-27 after the heading passes of [0022](0022-pdf-heading-convergence.md): 4502 +1500→1456 (wrapped account names now one block), 8752 37→38 (a heading the PDF +now reads). The live numbers and the bands that gate them are in `tests/test_pipeline_parity.py` (regenerate the table: `.venv/bin/python scripts/parity_table.py`). The Senate #89 residual `118-s-4795` recovers its heading hierarchy at a size-band ratio of 1.02 diff --git a/examples/hr8752_pdf_diff.html b/examples/hr8752_pdf_diff.html index 1ec03259..cdcc3cab 100644 --- a/examples/hr8752_pdf_diff.html +++ b/examples/hr8752_pdf_diff.html @@ -317,15 +317,15 @@ -
TITLE I (3)
DEPARTMENTAL MANAGEMENT, INTELLIGENCE, SITUATIONAL AWARENESS, AND OVERSIGHT (3)
    OFFICE OF THE SECRETARY AND EXECUTIVE MANAGEMENT (1)
      OPERATIONS AND SUPPORT (1)
      MANAGEMENT DIRECTORATE (2)
        OPERATIONS AND SUPPORT (1)
        PROCUREMENT, CONSTRUCTION, AND IMPROVEMENTS (1)
        TITLE II (10)
        SECURITY, ENFORCEMENT, AND INVESTIGATIONS (10)
          U.S. CUSTOMS AND BORDER PROTECTION (2)
            OPERATIONS AND SUPPORT (1)
            PROCUREMENT, CONSTRUCTION, AND IMPROVEMENTS (1)
            U.S. IMMIGRATION AND CUSTOMS ENFORCEMENT (1)
              OPERATIONS AND SUPPORT (1)
              TRANSPORTATION SECURITY ADMINISTRATION (2)
                OPERATIONS AND SUPPORT (1)
                PROCUREMENT, CONSTRUCTION, AND IMPROVEMENTS (1)
                COAST GUARD (2)
                  OPERATIONS AND SUPPORT (1)
                  PROCUREMENT, CONSTRUCTION, AND IMPROVEMENTS (1)
                  UNITED STATES SECRET SERVICE (1)
                    OPERATIONS AND SUPPORT (1)
                    ADMINISTRATIVE PROVISIONS (2)
                      SEC. 211 (1)
                      SEC. 212 (1)
                      TITLE III (2)
                      PROTECTION, PREPAREDNESS, RESPONSE, AND RECOVERY (2)
                        FEDERAL EMERGENCY MANAGEMENT AGENCY (2)
                          OPERATIONS AND SUPPORT (1)
                          FEDERAL ASSISTANCE (1)
                          TITLE IV (4)
                          RESEARCH, DEVELOPMENT, TRAINING, AND SERVICES (4)
                            SCIENCE AND TECHNOLOGY DIRECTORATE (1)
                              RESEARCH AND DEVELOPMENT (1)
                              ADMINISTRATIVE PROVISIONS (3)
                                SEC. 406 (1)
                                SEC. 413 (2)
                                TITLE V (15)
                                GENERAL PROVISIONS (15)
                                  SEC. 552 (1)
                                  SEC. 553 (1)
                                  SEC. 554 (1)
                                  SEC. 555 (1)
                                  SEC. 556 (1)
                                  SEC. 557 (1)
                                  SEC. 558 (1)
                                  SEC. 559 (1)
                                  SEC. 560 (1)
                                  SEC. 561 (1)
                                  SEC. 562 (1)
                                  SEC. 563 (1)
                                  SEC. 564 (1)
                                  SEC. 565 (1)
                                  SEC. 566 (1)
                                  Front Matter (1)
                                  TITLE V (2)
                                  +
                                  TITLE I (3)
                                  DEPARTMENTAL MANAGEMENT, INTELLIGENCE, SITUATIONAL AWARENESS, AND OVERSIGHT (3)
                                    OFFICE OF THE SECRETARY AND EXECUTIVE MANAGEMENT (1)
                                      OPERATIONS AND SUPPORT (1)
                                      MANAGEMENT DIRECTORATE (2)
                                        OPERATIONS AND SUPPORT (1)
                                        PROCUREMENT, CONSTRUCTION, AND IMPROVEMENTS (1)
                                        TITLE II (10)
                                        SECURITY, ENFORCEMENT, AND INVESTIGATIONS (10)
                                          U.S. CUSTOMS AND BORDER PROTECTION (2)
                                            OPERATIONS AND SUPPORT (1)
                                            PROCUREMENT, CONSTRUCTION, AND IMPROVEMENTS (1)
                                            U.S. IMMIGRATION AND CUSTOMS ENFORCEMENT (1)
                                              OPERATIONS AND SUPPORT (1)
                                              TRANSPORTATION SECURITY ADMINISTRATION (2)
                                                OPERATIONS AND SUPPORT (1)
                                                PROCUREMENT, CONSTRUCTION, AND IMPROVEMENTS (1)
                                                COAST GUARD (2)
                                                  OPERATIONS AND SUPPORT (1)
                                                  PROCUREMENT, CONSTRUCTION, AND IMPROVEMENTS (1)
                                                  UNITED STATES SECRET SERVICE (1)
                                                    OPERATIONS AND SUPPORT (1)
                                                    ADMINISTRATIVE PROVISIONS (2)
                                                      SEC. 211 (1)
                                                      SEC. 212 (1)
                                                      TITLE III (2)
                                                      PROTECTION, PREPAREDNESS, RESPONSE, AND RECOVERY (2)
                                                        FEDERAL EMERGENCY MANAGEMENT AGENCY (2)
                                                          OPERATIONS AND SUPPORT (1)
                                                          FEDERAL ASSISTANCE (1)
                                                          TITLE IV (4)
                                                          RESEARCH, DEVELOPMENT, TRAINING, AND SERVICES (4)
                                                            SCIENCE AND TECHNOLOGY DIRECTORATE (1)
                                                              RESEARCH AND DEVELOPMENT (1)
                                                              ADMINISTRATIVE PROVISIONS (3)
                                                                SEC. 406 (1)
                                                                SEC. 413 (2)
                                                                TITLE V (16)
                                                                GENERAL PROVISIONS (1)
                                                                  SPENDING REDUCTION ACCOUNT (1)
                                                                  SPENDING REDUCTION ACCOUNT (15)
                                                                    SEC. 552 (1)
                                                                    SEC. 553 (1)
                                                                    SEC. 554 (1)
                                                                    SEC. 555 (1)
                                                                    SEC. 556 (1)
                                                                    SEC. 557 (1)
                                                                    SEC. 558 (1)
                                                                    SEC. 559 (1)
                                                                    SEC. 560 (1)
                                                                    SEC. 561 (1)
                                                                    SEC. 562 (1)
                                                                    SEC. 563 (1)
                                                                    SEC. 564 (1)
                                                                    SEC. 565 (1)
                                                                    SEC. 566 (1)
                                                                    Front Matter (1)
                                                                    TITLE V (2)
                                                                    - +

                                                                    H.R. 8752 — Making appropriations for the Department of Homeland Security for the fiscal year ending September 30, 2025, and for other purposes.

                                                                    v1: reported-in-house → v2: engrossed-in-house · 118th Congress
                                                                    -
                                                                    modified 18added 16removed 2moved 1
                                                                    +
                                                                    modified 18added 17removed 2moved 1
                                                                    @@ -571,21 +571,33 @@

                                                                    TITLE IV > RESEARCH, DEVELOPMENT, TRAINING, AND SERVICES > ADMINISTRAT
                                                                    TITLE V
                                                                    GENERAL PROVISIONS -
                                                                    SEC. 552 +
                                                                    SPENDING REDUCTION ACCOUNT
                                                                    added -

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 552

                                                                    +

                                                                    TITLE V > GENERAL PROVISIONS > SPENDING REDUCTION ACCOUNT

                                                                    +
                                                                    +
                                                                    — (new in v2)p.97 L13
                                                                    +
                                                                    SPENDING REDUCTION ACCOUNT
                                                                    +
                                                                    +
                                                                    +
                                                                    +
                                                                    SPENDING REDUCTION ACCOUNT +
                                                                    SEC. 552 +
                                                                    +
                                                                    +added +

                                                                    TITLE V > SPENDING REDUCTION ACCOUNT > SEC. 552

                                                                    — (new in v2)p.97 L14
                                                                    SEC. 552. $0 (increased by $10,000,000).
                                                                    SEC. 553 -
                                                                    +
                                                                    added -

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 553

                                                                    +

                                                                    TITLE V > SPENDING REDUCTION ACCOUNT > SEC. 553

                                                                    — (new in v2)p.97 L15 – p.100 L6
                                                                    SEC. 553. (a) None of the funds made available by @@ -652,10 +664,10 @@

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 553

                                                                    SEC. 554 -
                                                                    +
                                                                    added -

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 554

                                                                    +

                                                                    TITLE V > SPENDING REDUCTION ACCOUNT > SEC. 554

                                                                    — (new in v2)p.100 L7 – p.100 L12
                                                                    SEC. 554. None of the funds made available by this @@ -666,10 +678,10 @@

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 554

                                                                    SEC. 555 -
                                                                    +
                                                                    added -

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 555

                                                                    +

                                                                    TITLE V > SPENDING REDUCTION ACCOUNT > SEC. 555

                                                                    — (new in v2)p.100 L13 – p.100 L16
                                                                    SEC. 555. None of the funds appropriated or otherwise made available by this Act may be used to implement, @@ -678,10 +690,10 @@

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 555

                                                                    SEC. 556 -
                                                                    +
                                                                    added -

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 556

                                                                    +

                                                                    TITLE V > SPENDING REDUCTION ACCOUNT > SEC. 556

                                                                    — (new in v2)p.100 L17 – p.100 L20
                                                                    SEC. 556. None of the funds made available in this @@ -691,10 +703,10 @@

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 556

                                                                    SEC. 557 -
                                                                    +
                                                                    added -

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 557

                                                                    +

                                                                    TITLE V > SPENDING REDUCTION ACCOUNT > SEC. 557

                                                                    — (new in v2)p.100 L21 – p.100 L22
                                                                    SEC. 557. None of the funds made available by this @@ -702,10 +714,10 @@

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 557

                                                                    SEC. 558 -
                                                                    +
                                                                    added -

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 558

                                                                    +

                                                                    TITLE V > SPENDING REDUCTION ACCOUNT > SEC. 558

                                                                    — (new in v2)p.100 L23 – p.100 L25
                                                                    SEC. 558. None of the funds made available by this @@ -714,10 +726,10 @@

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 558

                                                                    SEC. 559 -
                                                                    +
                                                                    added -

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 559

                                                                    +

                                                                    TITLE V > SPENDING REDUCTION ACCOUNT > SEC. 559

                                                                    — (new in v2)p.101 L1 – p.101 L2
                                                                    SEC. 559. None of the funds made available by this @@ -725,10 +737,10 @@

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 559

                                                                    SEC. 560 -
                                                                    +
                                                                    added -

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 560

                                                                    +

                                                                    TITLE V > SPENDING REDUCTION ACCOUNT > SEC. 560

                                                                    — (new in v2)p.101 L4 – p.101 L5
                                                                    SEC. 560. None of the funds made available under @@ -736,10 +748,10 @@

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 560

                                                                    SEC. 561 -
                                                                    +
                                                                    added -

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 561

                                                                    +

                                                                    TITLE V > SPENDING REDUCTION ACCOUNT > SEC. 561

                                                                    — (new in v2)p.101 L8 – p.101 L11
                                                                    SEC. 561. None of the funds appropriated or otherwise made available by this Act may be made available @@ -748,10 +760,10 @@

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 561

                                                                    SEC. 562 -
                                                                    +
                                                                    added -

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 562

                                                                    +

                                                                    TITLE V > SPENDING REDUCTION ACCOUNT > SEC. 562

                                                                    — (new in v2)p.101 L12 – p.102 L9
                                                                    SEC. 562. None of the funds made available by this @@ -771,10 +783,10 @@

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 562

                                                                    SEC. 563 -
                                                                    +
                                                                    added -

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 563

                                                                    +

                                                                    TITLE V > SPENDING REDUCTION ACCOUNT > SEC. 563

                                                                    — (new in v2)p.102 L10 – p.102 L12
                                                                    SEC. 563. The salary of Nejwa Ali, U.S. Citizenship @@ -783,10 +795,10 @@

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 563

                                                                    SEC. 564 -
                                                                    +
                                                                    added -

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 564

                                                                    +

                                                                    TITLE V > SPENDING REDUCTION ACCOUNT > SEC. 564

                                                                    — (new in v2)p.102 L13 – p.102 L16
                                                                    SEC. 564. None of the funds made available by this @@ -796,10 +808,10 @@

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 564

                                                                    SEC. 565 -
                                                                    +
                                                                    added -

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 565

                                                                    +

                                                                    TITLE V > SPENDING REDUCTION ACCOUNT > SEC. 565

                                                                    — (new in v2)p.102 L17 – p.102 L20
                                                                    SEC. 565. None of the funds made available by this @@ -809,10 +821,10 @@

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 565

                                                                    SEC. 566 -
                                                                    +
                                                                    added -

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 566

                                                                    +

                                                                    TITLE V > SPENDING REDUCTION ACCOUNT > SEC. 566

                                                                    — (new in v2)p.102 L21 – p.102 L25
                                                                    SEC. 566. None of the funds made available by this @@ -856,10 +868,10 @@

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 552

                                                                    Committed to the Committee of the Whole House on the State of the Union and ordered to be printed
                                                                    -
                                                                    +
                                                                    added -

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 567

                                                                    +

                                                                    TITLE V > SPENDING REDUCTION ACCOUNT > SEC. 567

                                                                    — (new in v2)p.103 L1 – p.104
                                                                    SEC. 567. None of the funds appropriated or otherwise made available by this Act may be used to make payments to the Department of Veterans Affairs for medical @@ -876,7 +888,7 @@

                                                                    TITLE V > GENERAL PROVISIONS > SEC. 567

                                                                    Homeland Security for the fiscal year ending
                                                                    -

                                                                    - + +