diff --git a/.github/workflows/paper.yml b/.github/workflows/paper.yml index 0853506..d8506ed 100644 --- a/.github/workflows/paper.yml +++ b/.github/workflows/paper.yml @@ -66,6 +66,13 @@ jobs: cd paper quarto render index.qmd --to pdf + - name: Upload rendered PDF + uses: actions/upload-artifact@v4 + with: + name: paper-pdf + path: paper/out/*.pdf + if-no-files-found: error + - name: Render paper HTML run: | cd paper diff --git a/Makefile b/Makefile index a9c4d70..3891637 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: sync check test lint paper +.PHONY: sync check test lint paper etl reproduce reproduce-heavy figures-sync sync: uv sync --extra dev --locked @@ -13,3 +13,39 @@ check: test lint paper: cd paper && TEXMFVAR=/tmp/firm-microsim-tex-cache latexmk -lualatex -interaction=nonstopmode -halt-on-error main.tex + +# --------------------------------------------------------------------------- +# Reproduction (issue #40). `reproduce` regenerates both synthetic vintages and +# every checked results/*.txt and figure the paper reads, then syncs figures +# into paper/figures/. `reproduce-heavy` adds the two multi-build artifacts +# (seed sensitivity = two extra full builds; placebo B = one extra build). +# The PDF is rendered by the Paper workflow in CI (quarto is not required +# locally); see paper/README.md. +# --------------------------------------------------------------------------- +etl: + uv run python scripts/etl_ons_tables.py + +reproduce: etl + uv run firm-microsim --seed 42 + uv run firm-microsim-static + uv run python analysis/static_results_dump.py + uv run firm-microsim-bunching --figures-only + uv run python scripts/bunching_inference.py + uv run firm-microsim-notch + uv run firm-microsim-dynamic + uv run firm-microsim-reform-menu + uv run firm-microsim-dominated-region + uv run firm-microsim-verify-optimum + uv run firm-microsim-formulation-a-optima + uv run python analysis/recovery_bunching.py + $(MAKE) figures-sync + +reproduce-heavy: reproduce + uv run python scripts/seed_sensitivity.py --seeds 42 7 99 + uv run firm-microsim-placebo + +figures-sync: + @for f in paper/figures/*.png; do \ + b=$$(basename $$f); \ + if [ -f results/$$b ]; then cp results/$$b paper/figures/$$b; else echo "no results/$$b"; fi; \ + done diff --git a/README.md b/README.md index a2494c7..9d3de8b 100644 --- a/README.md +++ b/README.md @@ -41,15 +41,27 @@ threshold: and intermediate inputs. These within-band values are explicit modelling assumptions, not recovered administrative microdata. 2. **Calibrate firm weights** by multi-objective optimisation (Adam, symmetric - relative-error loss) so weighted totals reproduce the official targets — HMRC - VAT-registered counts by turnover band and by sector, ONS employment-band - totals, and HMRC VAT-liability totals — with turnover bands weighted most - heavily. VAT registration is then assigned: mandatory above the threshold, - voluntary below at the HMRC-calibrated rate. - -The result is ~2.94M firm rows weighted to ~2.5M UK firms. Because the population -is calibrated **to** the HMRC aggregates, agreement with them is an internal -consistency check, not external validation. + relative-error loss) on **two declared universes** (issue #37): + - the **ONS VAT/PAYE enterprise frame** (~2.72M enterprises): the population + total, the employment-band totals and the OBR near-threshold shape; + - the **HMRC VAT-registered subset** (~2.18M traders): trader counts by + turnover band and by trade sector, and net VAT liability by band. Each + frame firm enters these rows with a *registration propensity* — the HMRC + count in its turnover band divided by the frame's mass in that band + (0.89 below the threshold; 0.51–1.00 above it up to £10m, the remainder + being PAYE-only or exempt-sector enterprises outside the VAT net; 0.64 in + the >£10m band, where the shortfall is mostly the uniform £5m–£50m draw for + the open ONS "5000+" band placing too many rows above £10m — issue #40). HMRC + negative/zero-turnover traders are appended before calibration as an + out-of-frame stratum. + + VAT **scope** and **registration** flags are then assigned by seeded weighted + selection per band so registered totals match HMRC to within one weight. + +The result is ~2.94M firm rows (2.72M frame draws plus 0.22M appended traders) +weighted to ~2.72M frame enterprises, of which ~2.18M are VAT-registered. +Because the population is calibrated **to** the HMRC aggregates, agreement +with them is an internal consistency check, not external validation. The official target surface is also being mirrored into PolicyEngine Ledger and Populace. This repository keeps the paper's archived CSV inputs and generator for @@ -102,7 +114,15 @@ df, report = firm_microsim.generate(return_report=True) Output is written to `data/synthetic/synthetic_firms.csv` (`sic_code, annual_turnover_k, annual_input_k, vat_liability_k, employment, -weight, vat_registered`). +weight, vat_scope, vat_registered, in_frame`). `in_frame` marks ONS-frame +enterprises; `vat_scope` marks firms in the VAT net (registered above the +threshold, registrable below it); `vat_registered` marks HMRC-count-matched +traders. Static and behavioural costings use in-scope firms only. + +The processed ONS tables are rebuilt from the raw workbooks by +`scripts/etl_ons_tables.py` (Table 8 for turnover sizebands, Table 3 for +employment sizebands, both counting enterprises); `--check` fails if a checked +CSV differs from the extract, and `tests/test_data_universes.py` runs it. ## Paper @@ -143,7 +163,8 @@ the manuscript without changing the computational results. The population is calibrated to **five** official ONS + HMRC target groups; the validator scores each dimension as -`accuracy = max(0, 1 − |synthetic − target| / |target|)`. The displayed error is +`accuracy = max(0, 1 − |synthetic − target| / |target|)`, HMRC dimensions on +VAT-registered rows and ONS dimensions on frame rows. The displayed error is the clipped complement of that score, not a signed relative error. **Overall** is the simple mean over the five calibrated dimensions below. Reproduce with: @@ -154,12 +175,12 @@ firm-microsim-report | Calibrated dimension | 85k (2023-24) | 90k (2024-25) | | --- | ---: | ---: | -| HMRC turnover bands | 97.3% | 96.6% | -| ONS population | 88.3% | 91.1% | -| Employment bands | 92.1% | 92.3% | -| Sector distribution | 90.1% | 90.4% | -| VAT liability by band (6 calibrated bands) | 96.5% | 97.1% | -| **Overall (5 calibrated dimensions)** | **92.9%** | **93.5%** | +| HMRC turnover bands (registered rows; hold by construction) | 100.0% | 100.0% | +| ONS population (frame rows) | 99.7% | 99.8% | +| Employment bands (frame rows) | 99.8% | 99.9% | +| Sector distribution (registered rows) | 93.3% | 92.2% | +| VAT liability by band (6 calibrated bands, registered rows) | 97.1% | 98.0% | +| **Overall (5 calibrated dimensions)** | **98.0%** | **98.0%** | **VAT liability by *sector*** is **not** a calibration target — it is reported as an informational diagnostic only, and neither is the **below-threshold @@ -190,8 +211,9 @@ remains a true total. A vintage builds in ~15 seconds instead of ~13 minutes; headline aggregates reproduce the full build within ~0.3% and local bunching statistics within ~5%. Use for development only — release artifacts are full-size. Generator-seed sensitivity of the full build is recorded in -`results/seed_sensitivity.txt` (E ±111, raise ±£1.4m / taper ±£0.3m across -seeds; reproduce with `scripts/seed_sensitivity.py`). +`results/seed_sensitivity.txt` (E ±166, raise ±£1.1m / taper ±£1.1m, base +±£0.45bn across seeds — the scope flag is a seeded draw; reproduce with +`scripts/seed_sensitivity.py`). ## Populace/Ledger migration check @@ -229,7 +251,7 @@ targets and the paper's processed 2024-25 numeric inputs: six normalized source tables checked, zero mismatches, max numeric difference 0. It does **not** exactly replicate the paper's generated synthetic population: Populace's shared optimizer landed at 93.8% overall accuracy under its own validator versus the paper's -then-90.5% (2024-25 scores 93.5% on the corrected build, +then-90.5% (2024-25 scores 98.0% on the current two-universe build, `results/calibration_accuracy.txt`), but that overall pair is **not like-for-like**: HMRC turnover-band accuracy uses different band sets, and sector distribution reflects different calibration-target definitions. The directly @@ -281,24 +303,27 @@ firm-microsim-static # -> results/{vat_threshold_revenue_impact,revenue - `vat_threshold_revenue_impact.png` — the £85k→£90k anchor reform vs HMRC's published costing, by fiscal year. **Built on the £85k / 2023-24 vintage** — - the pre-reform basis HMRC actually had at the 6 March 2024 costing (the - threshold was still £85k until 1 April 2024). Full-deregistration model - −357/−364/−224/−78/+119 vs HMRC −150/−185/−125/−50/+65 £m (43% voluntary - retention: −203/−207/−128/−44/+68); both turn positive by 2028-29. The gap - shows that registration dynamics and population scope matter. See - `results/static_sweep.txt`. + the pre-reform basis HMRC actually had at the 6 March 2024 costing. Each year + differences revenue under the £90k policy and under the counterfactual + threshold path (85/85/87/89/92k), releasing in-scope firms only below the + £88k deregistration threshold: −250/−234/−200/−65/+123 vs HMRC + −150/−185/−125/−50/+65 £m; with 43% voluntary retention −142/−134/−114/−37/+123. + HMRC's figures lie between the two conventions in every release year. See + `results/static_sweep.txt` for the whole-band and fixed-preference + sensitivities. - `revenue_impact_2025_26.png` / `firms_impact_2025_26.png` — the forward static sweep of registration thresholds (£70k–£120k) vs the current £90k baseline, **on the £90k / 2024-25 vintage**. -**Two vintages, two exercises.** The anchor reform uses the £85k vintage, where -the affected `[85,90)k` band sits *above* the £85k registration threshold and is -cleanly populated with registered firms — so a **simple band-sum** suffices (no -de-bunching). The forward sweep uses the current £90k vintage; there the -`[85,90)k` firms are *below* threshold and the calibration concentrates weight -on them, so the sweep instead fits the clean above-threshold firm/liability -profile and extrapolates it across the threshold -(`StaticVATModel._counterfactual_bins`, unaged turnover scaled to the fiscal -year by a nominal-growth factor). Revenue and the anchor reform match the paper -and HMRC closely; the forward-sweep firm-count magnitudes run low because the -regenerated population has a lower near-threshold VAT-paying-firm density. +**Two vintages, two exercises, one convention.** The anchor reform uses the +£85k vintage, where the affected `[85,90)k` band sits *above* the £85k +registration threshold and is cleanly populated with in-scope registered firms, +so a direct band-sum suffices. The forward sweep uses the current £90k vintage +and is likewise a direct mechanical reclassification of in-scope firms +(`StaticVATModel.threshold_sweep`); voluntary registrants below the data-year +threshold stay registered, firms aged across it are released by a rise unless +protected by the £2k deregistration gap, and out-of-scope enterprises never +remit. `results/static_sweep.txt` decomposes each anchor year's release. In both exercises turnover **and** liability are aged to the +fiscal year by the same nominal-growth factor (relative to each vintage's own +data year), so band membership is evaluated on aged turnover — the fiscal-drag +convention. `results/static_sweep.txt` holds the machine-readable table. diff --git a/analysis/static_results_dump.py b/analysis/static_results_dump.py index 1fedfff..deb4c31 100644 --- a/analysis/static_results_dump.py +++ b/analysis/static_results_dump.py @@ -11,7 +11,7 @@ from __future__ import annotations from firm_microsim.config import RESULTS_DIR -from firm_microsim.static.model import StaticVATModel, SWEEP_THRESHOLDS +from firm_microsim.static.model import FISCAL_YEARS, StaticVATModel, SWEEP_THRESHOLDS LLAT_VOLUNTARY_SHARE = 0.43 # Liu-Lockwood-Almunia-Tam (2021): ~43% below-threshold @@ -33,18 +33,47 @@ def main() -> None: anchor = anchor_model.anchor_reform() W(anchor.to_string(index=False)) W("") + W("Release decomposition (headline; liability GBPm of firms registered under the") + W("baseline threshold but not under GBP90k, by data-year status):") + for fy in FISCAL_YEARS: + df = anchor_model._aged(anchor_model._growth(fy["year"])) + base_t, pol_t = float(fy["baseline"]), float(fy["policy"]) + rel = anchor_model._registered(df, base_t) & ~anchor_model._registered(df, pol_t) + lw = df["liab"] * df["weight"] + vol = float(lw[rel & df["voluntary"]].sum()) / 1e6 + man = float(lw[rel & df["mandatory"]].sum()) / 1e6 + nev = float(lw[rel & ~df["voluntary"] & ~df["mandatory"]].sum()) / 1e6 + W(f" {fy['year']}: released {float(df['weight'][rel].sum()):,.0f} firms; " + f"voluntary-at-data-year {vol:.1f}m, mandatory-at-data-year {man:.1f}m, " + f"not-registered-at-data-year {nev:.1f}m") + W("") + W("Deregistration-threshold sensitivity: the headline releases registered") + W("firms only below the GBP88k deregistration threshold ([85k, 88k) in the") + W("raise years). Releasing the whole [baseline, 90k) band instead (gap = 0):") + for _, row in anchor_model.anchor_reform(gap=0.0).iterrows(): + W(f" {row['year']}: whole-band release {float(row['policyengine_impact_m']):+,.1f}m") + W("") W("Voluntary-retention sensitivity (anchor, per year): headline assumes") W("every released firm deregisters (full liability lost). If the Liu et") W(f"al. (2021) voluntary share ({LLAT_VOLUNTARY_SHARE:.0%}) of released-firm liability is") W("retained, the impact scales accordingly:") - for _, row in anchor.iterrows(): - impact = float(row["policyengine_impact_m"]) - W(f" {row['year']}: headline {impact:+,.1f}m -> retention-adjusted " - f"{impact * (1 - LLAT_VOLUNTARY_SHARE):+,.1f}m " + ret = anchor_model.anchor_reform(retention=LLAT_VOLUNTARY_SHARE) + for (_, row), (_, r2) in zip(anchor.iterrows(), ret.iterrows()): + W(f" {row['year']}: headline {float(row['policyengine_impact_m']):+,.1f}m -> " + f"retention-adjusted {float(r2['policyengine_impact_m']):+,.1f}m " f"(HMRC {float(row['hmrc_impact_m']):+,.0f}m)") W("") + W("Fixed-preference sensitivity: baseline voluntary registrants keep their") + W("registration wherever the threshold moves (treats the frame's ~89% below-") + W("threshold registered share as revealed preference; frame-selection bias") + W("makes this a lower bound on the revenue loss):") + for _, row in anchor_model.anchor_reform(retain_voluntary=True).iterrows(): + W(f" {row['year']}: fixed-preference {float(row['policyengine_impact_m']):+,.1f}m") + W("") W("Threshold sweep (2024-25 vintage, GBP 90k baseline, 2025-26 fiscal year)") - W("method: direct mechanical reclassification on the calibrated population") + W("method: direct mechanical reclassification of in-scope VAT firms; voluntary") + W("registrants below the data-year threshold held registered, firms aged across") + W("it released unless gap-protected; turnover and liability aged together") W("-" * 74) sweep = sweep_model.threshold_sweep(year="2025-26") W(sweep.to_string(index=False)) @@ -57,10 +86,10 @@ def main() -> None: W("-" * 74) for vintage, model in (("2023-24", anchor_model), ("2024-25", sweep_model)): df = model._aged(1.0) - thr = 85_000.0 if vintage == "2023-24" else 90_000.0 - reg = df["turnover"] >= thr - base = float((df.loc[reg, "liab"] * df.loc[reg, "weight"]).sum()) / 1e9 - W(f" {vintage}: registered base (>= threshold, unaged) = {base:.1f}bn") + base = model._mandatory_base(df, model.data_threshold) / 1e9 + vol = (model._revenue(df, model.data_threshold) - base * 1e9) / 1e9 + W(f" {vintage}: in-scope base (>= threshold, unaged) = {base:.1f}bn; " + f"model-implied voluntary below-threshold remittance = {vol:.1f}bn (not calibrated)") W("") W(f"Sweep thresholds: {SWEEP_THRESHOLDS}") diff --git a/data/README.md b/data/README.md index fc74662..8e385d1 100644 --- a/data/README.md +++ b/data/README.md @@ -49,7 +49,19 @@ threshold automatically. The default is `2023-24`. The raw workbooks are the authoritative inputs. Everything in `processed/` is a faithful extract of tables inside those workbooks (no modelling applied), kept as -CSV so the package can read them without an Excel engine. +CSV so the package can read them without an Excel engine. The two ONS tables per +vintage are produced by `scripts/etl_ons_tables.py` (run `--check` to verify the +checked CSVs against the workbooks); the HMRC tables are manual extracts of the +named sheets described below. + +### Statistical units and universes + +Both ONS tables count **enterprises** (Table 8: turnover sizebands; Table 3: +employment sizebands), so their per-SIC totals agree to ONS disclosure rounding. +An earlier 2023-24 employment CSV reproduced Table 18 — **local units**, i.e. +individual sites (3,173,660 against 2,724,775 enterprises) — and was replaced in +issue #37. The generator treats the ONS frame as the modelled population and the +HMRC trader tables as its VAT-registered subset (see the top-level README). ### 2024-25 band re-aggregation note @@ -98,8 +110,9 @@ liability cells are negative (net repayment positions). ### `ons_firm_turnover.csv` -Count of enterprises by SIC sector and **turnover band (£ thousands)**, from the -ONS UK Business 2024 workbook. +Count of enterprises by SIC division and **turnover band (£ thousands)**, from +**Table 8** (United Kingdom columns) of the ONS UK Business workbook for the +vintage. - Columns: `SIC Code`, `Description`, then one count column per turnover band, then `Total`. @@ -115,8 +128,9 @@ ONS UK Business 2024 workbook. ### `ons_firm_employment.csv` -Count of enterprises by SIC sector and **employment size band (number of -employees)**, from the ONS UK Business 2024 workbook. +Count of enterprises by SIC division and **employment size band (number of +employees)**, from **Table 3** (United Kingdom columns) of the ONS UK Business +workbook for the vintage. - Columns: `SIC Code`, `Description`, then one count column per employment band, then `Total`. diff --git a/data/processed/2023-24/ons_firm_employment.csv b/data/processed/2023-24/ons_firm_employment.csv index 1fd0a7f..93b4ccc 100644 --- a/data/processed/2023-24/ons_firm_employment.csv +++ b/data/processed/2023-24/ons_firm_employment.csv @@ -1,90 +1,90 @@ SIC Code,Description,0-4,5-9,10-19,20-49,50-99,100-249,250+,Total -01,Crop and animal production; hunting and related service activities,119405,11525,3095,1195,350,155,90,135815 -02,Forestry and logging,4180,530,250,105,25,5,0,5095 -03,Fishing and aquaculture,3640,345,125,25,10,0,0,4145 -05,Mining of coal and lignite,5,0,0,0,0,0,0,5 -06,Extraction of crude petroleum and natural gas,70,15,10,15,10,15,15,150 +01,Crop and animal production; hunting and related service activities,116395,11270,3075,1080,345,150,95,132410 +02,Forestry and logging,4105,475,190,70,15,5,5,4865 +03,Fishing and aquaculture,3590,275,95,15,5,5,5,3990 +05,Mining of coal and lignite,0,0,0,0,0,0,0,0 +06,Extraction of crude petroleum and natural gas,65,15,5,5,0,10,15,115 07,Mining of metal ores,5,0,0,0,0,0,0,5 -08,Other mining and quarrying,725,300,220,150,45,20,5,1465 -09,Mining support service activities,260,40,25,35,20,15,15,410 -10,Manufacture of food products,5315,1870,1365,1050,495,440,405,10940 -11,Manufacture of beverages,1910,445,315,200,80,60,45,3055 +08,Other mining and quarrying,400,120,80,65,25,15,5,710 +09,Mining support service activities,255,35,15,25,15,10,10,365 +10,Manufacture of food products,5070,1480,1095,850,380,295,250,9420 +11,Manufacture of beverages,1875,405,245,130,65,20,35,2775 12,Manufacture of tobacco products,0,5,0,0,0,0,0,5 -13,Manufacture of textiles,2875,765,455,310,125,60,10,4600 -14,Manufacture of wearing apparel,2830,515,255,160,50,10,5,3825 -15,Manufacture of leather and related products,415,160,65,40,15,5,5,705 -16,Manufacture of wood and of products of wood and cork; except furniture; manufacture of articles of straw and plaiting materials,7955,1395,795,510,175,60,15,10905 -17,Manufacture of paper and paper products,705,190,195,235,135,130,30,1620 -18,Printing and reproduction of recorded media,7245,1535,725,470,150,90,20,10235 -19,Manufacture of coke and refined petroleum products,40,15,10,15,15,10,10,115 -20,Manufacture of chemicals and chemical products,2000,505,350,435,230,160,65,3745 -21,Manufacture of basic pharmaceutical products and pharmaceutical preparations,400,75,40,75,50,40,55,735 -22,Manufacture of rubber and plastic products,2460,1095,835,805,415,265,65,5940 -23,Manufacture of other non-metallic mineral products,2800,925,615,465,215,130,30,5180 -24,Manufacture of basic metals,1070,275,235,250,105,65,35,2035 -25,Manufacture of fabricated metal products; except machinery and equipment,13980,3870,2855,2145,715,280,70,23915 -26,Manufacture of computer; electronic and optical products,3335,850,560,610,265,165,55,5840 -27,Manufacture of electrical equipment,1640,480,420,430,195,120,40,3325 -28,Manufacture of machinery and equipment n.e.c.,4155,1205,865,845,410,225,105,7810 -29,Manufacture of motor vehicles; trailers and semi-trailers,2455,475,295,275,170,125,95,3890 -30,Manufacture of other transport equipment,1430,245,145,135,105,85,95,2240 -31,Manufacture of furniture,4450,1075,650,450,175,95,30,6925 -32,Other manufacturing,7045,1590,655,420,150,90,35,9985 -33,Repair and installation of machinery and equipment,11995,1695,985,580,180,130,50,15615 -35,Electricity; gas; steam and air conditioning supply,5200,730,390,275,145,155,95,6990 -36,Water collection; treatment and supply,375,170,145,165,65,45,40,1005 -37,Sewerage,975,260,175,180,45,35,5,1675 -38,Waste collection; treatment and disposal activities; materials recovery,4630,1480,1065,950,395,220,55,8795 -39,Remediation activities and other waste management services,845,210,135,85,15,5,0,1295 -41,Construction of buildings,110965,9690,4175,1665,625,395,80,127595 -42,Civil engineering,18605,2615,1330,955,415,205,95,24220 -43,Specialised construction activities,203530,21540,8795,4060,1010,375,70,239380 -45,Wholesale and retail trade and repair of motor vehicles and motorcycles,64535,16060,6460,4050,1130,330,50,92615 -46,Wholesale trade; except of motor vehicles and motorcycles,75485,21110,13315,7750,2300,1005,390,121355 -47,Retail trade; except of motor vehicles and motorcycles,185010,58960,35145,16905,3285,2255,1040,302600 -49,Land transport and transport via pipelines,52885,6295,3610,2665,1080,665,320,67520 -50,Water transport,1060,190,115,85,35,10,10,1505 -51,Air transport,950,115,55,70,35,30,50,1305 -52,Warehousing and support activities for transportation,16105,3795,2465,2145,950,680,440,26580 -53,Postal and courier activities,32395,1385,595,925,700,420,125,36545 -55,Accommodation,12930,3600,3375,3925,1435,630,170,26065 -56,Food and beverage service activities,91715,47075,32560,20325,2915,1345,135,196070 -58,Publishing activities,9985,1205,755,525,180,90,70,12810 -59,Motion picture; video and television programme production; sound recording and music publishing activities,30560,1500,900,745,270,95,30,34100 -60,Programming and broadcasting activities,1885,165,80,100,35,30,25,2320 -61,Telecommunications,7520,1440,1005,795,350,195,135,11440 -62,Computer programming; consultancy and related activities,107640,7760,5010,3550,1350,680,350,126340 -63,Information service activities,6740,830,575,325,115,70,55,8710 -64,Financial service activities; except insurance and pension funding,19050,3625,2290,1140,290,190,300,26885 -65,Insurance; reinsurance and pension funding; except compulsory social security,6970,170,130,100,50,60,105,7585 -66,Activities auxiliary to financial services and insurance activities,26920,4315,2270,1395,690,445,280,36315 -68,Real estate activities,107155,14675,6180,1825,475,335,195,130840 -69,Legal and accounting activities,59055,9660,5675,2980,960,585,355,79270 -70,Activities of head offices; management consultancy activities,142325,9000,5030,2955,1180,695,370,161555 -71,Architectural and engineering activities; technical testing and analysis,68340,7175,4330,2720,935,445,180,84125 -72,Scientific research and development,4070,790,660,535,250,165,135,6605 -73,Advertising and market research,19425,1960,1240,900,325,160,110,24120 -74,Other professional; scientific and technical activities,67880,5225,2245,955,220,75,25,76625 -75,Veterinary activities,2770,1460,1610,885,135,40,10,6910 -77,Rental and leasing activities,14710,3585,2190,1115,300,125,35,22060 -78,Employment activities,25925,4335,3385,2715,1715,1265,675,40015 -79,Travel agency; tour operator and other reservation service and related activities,7555,1875,695,350,140,80,30,10725 -80,Security and investigation activities,11060,1925,1175,980,365,205,130,15840 -81,Services to buildings and landscape activities,54515,9800,5185,2885,980,485,340,74190 -82,Office administrative; office support and other business support activities,89580,9640,4400,1950,760,420,295,107045 -84,Public administration and defence; compulsory social security,10920,3080,2885,3200,1615,1370,1325,24395 -85,Education,31990,8755,8850,14750,8445,4670,935,78395 -86,Human health activities,49340,11185,10020,7510,2355,1430,1050,82890 -87,Residential care activities,9530,5460,6725,7160,3885,870,85,33715 -88,Social work activities without accommodation,20760,12015,11720,8555,2270,845,255,56420 -90,Creative; arts and entertainment activities,29555,1665,820,405,170,100,20,32735 -91,Libraries; archives; museums and other cultural activities,2775,1205,835,625,255,125,40,5860 -92,Gambling and betting activities,4085,4160,500,320,95,40,10,9210 -93,Sports activities and amusement and recreation activities,27855,6455,4600,3650,1330,515,130,44535 -94,Activities of membership organisations,17845,4915,2240,1285,415,225,85,27010 -95,Repair of computers and personal and household goods,8545,875,345,210,55,40,20,10090 -96,Other personal service activities,69185,13545,4365,1170,195,95,25,88580 +13,Manufacture of textiles,2760,705,415,280,115,50,20,4345 +14,Manufacture of wearing apparel,2780,500,235,150,50,5,5,3725 +15,Manufacture of leather and related products,405,85,45,30,15,10,5,595 +16,Manufacture of wood and of products of wood and cork; except furniture; manufacture of articles of straw and plaiting materials,7840,1325,705,425,140,55,25,10515 +17,Manufacture of paper and paper products,670,175,165,180,105,65,35,1395 +18,Printing and reproduction of recorded media,7070,1460,660,425,125,70,30,9840 +19,Manufacture of coke and refined petroleum products,35,10,0,15,10,5,10,85 +20,Manufacture of chemicals and chemical products,1885,420,270,315,190,120,70,3270 +21,Manufacture of basic pharmaceutical products and pharmaceutical preparations,385,65,35,50,30,35,50,650 +22,Manufacture of rubber and plastic products,2220,970,705,665,340,230,90,5220 +23,Manufacture of other non-metallic mineral products,2225,740,450,325,140,80,40,4000 +24,Manufacture of basic metals,1030,245,185,195,90,60,30,1835 +25,Manufacture of fabricated metal products; except machinery and equipment,13510,3615,2565,1905,640,275,85,22595 +26,Manufacture of computer; electronic and optical products,3205,775,470,500,230,145,60,5385 +27,Manufacture of electrical equipment,1565,420,360,370,180,90,45,3030 +28,Manufacture of machinery and equipment n.e.c.,3965,1080,720,715,365,195,100,7140 +29,Manufacture of motor vehicles; trailers and semi-trailers,2380,440,255,185,135,105,75,3575 +30,Manufacture of other transport equipment,1365,220,100,80,70,45,65,1945 +31,Manufacture of furniture,4360,1035,600,405,170,85,30,6685 +32,Other manufacturing,6880,1525,590,350,135,80,40,9600 +33,Repair and installation of machinery and equipment,11735,1505,795,390,125,75,60,14685 +35,Electricity; gas; steam and air conditioning supply,4935,600,280,90,30,25,40,6000 +36,Water collection; treatment and supply,35,10,5,20,5,5,20,100 +37,Sewerage,820,195,90,90,20,5,5,1225 +38,Waste collection; treatment and disposal activities; materials recovery,3850,925,635,480,160,90,65,6205 +39,Remediation activities and other waste management services,825,180,110,75,15,5,0,1210 +41,Construction of buildings,108170,8375,3395,1200,390,185,100,121815 +42,Civil engineering,18025,2260,1025,660,240,115,85,22410 +43,Specialised construction activities,202175,20890,8145,3530,790,330,115,235975 +45,Wholesale and retail trade and repair of motor vehicles and motorcycles,61960,13180,4025,1540,450,260,215,81630 +46,Wholesale trade; except of motor vehicles and motorcycles,69275,14885,9145,5425,1865,870,505,101970 +47,Retail trade; except of motor vehicles and motorcycles,160890,33340,12820,5340,1250,505,475,214620 +49,Land transport and transport via pipelines,51590,5720,2960,1855,595,275,180,63175 +50,Water transport,1040,165,75,60,25,10,10,1385 +51,Air transport,880,60,30,30,15,15,20,1050 +52,Warehousing and support activities for transportation,12865,2075,1050,795,380,255,170,17590 +53,Postal and courier activities,31870,1125,380,180,65,25,30,33675 +55,Accommodation,12050,2985,2245,2220,935,420,205,21060 +56,Food and beverage service activities,79940,37340,22445,10235,1475,520,500,152455 +58,Publishing activities,9850,1140,685,460,155,70,55,12415 +59,Motion picture; video and television programme production; sound recording and music publishing activities,30420,1425,790,385,170,75,40,33305 +60,Programming and broadcasting activities,1840,130,45,40,10,15,15,2095 +61,Telecommunications,6340,610,375,320,145,95,70,7955 +62,Computer programming; consultancy and related activities,106880,7365,4555,3065,1110,520,295,123790 +63,Information service activities,6545,720,490,265,90,45,45,8200 +64,Financial service activities; except insurance and pension funding,17250,1230,465,320,150,115,155,19685 +65,Insurance; reinsurance and pension funding; except compulsory social security,6910,140,85,50,25,20,60,7290 +66,Activities auxiliary to financial services and insurance activities,24790,3525,1675,855,475,270,220,31810 +68,Real estate activities,100060,11320,4480,1225,350,225,235,117895 +69,Legal and accounting activities,56980,8050,4240,2235,750,445,300,73000 +70,Activities of head offices; management consultancy activities,140890,8230,4285,2180,760,330,190,156865 +71,Architectural and engineering activities; technical testing and analysis,66935,6205,3330,1930,650,295,185,79530 +72,Scientific research and development,3890,675,545,380,160,95,105,5850 +73,Advertising and market research,19245,1870,1130,760,295,130,115,23545 +74,Other professional; scientific and technical activities,67090,5000,2110,850,210,70,35,75365 +75,Veterinary activities,1945,605,530,260,75,25,15,3455 +77,Rental and leasing activities,13585,2355,1265,675,225,120,70,18295 +78,Employment activities,21535,3375,2655,1845,1090,640,485,31625 +79,Travel agency; tour operator and other reservation service and related activities,6965,955,410,265,115,70,35,8815 +80,Security and investigation activities,7915,1085,675,515,245,145,85,10665 +81,Services to buildings and landscape activities,40085,6685,3340,1600,600,335,315,52960 +82,Office administrative; office support and other business support activities,88460,9155,4025,1590,570,285,225,104310 +84,Public administration and defence; compulsory social security,6435,550,210,130,45,55,415,7840 +85,Education,29835,6215,4190,2810,1630,1650,1670,48000 +86,Human health activities,45015,7795,6345,4195,1200,385,495,65430 +87,Residential care activities,3190,1255,1875,2905,1520,690,425,11860 +88,Social work activities without accommodation,15240,7775,6755,4235,1515,640,330,36490 +90,Creative; arts and entertainment activities,29445,1615,750,330,120,75,25,32360 +91,Libraries; archives; museums and other cultural activities,915,300,230,210,135,80,55,1925 +92,Gambling and betting activities,375,190,130,90,35,30,30,880 +93,Sports activities and amusement and recreation activities,26655,5370,3380,2290,600,290,225,38810 +94,Activities of membership organisations,15345,4125,1725,905,275,160,95,22630 +95,Repair of computers and personal and household goods,8265,820,285,150,40,20,15,9595 +96,Other personal service activities,64505,12325,3740,1040,200,70,25,81905 97,Activities of households as employers of domestic personnel,0,0,0,0,0,0,0,0 98,Undifferentiated goods- and services-producing activities of private households for own use,0,0,0,0,0,0,0,0 -99,Activities of extraterritorial organisations and bodies,0,0,0,0,5,0,0,5 -,Total,2269010,404720,242235,160880,54820,29115,12880,3173660 +99,Activities of extraterritorial organisations and bodies,0,0,0,0,0,0,0,0 +,Total,2122850,305890,155785,85380,28695,14885,11285,2724770 diff --git a/data/processed/2024-25/ons_firm_employment.csv b/data/processed/2024-25/ons_firm_employment.csv index c2cea20..27f5633 100644 --- a/data/processed/2024-25/ons_firm_employment.csv +++ b/data/processed/2024-25/ons_firm_employment.csv @@ -1,90 +1,90 @@ SIC Code,Description,0-4,5-9,10-19,20-49,50-99,100-249,250+,Total -01,Crop and animal production; hunting and related service activities,116500,10560,3175,1090,355,140,110,131930 -02,Forestry and logging,4230,470,180,65,20,0,5,4970 -03,Fishing and aquaculture,3535,260,100,15,5,0,5,3920 +01,Crop and animal production; hunting and related service activities,116525,10565,3155,1090,360,150,100,131945 +02,Forestry and logging,4235,470,175,65,15,5,5,4970 +03,Fishing and aquaculture,3535,255,105,15,5,5,5,3925 05,Mining of coal and lignite,5,0,0,0,0,0,0,5 -06,Extraction of crude petroleum and natural gas,70,15,5,5,0,5,15,115 -07,Mining of metal ores,5,0,0,0,0,0,0,5 -08,Other mining and quarrying,375,120,85,50,20,15,15,680 -09,Mining support service activities,250,35,15,25,15,5,15,360 -10,Manufacture of food products,5070,1470,1105,870,360,315,245,9435 -11,Manufacture of beverages,1780,390,265,145,65,25,30,2700 +06,Extraction of crude petroleum and natural gas,70,10,5,5,0,10,15,115 +07,Mining of metal ores,0,0,0,0,0,0,0,0 +08,Other mining and quarrying,375,120,85,55,25,15,10,685 +09,Mining support service activities,250,35,15,25,15,5,10,355 +10,Manufacture of food products,5070,1480,1095,870,370,300,250,9435 +11,Manufacture of beverages,1780,385,260,145,65,25,35,2695 12,Manufacture of tobacco products,0,5,0,0,0,0,0,5 -13,Manufacture of textiles,2725,695,405,270,110,40,20,4265 -14,Manufacture of wearing apparel,2660,455,215,130,45,10,0,3515 -15,Manufacture of leather and related products,370,80,40,30,15,5,10,550 -16,Manufacture of wood and of products of wood and cork; except furniture; manufacture of articles of straw and plaiting materials,7795,1280,650,415,140,50,30,10360 -17,Manufacture of paper and paper products,650,155,160,180,100,70,25,1340 -18,Printing and reproduction of recorded media,6905,1410,615,390,120,70,25,9535 -19,Manufacture of coke and refined petroleum products,30,10,0,15,10,10,10,85 -20,Manufacture of chemicals and chemical products,1890,415,270,295,195,130,65,3260 -21,Manufacture of basic pharmaceutical products and pharmaceutical preparations,370,60,30,45,35,40,50,630 -22,Manufacture of rubber and plastic products,2240,935,660,630,365,225,90,5145 -23,Manufacture of other non-metallic mineral products,2270,705,440,315,115,100,40,3985 -24,Manufacture of basic metals,1040,235,190,195,80,45,40,1825 -25,Manufacture of fabricated metal products; except machinery and equipment,12900,3530,2490,1810,650,240,75,21695 -26,Manufacture of computer; electronic and optical products,3165,750,465,475,235,140,55,5285 -27,Manufacture of electrical equipment,1565,425,350,365,170,95,45,3015 -28,Manufacture of machinery and equipment n.e.c.,3920,1055,725,705,345,190,100,7040 -29,Manufacture of motor vehicles; trailers and semi-trailers,2395,420,240,185,135,100,70,3545 -30,Manufacture of other transport equipment,1330,215,90,75,65,50,65,1890 -31,Manufacture of furniture,4375,995,580,380,160,85,25,6600 -32,Other manufacturing,6850,1435,590,330,140,75,35,9455 -33,Repair and installation of machinery and equipment,12045,1500,800,385,125,85,55,14995 -35,Electricity; gas; steam and air conditioning supply,5845,550,275,90,35,20,45,6860 +13,Manufacture of textiles,2725,690,405,275,110,45,20,4270 +14,Manufacture of wearing apparel,2655,455,215,130,45,10,5,3515 +15,Manufacture of leather and related products,370,75,40,30,15,5,5,540 +16,Manufacture of wood and of products of wood and cork; except furniture; manufacture of articles of straw and plaiting materials,7795,1280,655,415,145,50,25,10365 +17,Manufacture of paper and paper products,645,160,165,180,105,70,30,1355 +18,Printing and reproduction of recorded media,6905,1415,615,385,120,70,30,9540 +19,Manufacture of coke and refined petroleum products,30,10,0,15,10,5,10,80 +20,Manufacture of chemicals and chemical products,1885,415,275,295,190,125,70,3255 +21,Manufacture of basic pharmaceutical products and pharmaceutical preparations,370,60,30,45,35,35,55,630 +22,Manufacture of rubber and plastic products,2250,935,665,630,355,225,90,5150 +23,Manufacture of other non-metallic mineral products,2270,705,450,325,125,80,40,3995 +24,Manufacture of basic metals,1035,230,180,200,85,60,35,1825 +25,Manufacture of fabricated metal products; except machinery and equipment,12895,3535,2490,1810,635,255,80,21700 +26,Manufacture of computer; electronic and optical products,3165,740,465,480,225,145,60,5280 +27,Manufacture of electrical equipment,1565,430,345,355,175,90,50,3010 +28,Manufacture of machinery and equipment n.e.c.,3920,1065,720,705,345,195,100,7050 +29,Manufacture of motor vehicles; trailers and semi-trailers,2400,420,235,190,135,105,70,3555 +30,Manufacture of other transport equipment,1320,215,95,80,65,50,65,1890 +31,Manufacture of furniture,4370,995,585,380,160,85,25,6600 +32,Other manufacturing,6855,1430,595,320,140,80,35,9455 +33,Repair and installation of machinery and equipment,12045,1505,800,380,125,80,60,14995 +35,Electricity; gas; steam and air conditioning supply,5845,550,275,90,35,25,40,6860 36,Water collection; treatment and supply,45,15,5,15,10,5,20,115 -37,Sewerage,845,200,100,90,20,5,5,1265 -38,Waste collection; treatment and disposal activities; materials recovery,3915,905,625,440,165,95,70,6215 -39,Remediation activities and other waste management services,845,180,110,70,25,5,0,1235 -41,Construction of buildings,109235,8140,3165,1185,400,185,95,122405 -42,Civil engineering,18090,2200,1010,640,235,130,80,22385 -43,Specialised construction activities,206250,20775,8115,3615,820,335,120,240030 -45,Wholesale and retail trade and repair of motor vehicles and motorcycles,63020,13245,4150,1530,450,255,210,82860 -46,Wholesale trade; except of motor vehicles and motorcycles,68380,14600,9120,5370,1870,865,510,100715 -47,Retail trade; except of motor vehicles and motorcycles,159710,32790,12915,5305,1260,475,455,212910 -49,Land transport and transport via pipelines,50575,5235,2910,1755,620,265,185,61545 -50,Water transport,1135,175,75,55,20,10,10,1480 -51,Air transport,810,50,30,25,20,20,15,970 -52,Warehousing and support activities for transportation,12600,1765,1015,805,395,245,190,17015 -53,Postal and courier activities,31625,1100,365,170,60,30,25,33375 -55,Accommodation,12500,2930,2245,2155,965,445,210,21450 -56,Food and beverage service activities,81960,37480,23170,10110,1515,520,485,155240 -58,Publishing activities,9940,1150,680,440,165,70,55,12500 -59,Motion picture; video and television programme production; sound recording and music publishing activities,31545,1385,750,380,165,85,45,34355 +37,Sewerage,850,200,100,90,20,5,0,1265 +38,Waste collection; treatment and disposal activities; materials recovery,3915,905,620,445,165,90,70,6210 +39,Remediation activities and other waste management services,845,180,110,70,20,0,0,1225 +41,Construction of buildings,109235,8145,3165,1180,400,185,100,122410 +42,Civil engineering,18095,2200,1015,640,235,120,80,22385 +43,Specialised construction activities,206260,20775,8120,3605,820,330,125,240035 +45,Wholesale and retail trade and repair of motor vehicles and motorcycles,63025,13245,4155,1530,445,255,210,82865 +46,Wholesale trade; except of motor vehicles and motorcycles,68380,14600,9140,5375,1865,855,510,100725 +47,Retail trade; except of motor vehicles and motorcycles,159695,32800,12910,5295,1265,480,455,212900 +49,Land transport and transport via pipelines,50575,5240,2910,1755,620,265,185,61550 +50,Water transport,1130,170,75,55,20,10,10,1470 +51,Air transport,810,50,30,25,15,20,15,965 +52,Warehousing and support activities for transportation,12595,1775,1015,805,400,240,190,17020 +53,Postal and courier activities,31630,1095,365,170,60,30,25,33375 +55,Accommodation,12500,2935,2240,2150,960,445,210,21440 +56,Food and beverage service activities,81960,37485,23170,10110,1515,520,490,155250 +58,Publishing activities,9940,1160,675,440,155,75,55,12500 +59,Motion picture; video and television programme production; sound recording and music publishing activities,31550,1380,750,380,165,85,40,34350 60,Programming and broadcasting activities,1930,130,45,35,10,10,20,2180 -61,Telecommunications,6250,595,355,325,140,105,75,7845 -62,Computer programming; consultancy and related activities,105530,7495,4530,3085,1165,505,320,122630 -63,Information service activities,6515,745,495,245,105,45,45,8195 -64,Financial service activities; except insurance and pension funding,19900,1230,465,335,175,110,160,22375 -65,Insurance; reinsurance and pension funding; except compulsory social security,5010,130,85,60,20,20,60,5385 -66,Activities auxiliary to financial services and insurance activities,21885,3435,1630,825,460,260,235,28730 -68,Real estate activities,104180,11490,4580,1255,345,220,240,122310 -69,Legal and accounting activities,57555,8085,4205,2190,760,425,305,73525 -70,Activities of head offices; management consultancy activities,142800,8180,4385,2245,800,340,195,158945 -71,Architectural and engineering activities; technical testing and analysis,66965,6195,3265,1935,685,310,190,79545 +61,Telecommunications,6240,595,360,330,140,105,75,7845 +62,Computer programming; consultancy and related activities,105525,7495,4525,3090,1170,510,320,122635 +63,Information service activities,6510,740,495,250,100,50,45,8190 +64,Financial service activities; except insurance and pension funding,19900,1225,465,340,175,105,160,22370 +65,Insurance; reinsurance and pension funding; except compulsory social security,5010,135,85,60,20,25,55,5390 +66,Activities auxiliary to financial services and insurance activities,21885,3435,1630,820,465,260,230,28725 +68,Real estate activities,104180,11490,4580,1245,350,220,245,122310 +69,Legal and accounting activities,57555,8080,4210,2190,755,425,305,73520 +70,Activities of head offices; management consultancy activities,142800,8175,4385,2245,805,340,195,158945 +71,Architectural and engineering activities; technical testing and analysis,66970,6200,3265,1935,685,310,185,79550 72,Scientific research and development,3875,730,530,395,165,95,105,5895 -73,Advertising and market research,19865,1820,1115,755,315,135,105,24110 -74,Other professional; scientific and technical activities,65785,4775,2130,885,240,70,30,73915 +73,Advertising and market research,19870,1820,1110,755,310,130,105,24100 +74,Other professional; scientific and technical activities,65785,4775,2135,885,235,70,30,73915 75,Veterinary activities,2060,610,565,275,75,30,15,3630 -77,Rental and leasing activities,13720,2275,1255,660,220,120,80,18330 -78,Employment activities,21825,3270,2665,1790,1115,655,445,31765 -79,Travel agency; tour operator and other reservation service and related activities,7395,975,425,265,125,70,35,9290 -80,Security and investigation activities,8235,1070,720,505,235,160,90,11015 -81,Services to buildings and landscape activities,40945,6675,3305,1550,610,325,315,53725 -82,Office administrative; office support and other business support activities,83570,7815,3845,1595,610,290,230,97955 -84,Public administration and defence; compulsory social security,6535,560,205,135,45,50,425,7955 -85,Education,30855,6175,4345,2785,1650,1575,1710,49095 -86,Human health activities,47895,7860,6485,4205,1285,420,505,68655 -87,Residential care activities,3295,1300,1970,2930,1600,695,435,12225 -88,Social work activities without accommodation,15180,7615,6850,4200,1520,640,355,36360 -90,Creative; arts and entertainment activities,30330,1655,740,340,135,80,25,33305 +77,Rental and leasing activities,13720,2270,1250,660,235,115,70,18320 +78,Employment activities,21820,3270,2665,1790,1115,660,445,31765 +79,Travel agency; tour operator and other reservation service and related activities,7400,975,425,265,120,70,35,9290 +80,Security and investigation activities,8235,1070,720,505,240,160,90,11020 +81,Services to buildings and landscape activities,40950,6675,3300,1550,610,325,315,53725 +82,Office administrative; office support and other business support activities,83560,7815,3850,1590,615,290,230,97950 +84,Public administration and defence; compulsory social security,6535,555,210,135,45,55,430,7965 +85,Education,30860,6175,4350,2795,1640,1575,1705,49100 +86,Human health activities,47900,7860,6480,4210,1285,415,500,68650 +87,Residential care activities,3295,1295,1970,2920,1605,700,435,12220 +88,Social work activities without accommodation,15180,7615,6850,4200,1520,640,350,36355 +90,Creative; arts and entertainment activities,30325,1650,740,340,135,85,30,33305 91,Libraries; archives; museums and other cultural activities,930,310,230,215,130,90,55,1960 -92,Gambling and betting activities,380,175,120,90,40,30,30,865 -93,Sports activities and amusement and recreation activities,27415,5435,3455,2295,605,290,235,39730 -94,Activities of membership organisations,15295,4030,1770,895,295,145,100,22530 -95,Repair of computers and personal and household goods,8260,810,285,150,40,15,15,9575 -96,Other personal service activities,66855,12040,3775,1015,205,80,30,84000 +92,Gambling and betting activities,380,175,120,90,35,30,30,860 +93,Sports activities and amusement and recreation activities,27415,5435,3450,2300,610,290,235,39735 +94,Activities of membership organisations,15295,4025,1770,900,290,145,100,22525 +95,Repair of computers and personal and household goods,8260,805,285,150,40,15,15,9570 +96,Other personal service activities,66855,12040,3775,1010,215,70,35,84000 97,Activities of households as employers of domestic personnel,0,0,0,0,0,0,0,0 98,Undifferentiated goods- and services-producing activities of private households for own use,0,0,0,0,0,0,0,0 99,Activities of extraterritorial organisations and bodies,0,0,0,0,0,0,0,0 -,Total,2137200,300645,156590,84595,29335,14835,11415,2734615 +,Total,2137205,300645,156590,84595,29335,14835,11415,2734620 diff --git a/data/processed/2024-25/ons_firm_turnover.csv b/data/processed/2024-25/ons_firm_turnover.csv index 5433919..4fd13a0 100644 --- a/data/processed/2024-25/ons_firm_turnover.csv +++ b/data/processed/2024-25/ons_firm_turnover.csv @@ -1,89 +1,89 @@ SIC Code,Description,0-49,50-99,100-249,250-499,500-999,1000-4999,5000+,Total -01,Crop and animal production; hunting and related service activities,41610,24100,29865,16365,10680,8240,1070,131930 -02,Forestry and logging,855,1060,1705,685,340,275,50,4970 -03,Fishing and aquaculture,1180,750,1010,490,215,230,45,3920 -05,Mining of coal and lignite,0,0,0,0,0,5,0,5 -06,Extraction of crude petroleum and natural gas,20,10,20,5,10,10,40,115 -07,Mining of metal ores,5,0,0,0,0,0,0,5 -08,Other mining and quarrying,80,50,120,80,75,150,120,675 -09,Mining support service activities,55,25,120,25,15,45,70,355 -10,Manufacture of food products,1480,1215,2180,1215,880,1225,1230,9425 -11,Manufacture of beverages,630,405,635,315,275,275,160,2695 -12,Manufacture of tobacco products,0,0,0,0,5,0,0,5 -13,Manufacture of textiles,465,805,1255,595,410,515,220,4265 -14,Manufacture of wearing apparel,555,670,1315,400,260,240,75,3515 -15,Manufacture of leather and related products,110,120,125,60,40,70,25,550 -16,Manufacture of wood and of products of wood and cork; except furniture; manufacture of articles of straw and plaiting materials,715,2495,3520,1310,945,1025,350,10360 -17,Manufacture of paper and paper products,155,170,215,115,110,265,325,1355 -18,Printing and reproduction of recorded media,1615,1980,2685,1255,800,915,285,9535 -19,Manufacture of coke and refined petroleum products,10,5,10,5,5,5,45,85 -20,Manufacture of chemicals and chemical products,480,455,595,320,270,505,635,3260 -21,Manufacture of basic pharmaceutical products and pharmaceutical preparations,85,70,120,75,50,80,150,630 -22,Manufacture of rubber and plastic products,400,420,865,650,675,1260,875,5145 -23,Manufacture of other non-metallic mineral products,255,540,1015,555,490,720,410,3985 -24,Manufacture of basic metals,155,235,430,200,185,325,290,1820 -25,Manufacture of fabricated metal products; except machinery and equipment,1880,3975,4940,2950,2530,3920,1500,21695 -26,Manufacture of computer; electronic and optical products,840,675,1185,570,485,895,635,5285 -27,Manufacture of electrical equipment,355,310,595,330,305,640,480,3015 -28,Manufacture of machinery and equipment n.e.c.,775,770,1500,840,750,1405,1005,7045 -29,Manufacture of motor vehicles; trailers and semi-trailers,310,480,1145,485,295,450,380,3545 -30,Manufacture of other transport equipment,270,420,460,215,130,195,200,1890 -31,Manufacture of furniture,485,1150,2050,960,765,875,315,6600 -32,Other manufacturing,1530,1850,2645,1240,925,915,360,9465 -33,Repair and installation of machinery and equipment,1495,3645,4790,1890,1280,1430,465,14995 -35,Electricity; gas; steam and air conditioning supply,2010,620,1265,820,720,960,465,6860 -36,Water collection; treatment and supply,10,10,10,10,10,25,40,115 -37,Sewerage,95,205,340,220,175,175,55,1265 -38,Waste collection; treatment and disposal activities; materials recovery,495,610,1610,960,720,1130,695,6220 -39,Remediation activities and other waste management services,120,185,310,165,155,230,70,1235 -41,Construction of buildings,15220,13905,27845,29025,20730,12540,3140,122405 -42,Civil engineering,2240,2985,7795,3270,2205,2605,1290,22390 -43,Specialised construction activities,16140,53415,92710,35425,19980,18225,4135,240030 -45,Wholesale and retail trade and repair of motor vehicles and motorcycles,5640,12715,26180,16060,9815,9260,3190,82860 -46,Wholesale trade; except of motor vehicles and motorcycles,12420,9890,18860,14185,12835,20465,12075,100730 -47,Retail trade; except of motor vehicles and motorcycles,20895,28235,65640,42860,28420,22340,4520,212910 -49,Land transport and transport via pipelines,9215,14185,21465,6170,4185,4795,1530,61545 -50,Water transport,405,235,295,130,115,160,140,1480 +01,Crop and animal production; hunting and related service activities,41615,24105,29875,16355,10675,8245,1085,131955 +02,Forestry and logging,850,1065,1705,680,345,270,50,4965 +03,Fishing and aquaculture,1180,750,1015,490,215,220,50,3920 +05,Mining of coal and lignite,0,0,0,0,0,0,0,0 +06,Extraction of crude petroleum and natural gas,20,10,20,10,5,10,40,115 +07,Mining of metal ores,0,0,0,0,0,0,0,0 +08,Other mining and quarrying,85,55,125,85,75,150,110,685 +09,Mining support service activities,55,25,120,25,15,40,75,355 +10,Manufacture of food products,1475,1215,2180,1220,890,1235,1220,9435 +11,Manufacture of beverages,630,405,640,315,270,275,155,2690 +12,Manufacture of tobacco products,0,0,0,0,0,0,0,0 +13,Manufacture of textiles,470,810,1250,600,415,510,215,4270 +14,Manufacture of wearing apparel,550,675,1310,405,260,240,75,3515 +15,Manufacture of leather and related products,105,120,130,60,40,60,30,545 +16,Manufacture of wood and of products of wood and cork; except furniture; manufacture of articles of straw and plaiting materials,715,2490,3515,1305,950,1030,360,10365 +17,Manufacture of paper and paper products,160,170,220,115,115,260,315,1355 +18,Printing and reproduction of recorded media,1620,1980,2685,1255,805,900,290,9535 +19,Manufacture of coke and refined petroleum products,5,0,10,5,5,5,45,75 +20,Manufacture of chemicals and chemical products,480,450,600,330,275,500,625,3260 +21,Manufacture of basic pharmaceutical products and pharmaceutical preparations,90,65,125,75,45,80,150,630 +22,Manufacture of rubber and plastic products,395,425,865,650,675,1260,880,5150 +23,Manufacture of other non-metallic mineral products,265,540,1010,560,475,725,415,3990 +24,Manufacture of basic metals,160,230,425,205,190,325,285,1820 +25,Manufacture of fabricated metal products; except machinery and equipment,1880,3970,4940,2955,2535,3915,1505,21700 +26,Manufacture of computer; electronic and optical products,845,675,1190,560,480,890,640,5280 +27,Manufacture of electrical equipment,355,305,595,330,305,640,480,3010 +28,Manufacture of machinery and equipment n.e.c.,780,775,1505,845,760,1395,995,7055 +29,Manufacture of motor vehicles; trailers and semi-trailers,315,480,1140,485,300,445,385,3550 +30,Manufacture of other transport equipment,270,425,455,215,130,195,205,1895 +31,Manufacture of furniture,485,1150,2050,960,765,870,315,6595 +32,Other manufacturing,1520,1845,2650,1240,925,920,360,9460 +33,Repair and installation of machinery and equipment,1495,3650,4790,1895,1280,1425,465,15000 +35,Electricity; gas; steam and air conditioning supply,2015,615,1260,820,720,965,460,6855 +36,Water collection; treatment and supply,10,5,10,10,10,25,40,110 +37,Sewerage,95,205,340,220,175,170,55,1260 +38,Waste collection; treatment and disposal activities; materials recovery,490,610,1615,960,720,1130,690,6215 +39,Remediation activities and other waste management services,120,185,310,165,155,235,70,1240 +41,Construction of buildings,15220,13910,27845,29020,20730,12540,3140,122405 +42,Civil engineering,2240,2990,7795,3265,2205,2605,1285,22385 +43,Specialised construction activities,16145,53410,92715,35415,19975,18240,4135,240035 +45,Wholesale and retail trade and repair of motor vehicles and motorcycles,5635,12715,26180,16065,9815,9265,3190,82865 +46,Wholesale trade; except of motor vehicles and motorcycles,12430,9890,18865,14190,12825,20465,12055,100720 +47,Retail trade; except of motor vehicles and motorcycles,20890,28235,65610,42860,28430,22340,4535,212900 +49,Land transport and transport via pipelines,9220,14180,21460,6170,4185,4800,1530,61545 +50,Water transport,400,235,300,135,110,160,140,1480 51,Air transport,165,75,110,270,130,105,115,970 -52,Warehousing and support activities for transportation,2030,3295,3620,1840,2075,2580,1575,17015 -53,Postal and courier activities,19705,6480,4305,1295,715,675,200,33375 -55,Accommodation,1950,3180,6520,3045,2390,3250,1115,21450 -56,Food and beverage service activities,9870,26495,63780,30000,15630,8190,1275,155240 -58,Publishing activities,2665,2450,3630,1500,855,985,415,12500 -59,Motion picture; video and television programme production; sound recording and music publishing activities,5970,9010,12455,2895,1455,1770,800,34355 -60,Programming and broadcasting activities,330,490,825,240,120,105,70,2180 -61,Telecommunications,1115,1060,1900,1430,870,920,550,7845 -62,Computer programming; consultancy and related activities,20260,32990,44455,9100,5620,7040,3170,122635 -63,Information service activities,1450,2385,2145,725,585,600,305,8195 -64,Financial service activities; except insurance and pension funding,12710,1055,3130,2365,1095,1060,960,22375 -65,Insurance; reinsurance and pension funding; except compulsory social security,4520,20,125,185,135,185,215,5385 -66,Activities auxiliary to financial services and insurance activities,5945,1095,10525,4065,2825,2620,1655,28730 +52,Warehousing and support activities for transportation,2025,3290,3625,1845,2080,2575,1575,17015 +53,Postal and courier activities,19705,6480,4305,1290,715,680,200,33375 +55,Accommodation,1950,3180,6520,3040,2390,3250,1115,21445 +56,Food and beverage service activities,9870,26495,63775,30000,15630,8195,1280,155245 +58,Publishing activities,2660,2450,3630,1505,855,980,420,12500 +59,Motion picture; video and television programme production; sound recording and music publishing activities,5970,9015,12450,2890,1460,1760,800,34345 +60,Programming and broadcasting activities,330,485,825,240,120,105,70,2175 +61,Telecommunications,1110,1065,1900,1430,870,925,550,7850 +62,Computer programming; consultancy and related activities,20255,32990,44450,9100,5625,7045,3170,122635 +63,Information service activities,1450,2380,2150,725,580,600,305,8190 +64,Financial service activities; except insurance and pension funding,12700,1050,3135,2365,1095,1065,965,22375 +65,Insurance; reinsurance and pension funding; except compulsory social security,4515,25,125,185,140,190,210,5390 +66,Activities auxiliary to financial services and insurance activities,5950,1095,10520,4060,2820,2625,1655,28725 68,Real estate activities,25260,29645,35000,15550,8450,6530,1875,122310 -69,Legal and accounting activities,9300,15735,26315,10065,5595,4985,1530,73525 -70,Activities of head offices; management consultancy activities,26200,27725,67425,18780,8715,7495,2610,158950 -71,Architectural and engineering activities; technical testing and analysis,8765,21490,29505,8060,4575,5190,1960,79545 -72,Scientific research and development,1470,900,1450,620,430,595,430,5895 -73,Advertising and market research,3305,3685,9305,3055,1770,2060,930,24110 -74,Other professional; scientific and technical activities,9530,18980,31535,6615,3510,2995,750,73915 -75,Veterinary activities,285,780,815,400,510,740,100,3630 -77,Rental and leasing activities,3030,2600,4195,2825,2100,2560,1015,18325 -78,Employment activities,3385,4210,9305,4535,3325,4855,2150,31765 -79,Travel agency; tour operator and other reservation service and related activities,1555,930,2015,1870,1170,1175,575,9290 -80,Security and investigation activities,965,1775,4360,1470,955,1150,335,11010 -81,Services to buildings and landscape activities,7300,10965,21375,7380,3290,2715,700,53725 -82,Office administrative; office support and other business support activities,9775,12015,33365,20970,10710,9020,2095,97950 -84,Public administration and defence; compulsory social security,295,4955,1330,545,225,190,415,7955 -85,Education,4325,12155,13340,6640,4780,4930,2925,49095 -86,Human health activities,1780,25625,18635,8240,6070,7070,1235,68655 -87,Residential care activities,1285,1175,1280,1415,1755,4220,1095,12225 -88,Social work activities without accommodation,1740,6050,8970,7475,6120,5190,815,36360 -90,Creative; arts and entertainment activities,5725,6700,11715,4395,2395,1950,425,33305 -91,Libraries; archives; museums and other cultural activities,260,290,500,275,235,270,130,1960 -92,Gambling and betting activities,55,100,200,160,120,135,95,865 -93,Sports activities and amusement and recreation activities,9005,5655,12905,5360,3390,2760,655,39730 -94,Activities of membership organisations,4665,4345,6815,3335,1680,1330,355,22525 -95,Repair of computers and personal and household goods,935,1790,4870,1010,450,390,115,9560 -96,Other personal service activities,14645,28820,28215,6555,3135,2195,435,84000 +69,Legal and accounting activities,9300,15735,26315,10070,5595,4980,1530,73525 +70,Activities of head offices; management consultancy activities,26200,27730,67425,18780,8715,7500,2605,158955 +71,Architectural and engineering activities; technical testing and analysis,8765,21490,29510,8060,4575,5180,1960,79540 +72,Scientific research and development,1470,905,1445,620,425,595,430,5890 +73,Advertising and market research,3305,3690,9300,3055,1770,2060,930,24110 +74,Other professional; scientific and technical activities,9535,18980,31535,6620,3510,3000,740,73920 +75,Veterinary activities,285,780,815,400,510,740,105,3635 +77,Rental and leasing activities,3025,2605,4200,2820,2100,2565,1005,18320 +78,Employment activities,3385,4210,9300,4535,3325,4855,2155,31765 +79,Travel agency; tour operator and other reservation service and related activities,1555,930,2015,1870,1165,1170,580,9285 +80,Security and investigation activities,965,1775,4365,1470,955,1155,330,11015 +81,Services to buildings and landscape activities,7300,10965,21375,7380,3290,2715,695,53720 +82,Office administrative; office support and other business support activities,9780,12010,33365,20965,10715,9030,2085,97950 +84,Public administration and defence; compulsory social security,300,4960,1325,545,225,185,415,7955 +85,Education,4325,12145,13335,6640,4770,4930,2950,49095 +86,Human health activities,1780,25625,18630,8240,6070,7070,1240,68655 +87,Residential care activities,1285,1170,1280,1420,1755,4215,1105,12230 +88,Social work activities without accommodation,1740,6055,8970,7470,6125,5190,810,36360 +90,Creative; arts and entertainment activities,5725,6700,11720,4385,2400,1950,425,33305 +91,Libraries; archives; museums and other cultural activities,260,290,500,270,235,270,130,1955 +92,Gambling and betting activities,55,100,200,160,120,140,95,870 +93,Sports activities and amusement and recreation activities,9000,5650,12905,5360,3385,2770,655,39725 +94,Activities of membership organisations,4670,4340,6820,3335,1680,1335,350,22530 +95,Repair of computers and personal and household goods,940,1795,4875,1005,445,395,120,9575 +96,Other personal service activities,14640,28825,28215,6560,3130,2195,430,83995 97,Activities of households as employers of domestic personnel,0,0,0,0,0,0,0,0 98,Undifferentiated goods- and services-producing activities of private households for own use,0,0,0,0,0,0,0,0 99,Activities of extraterritorial organisations and bodies,0,0,0,0,0,0,0,0 diff --git a/paper/Appendix/a_data.tex b/paper/Appendix/a_data.tex index 635c895..a5707eb 100644 --- a/paper/Appendix/a_data.tex +++ b/paper/Appendix/a_data.tex @@ -8,15 +8,42 @@ \subsection{Data construction detail} Each of the approximately $2.94$ million firm rows carries a turnover, an input expenditure, a net VAT liability ($v_i = 0.20\,(y_i - x_i)$), a calibration -weight, and a registration status; the weights are chosen so the weighted -totals reproduce the official targets while the population sums to about -$2.5$ million firms. Because the population is calibrated \emph{to} the HMRC -aggregates, agreement between my weighted outputs and those aggregates is a -check of \emph{internal consistency}, not external validation: any target used -in calibration is reproduced by construction. The headline calibration -accuracies (92.9\% and 93.5\% for the two vintages) and the anchor comparison -of Section~\ref{sec:static} are therefore consistency checks on the pipeline, -not out-of-sample tests of firm behaviour. +weight, a frame indicator, a VAT-scope flag, and a registration flag. The +weights are chosen so the weighted frame rows reproduce the ONS enterprise +total and employment bands while the weighted registered rows reproduce the +HMRC trader counts and liability totals (Table~\ref{tab:calibration}). Because +the population is calibrated \emph{to} the HMRC aggregates, agreement between +my weighted outputs and those aggregates is a check of \emph{internal +consistency}, not external validation; the turnover-band rows in particular +hold by construction through the registration propensity. The headline +calibration accuracies (98.0\% and 98.0\% for the two +vintages) and the anchor comparison of Section~\ref{sec:static} are therefore +consistency checks on the pipeline, not out-of-sample tests of firm behaviour. + +\begin{table}[htbp] +\centering +\caption{Calibration accuracy by dimension, $\max\{0,1-\lvert\widehat{T}-T\rvert/\lvert T\rvert\}$ +averaged over the dimension's targets. HMRC dimensions are scored on +VAT-registered rows, ONS dimensions on frame rows. Source: +\texttt{results/calibration\_accuracy.txt}.} +\label{tab:calibration} +\begin{tabular}{lrr} +\toprule +Dimension (universe) & 2023--24 & 2024--25 \\ +\midrule +HMRC turnover bands (registered) & 100.0\% & 100.0\% \\ +ONS enterprise total (frame) & 99.7\% & 99.8\% \\ +ONS employment bands (frame) & 99.8\% & 99.9\% \\ +HMRC sector counts (registered) & 93.3\% & 92.2\% \\ +HMRC net liability by band (registered) & 97.1\% & 98.0\% \\ +\midrule +\textbf{Overall (five calibrated dimensions)} & 98.0\% & 98.0\% \\ +\midrule +\emph{Diagnostic:} net liability by sector (registered) & 42.2\% & 45.7\% \\ +\emph{Diagnostic:} net liability below threshold & 31.2\% & 0.0\% \\ +\bottomrule +\end{tabular} +\end{table} The reported calibration score is a bounded summary statistic. For each dimension I compute @@ -61,8 +88,10 @@ \subsection{Data construction detail} so the experimental Populace firm generator inherits the same defect, which I have reported upstream; the target-parity result (zero mismatches) is unaffected, since it concerns the input tables, not the generated population. -Second, the paper validator's scores on the corrected build are 92.9 percent -(2023--24) and 93.5 percent (2024--25). The comparison table and structured provenance are +Second, the paper validator's scores on the current build are +98.0\% (2023--24) and 98.0\% (2024--25), on the +two-universe target layout of Section~\ref{sec:data}, which the pinned +snapshot predates. The comparison table and structured provenance are reproduced by \texttt{firm-microsim-populace-ledger} and checked into \path{results/populace_ledger_comparison.txt} and \path{results/populace_ledger_provenance.json}. diff --git a/paper/Appendix/a_inference.tex b/paper/Appendix/a_inference.tex index 4301bb6..fa00996 100644 --- a/paper/Appendix/a_inference.tex +++ b/paper/Appendix/a_inference.tex @@ -43,7 +43,7 @@ \subsubsection*{Mass-conservation constraint} \int_{T^{*}-W}^{T^{*}} \bigl[f^{\mathrm{obs}}(y)-f^{\mathrm{cf}}(y)\bigr]_{+}\,dy \;=\; E . \] -On the 2023--24 population the baseline gives $E=10{,}418$, +On the 2023--24 population the baseline gives $E=5{,}498$, $\Delta_R=0$, and $y_R=\pounds100$k, the search cap. These mass-conservation outputs are estimator diagnostics, not evidence of a @@ -62,9 +62,11 @@ \subsubsection*{Why no standard errors are reported} estimates move with the estimator's own choices. Generator Monte Carlo error is quantified separately, by rebuilding the full 2023--24 population under alternative generator seeds (\texttt{results/seed\_sensitivity.txt}): across -seeds the half-ranges are $\pm111$ on $E$, $\pm0.07$ on $b_{\mathrm{LLAT}}$, -$\pm\pounds1.4$m on the raise costing, $\pm\pounds0.3$m on the taper costing, -and $\pm\pounds0.03$bn on the registered base. Table~\ref{tab:boot} +seeds the half-ranges are $\pm166$ on $E$, $\pm0.21$ on +$b_{\mathrm{LLAT}}$, $\pm\pounds1.1$m on the raise costing, +$\pm\pounds1.1$m on the taper costing, and $\pm\pounds0.45$bn +on the in-scope base---the last larger than in earlier builds because the +scope flag is itself a seeded draw within each turnover band. Table~\ref{tab:boot} reports the point estimates. Two conventions matter for reading them. Bins are centred on integer \pounds k values (edges at half-integers), so the bin containing the threshold spans @@ -84,7 +86,8 @@ \subsubsection*{Why no standard errors are reported} \toprule Vintage & $E$ & $\Delta_R$ & $b_{\mathrm{LLAT}}$ & $y_R$ (\pounds k) \\ \midrule -2023--24 (\pounds85k, OBR fine targets) & $10{,}418$ & $0$ & $5.127$ & $100.0$ \\ +2023--24 (\pounds85k, OBR fine targets) & $5{,}498$ & $0$ & $5.357$ & $100.0$ (censored) \\ +2024--25 (\pounds90k, coarse bands only) & $10{,}273$ & $4{,}890$ & $4.971$ & $105.0$ (censored) \\ \bottomrule \end{tabular} \end{table} @@ -94,7 +97,8 @@ \subsubsection*{Sensitivity to polynomial degree and exclusion window} Table entries for a grid of polynomial degree $\in\{5,6,7,8\}$ and symmetric exclusion window $\in\{\pounds10\text{k},\ldots,\pounds25\text{k}\}$ are in \texttt{results/bunching\_inference.txt}; the pattern matters more than the -cells. In the 2023--24 grid, $E$ ranges from zero to about $25{,}100$ firms -and $b_{\mathrm{LLAT}}$ from zero to about $8.98$. This specification +cells. In the 2023--24 grid, $E$ ranges from about 560 to about +20{,}800 firms and $b_{\mathrm{LLAT}}$ from about 0.06 to about +8.40. This specification sensitivity, together with the absence of a location-choice mechanism in the generator, precludes interpreting the baseline as a behavioural estimate. diff --git a/paper/Sections/behavioural.tex b/paper/Sections/behavioural.tex index 10c49e7..e9ee2c7 100644 --- a/paper/Sections/behavioural.tex +++ b/paper/Sections/behavioural.tex @@ -46,8 +46,10 @@ \subsection{An iso-elastic simulator on the intensive margin} cost}---the labour and capital effort the firm itself supplies---and \emph{not} its total cost of production. Reading $C$ as total cost would double-count the bought-in inputs, which the value-added factor has already removed. Marginal own-factor cost -is $(y/n)^{1/e}$, so the undistorted optimum is $y=n$: ability $n$ is the firm's -frictionless turnover. A firm registered under a flat rate ($f\equiv1$) chooses +is $(y/n)^{1/e}$, so with no tax and no bought-in inputs ($\delta=0$) the +optimum is $y=n$; with a positive deductible share the untaxed optimum is +$n(1-\delta)^{e}$, and ability $n$ is the firm's frictionless turnover in the +own-factor technology. A firm registered under a flat rate ($f\equiv1$) chooses \begin{equation} y^{*} \;=\; n\,\bigl[(1-\delta)(1-\tau)\bigr]^{e}, \label{eq:flat-optimum} @@ -88,8 +90,10 @@ \subsection{An iso-elastic simulator on the intensive margin} The intensive response depends only on $e$, the statutory rate, and the schedule fractions---not on $\delta_i$, not on the firm's net remittance rate, and not on the recovered ability level. Ability recovery -($n_i=y_{\mathrm{obs},i}$ below the threshold, -$n_i=y_{\mathrm{obs},i}/[(1-\delta_i)(1-\tau)]^{e}$ above) is an +($n_i=y_{\mathrm{obs},i}/(1-\delta_i)^{e}$ below the threshold, +$n_i=y_{\mathrm{obs},i}/[(1-\delta_i)(1-\tau)]^{e}$ above; the released code +sets $\delta_i=0$ in this anchor, which is immaterial because no reform number +depends on it) is an \emph{accounting anchor}---it rationalises the observed allocation given $e$ and reproduces the baseline by construction---but no reform number below depends on it. @@ -135,10 +139,15 @@ \subsection{Results} bands---measured as the change in revenue relative to the \pounds85{,}000-notch baseline, alongside the static figure of Table~\ref{tab:schedule_costs}, for each $e$. For context, the marginal -buncher $n_H(e)$---the highest ability that still finds bunching at the -threshold optimal, the extensive-margin object held fixed here---sits at -\pounds112{,}795, \pounds127{,}382 and \pounds143{,}527 for -$e=0.05, 0.17, 0.32$. +buncher $n_H(e,\delta)$---the highest ability that still finds bunching at the +threshold optimal, the extensive-margin object held fixed here---depends on the +deductible share, because $\delta$ scales revenue but not own-factor cost in +formulation~A: with $\delta=0$ it sits at \pounds112{,}795, \pounds127{,}382 +and \pounds143{,}527 for $e=0.05, 0.17, 0.32$; at the population-mean input +share $\delta=0.6$ it rises to about \pounds118{,}100, \pounds148{,}900 and +\pounds192{,}400 (\texttt{results/dynamic\_reform\_results.txt}). Firms with +higher input shares find registration less attractive relative to bunching, as +Figure~\ref{fig:formulation_a_optima} shows. \begin{table}[htbp] \centering @@ -152,33 +161,35 @@ \subsection{Results} on the assumed $e$ (Section~\ref{ssec:simulator}). The graduated taper is excluded; see the paragraph below.} \label{tab:behavioural_costs} -\resizebox{\textwidth}{!}{% -\begin{tabular}{lrrrrr} +\small +\begin{tabular}{@{}lrrrrr@{}} \toprule Reform & Static & $e=0.05$ & $e=0.17$ & $e=0.32$ & Firms ($e{=}0.17$) \\ \midrule Raise threshold to \pounds100{,}000 - & $-\pounds876$m & $-\pounds876$m & $-\pounds876$m & $-\pounds876$m & $127{,}261$ \\ + & $-\pounds742$m & $-\pounds742$m & $-\pounds742$m & $-\pounds742$m & $114{,}452$ \\ Reduced rate $10\%$ $[\pounds85\text{k},\pounds105\text{k}]$ - & $-\pounds515$m & $-\pounds512$m & $-\pounds505$m & $-\pounds497$m & $149{,}581$ \\ + & $-\pounds446$m & $-\pounds443$m & $-\pounds437$m & $-\pounds430$m & $136{,}112$ \\ Reduced rate $15\%$ $[\pounds85\text{k},\pounds105\text{k}]$ - & $-\pounds258$m & $-\pounds255$m & $-\pounds250$m & $-\pounds243$m & $149{,}581$ \\ + & $-\pounds223$m & $-\pounds221$m & $-\pounds216$m & $-\pounds210$m & $136{,}112$ \\ \bottomrule \end{tabular} -} \smallskip -{\footnotesize \emph{Note.} The graduated taper is omitted: its rate varies -continuously with turnover, so its intensive response is outside the -region-confined solve, and its marginal-rate channel is not represented by the -flat-rate first-order condition. Its $e$-free figures---the static cost of -\(-\)\pounds1{,}525m and the exact removal of the dominated region---are in -Sections~\ref{sec:static} and~\ref{sec:model}.} +{\footnotesize \emph{Note.} The tapers are omitted: their rate varies +continuously with turnover, so their intensive response is outside the +region-confined solve, and their marginal-rate channel is not represented by +the flat-rate first-order condition. Their $e$-free figures---the static costs +of \(-\)\pounds1{,}406m (linear) and \(-\)\pounds968m (constant $50\%$) and the exact +removal of the dominated region---are in Sections~\ref{sec:static} +and~\ref{sec:model}. Firms are in-scope VAT firms with turnover in +\pounds70{,}000--\pounds130{,}000, the range the region-confined solve +re-optimises.} \end{table} \paragraph{A level rise has no intensive-margin offset.} The raise-to- -\pounds100{,}000 row is identical to its static cost at every $e$, and this is -a result, not a failure to respond. Partition the firms. Those with observed +\pounds100{,}000 row is identical to its static cost at every $e$; this follows +from the region-confined convention rather than from a failure to respond. Partition the firms. Those with observed turnover below \pounds85{,}000 face $f=0$ before and after; nothing changes. Those released by the raise ($y_{\mathrm{obs}}\in[\pounds85\text{k}, \pounds100\text{k})$) see their rate fall to zero and expand @@ -187,22 +198,26 @@ \subsection{Results} base, so the revenue loss is their full baseline remittance, exactly the static figure. Those above \pounds100{,}000 face $f=1$ before and after; nothing changes. The only candidates for a taxed response are released firms -whose expansion would carry them across the new threshold, and for every such -firm the new notch's bunching condition binds (its ability lies strictly below -the new marginal buncher), so it optimally stops at the threshold and remits -nothing. Base broadening from a threshold rise is therefore real economic -activity but generates no VAT: conditional on this model, the static costing -of a level move is not an upper bound that behaviour erodes---it \emph{is} the -behavioural costing on the intensive margin. The simulator asserts this -invariance programmatically (to \pounds0.1m at each swept $e$). +whose expansion would carry them across the new threshold. The region-confined +solve does not evaluate that discrete choice: it clips such firms at the new +threshold by construction, so the invariance is a property of the maintained +status convention---released firms deregister and are not permitted to re-cross +the new notch---rather than a computed statement about their global optimum. +Under that convention, base broadening from a threshold rise is real economic +activity that generates no VAT, and the static costing of a level move +coincides with the intensive-margin behavioural costing. A released firm whose +ability exceeded the new marginal buncher would instead register at its taxed +optimum and remit VAT; that is an extensive-margin case outside this exercise. +The simulator asserts the conditional invariance programmatically (to +\pounds0.1m at each swept $e$). \paragraph{Reduced-rate offsets are second order.} A reduced-rate band lowers the rate on band firms from $\tau$ to $r$, so each scales up by~\eqref{eq:rescale} with $f_1 = r/\tau$: about $+2.0\%$ at the $10\%$ rate and $e=0.17$, and about $+1.0\%$ at $15\%$. Remittance scales with turnover, so the offset is roughly the scale gain times the band's (rate-reduced) -remittance---about \pounds10m against a static \pounds515m at $e=0.17$ for the $10\%$ -band, reaching about \pounds18m only at $e=0.32$. The direction is +remittance---about \pounds9m against a static \pounds446m at $e=0.17$ for +the $10\%$ band, reaching about \pounds16m only at $e=0.32$. The direction is uniform---a larger $e$ makes the bands slightly cheaper---but the magnitude never approaches the static term. Figure~\ref{fig:reform_dist} traces the costs across the full range of $e$: the raise is exactly flat, the bands diff --git a/paper/Sections/bunching.tex b/paper/Sections/bunching.tex index 0ff272c..3acd001 100644 --- a/paper/Sections/bunching.tex +++ b/paper/Sections/bunching.tex @@ -32,7 +32,7 @@ \section{Bunching and the limits of band-calibrated synthetic data} total, which belongs to voluntary registrants the liability model does not represent, was initially calibrated against the whole below-threshold population, draining weights around the fine-target window; on an -intermediate build it also manufactured a spectacular ($E\approx196{,}000$), +intermediate build it also produced a large ($E\approx196{,}000$), specification-robust artifact at the \pounds90{,}000 band edge. It is now an informational diagnostic (Section~\ref{sec:data}). Aggregate calibration scores barely moved under any of these changes---the optimiser hit the band @@ -56,37 +56,52 @@ \subsection{Methodology} where $c_j$ is the number of firms in bin $j$, $y_j$ is the distance of bin $j$ from the threshold, $q$ is the polynomial order, and $[y^{*}_{-},\,y^{*}_{+}]$ is the excluded window. The fitted polynomial is then projected into the -excluded region to recover the counterfactual density $g(y)$, with a -mass-conservation restriction that locates the upper edge of the manipulation -window endogenously (excess mass below the threshold must equal missing mass -above it; Appendix~\ref{app:inference}). Excess bunching is the integrated -difference between the observed and counterfactual densities just below the -threshold, +excluded region to recover the counterfactual density $g(y)$. The upper edge +of the manipulation window, $y_R$, is then searched for by walking upward from +the threshold until cumulative missing mass equals the excess mass below; when +the missing mass inside the search window is insufficient---as in the headline +specification, where the fitted missing mass is zero---$y_R$ is censored at the +window edge and reported as such (Appendix~\ref{app:inference}). Excess mass is +the integrated positive part of the gap between the observed density $f$ and +the counterfactual $g$ in the excluded window below the threshold, \[ - B(y^{*}) \;=\; \int_{y^{*}-\Delta y^{*}}^{y^{*}} g(y)\,\mathrm{d}y - \;\approx\; g(y^{*})\,\Delta y^{*}, + E \;=\; \int_{y^{*}-W}^{y^{*}} \bigl[f(y)-g(y)\bigr]_{+}\,\mathrm{d}y , \] -the approximation holding when $g$ is locally smooth. The bunching ratio -$b(y^{*}) = B(y^{*})/g(y^{*}) \approx \Delta y^{*}/y^{*}$ is the width of the -bunching region expressed as a fraction of the threshold---a normalised -\emph{ratio}, not an elasticity. The estimator delivers the raw geometry of any +a gross rather than net statistic. The normalised excess $b_{\mathrm{LLAT}}$ +divides the integrated positive-part gap over the whole manipulation region +$[y^{*}-W,\,y_R]$ by the average counterfactual height over that region---a +literature-style normalisation in the spirit of \citet{liuetal2021}, who report +excess mass relative to the counterfactual density, though it does not +reproduce their estimator exactly. Because it also counts positive gaps above +the threshold, $b_{\mathrm{LLAT}}$ is not $E$ divided by a density, and the two +statistics need not move together. Neither is an elasticity. The estimator +delivers the raw geometry of any density step at the threshold; whether that step reflects firm behaviour is a -separate question. The estimator reports geometry only ($b$, excess mass $E$, -and the mass-conservation objects): no elasticity is computed from it, both +separate question. The estimator reports geometry only ($b_{\mathrm{LLAT}}$, +excess mass $E$, and the mass-conservation objects): no elasticity is computed from it, both because the identification critique of bunching designs applies \citep{blomquistetal2021,bertanhaetal2023} and because on these data there is no behavioural mass to convert. \subsection{Results: diagnostics, not behavioural estimates} -At the \pounds85{,}000 threshold the baseline specification returns -$E=10{,}418$ weighted firms and $b_{\mathrm{LLAT}}=5.127$. This reads back a -profile calibrated to OBR fine-band targets; the generator contains no firm -location choice, so it is not a behavioural estimate. Across the degree--window -grid, $E$ ranges from zero to about 25{,}100 firms and the headline fitted -missing mass above the threshold is zero. The figure is retained as an -audit of what the estimator reads from the synthetic construction, not as -evidence about firm responses. +At the \pounds85{,}000 threshold (Figure~\ref{fig:bunch85}) the baseline specification returns +$E=5{,}498$ weighted firms and $b_{\mathrm{LLAT}}=5.357$ on the +frame density. This reads back a profile calibrated to OBR fine-band targets; +the generator contains no firm location choice, so it is not a behavioural +estimate. Across the degree--window grid, $E$ ranges from about 560 +to about 20{,}800 firms and the headline fitted missing mass above the +threshold is zero, with $y_R$ censored at the search cap +(Appendix~\ref{app:inference}). The 2024--25 population, which has no +fine-band targets, makes the complementary point: at its \pounds90{,}000 +threshold the same estimator returns $E=10{,}273$ and +$b_{\mathrm{LLAT}}=4.971$, an excess mass produced entirely by the +step between the two HMRC coarse bands that meet at \pounds90{,}000, which the +weight optimiser resolves as a density step. Neither figure is evidence about +firm responses; both are audits of what the estimator reads from the synthetic +construction, and the second is the reason the $\pm\pounds5{,}000$ rows of +Table~\ref{tab:static_revenue} are flagged as conditional on the within-band +allocation. \begin{figure}[htbp] \centering @@ -95,8 +110,8 @@ \subsection{Results: diagnostics, not behavioural estimates} counterfactual at the 2023--24 registration threshold. The 2023--24 population reproduces the administratively observed bunching profile because it is calibrated to the OBR \pounds1{,}000-band targets; the -estimator reads the target-inherited excess back as $E = 10{,}418$ -($b_{\mathrm{LLAT}} = 5.127$). This is not behavioural: +estimator reads the target-inherited excess back as $E = 5{,}498$ +($b_{\mathrm{LLAT}} = 5.357$). This is not behavioural: the generator has no firm location-choice, so all sub-band structure is inherited from the targets and the weight optimiser.} \label{fig:bunch85} @@ -111,7 +126,7 @@ \subsection{Comparison with the literature} from \pounds58{,}000 to \pounds81{,}000, with an excess-bunching ratio of $b = 1.361$ (SE $0.202$) around the year-specific threshold and a manipulation window of $-\pounds14{,}000$ to $+\pounds24{,}000$. The synthetic baseline's -$b_{\mathrm{LLAT}}=5.127$ is not a second estimate of their parameter: it is +$b_{\mathrm{LLAT}}=5.357$ is not a second estimate of their parameter: it is an unstable readback of the OBR target profile. What does carry across is institutional context: LLAT report that almost half (roughly $43\%$) of eligible below-threshold firms register voluntarily, which dampens diff --git a/paper/Sections/conclusion.tex b/paper/Sections/conclusion.tex index fafefd1..d3f9637 100644 --- a/paper/Sections/conclusion.tex +++ b/paper/Sections/conclusion.tex @@ -20,29 +20,35 @@ \section{Conclusion} data-year baseline.} \label{tab:summary} \small -\begin{tabularx}{\textwidth}{@{}Xlll@{}} +\begin{tabularx}{\textwidth}{@{}>{\raggedright\arraybackslash}p{0.30\textwidth}>{\raggedright\arraybackslash}p{0.19\textwidth}Xl@{}} \toprule -Object & Symbol / definition & Value & Source \\ +Object & Definition & Value & Source \\ \midrule -Static cost, \pounds85k to \pounds90k anchor & dereg.\ / retention conventions & $-\pounds364$m / $-\pounds207$m & Section~\ref{sec:static} \\ -Static sweep, \pounds90k baseline & \pounds70k to \pounds120k & $+\pounds1.46$bn to $-\pounds1.55$bn & Table~\ref{tab:static_revenue} \\ -Dominated region & $a=T^{*}\tau/(1-\tau)$ & \pounds21{,}250 & Section~\ref{ssec:dominated} \\ -Firms in the dominated band (weighted) & --- & $\approx157{,}000$ & Section~\ref{ssec:dominated} \\ -Raise to \pounds100k, static and behavioural & invariant in $e$ & $-\pounds876$m & Sections~\ref{sec:static}, \ref{sec:behavioural} \\ -Graduated taper $[\pounds85\text{k},\pounds141.7\text{k}]$, static & --- & $-\pounds1{,}525$m & Section~\ref{ssec:schedule_costs} \\ +Anchor, \pounds85k to \pounds90k, 2025--26 & dereg.\ / 43\% retention & $-\pounds234$m / $-\pounds134$m (HMRC $-\pounds185$m) & \S\ref{sec:static} \\ +Sweep, \pounds90k baseline & \pounds70k to \pounds120k & $+\pounds0.19$bn to $-\pounds1.19$bn & Table~\ref{tab:static_revenue} \\ +Dominated region & $a=T^{*}\tau/(1-\tau)$ & \pounds21{,}250 & \S\ref{ssec:dominated} \\ +In-scope firms in dominated band & weighted & $\approx$143{,}000 & \S\ref{ssec:dominated} \\ +Raise to \pounds100k & static $=$ behavioural & $-\pounds742$m & \S\ref{sec:static}, \S\ref{sec:behavioural} \\ +Taper, \pounds85k--\pounds141.7k & linear / constant $50\%$ & $-\pounds1{,}406$m / $-\pounds968$m & \S\ref{ssec:schedule_costs} \\ +Reduced rate, \pounds85k--\pounds105k & $10\%$ / $15\%$ & $-\pounds446$m / $-\pounds223$m & \S\ref{ssec:schedule_costs} \\ +Bunching readback, 2023--24 & $E$; $b_{\mathrm{LLAT}}$ & $5{,}498$; $5.357$ (target-inherited) & \S\ref{sec:bunching} \\ \bottomrule \end{tabularx} \end{table} \paragraph{Findings.} On the common \pounds85{,}000 baseline, raising the -threshold to \pounds100{,}000 costs \pounds876m, the graduated taper costs -\pounds1{,}525m across its wider \pounds85{,}000--\pounds141{,}667 band, and the -10\% and 15\% reduced-rate bands cost \pounds515m and \pounds258m. The anchor reform still overshoots HMRC's published costing under -both deregistration conventions, so it is an internal consistency check rather -than external validation. +threshold to \pounds100{,}000 costs \pounds742m, a taper across the +\pounds85{,}000--\pounds141{,}667 band costs \pounds1{,}406m (linear phase-in) or +\pounds968m (constant 50\% marginal rate), and the 10\% and 15\% +reduced-rate bands cost \pounds446m and \pounds223m. The anchor reform +brackets HMRC's published costing between the full-deregistration and 43\% +retention conventions once releases are limited by the deregistration +threshold; it is an internal consistency check rather than external +validation, and the registration behaviour of released firms remains the +decisive convention. The notch creates an exact \pounds21{,}250 dominated region. Raising the -threshold relocates it, and a reduced-rate band divides it between two notches; +threshold relocates and widens it, and a reduced-rate band divides it between two notches; only the continuous taper removes it. Intensive-margin responses change the reduced-rate costings only modestly under the assumed elasticities, while a threshold rise has no intensive-margin revenue offset in this model. @@ -55,7 +61,15 @@ \section{Conclusion} \paragraph{Limitations and future work.} The model omits location choice, voluntary-registration choice, compliance costs, sectoral incidence, -pass-through, and general-equilibrium effects. It also cannot capture persistent +pass-through, and general-equilibrium effects. Its population is the ONS +VAT/PAYE frame with an HMRC-calibrated registered subset: threshold cuts are +therefore costed only on frame enterprises, most of which are already +registered, and omit the unregistered sole traders outside the frame that a +lower threshold would mainly reach; extending the frame to the DBT business +population is the natural next build. A fixed cross-section also has no +registration response or entry margin, so the anchor brackets HMRC's profile +only through explicit deregistration conventions. The nominal-growth factors +used for ageing are assumptions recorded in the replication repository. It also cannot capture persistent growth effects extending beyond the cutoff, as documented for a different payroll-tax notch by \citet{jysmabenzartiharju2026}. Those estimates do not transfer directly to UK VAT. Administrative firm-level validation, an explicit diff --git a/paper/Sections/data.tex b/paper/Sections/data.tex index 83a0f9a..ec3b96f 100644 --- a/paper/Sections/data.tex +++ b/paper/Sections/data.tex @@ -35,12 +35,43 @@ \section{Data} published band boundary. I interpret integer-labelled bands as contiguous half-open intervals: for example, the published ``50--99'' band is represented as $[50,100)$ thousand pounds. -The ONS register covers businesses registered for VAT and/or -PAYE---about 2.7 million---so the synthetic population represents that -registered-business frame, not the full business population of roughly -5.6 million in the DBT business population estimates; the difference is -concentrated below the threshold and matters for threshold \emph{cuts}, which I -flag where relevant. Each firm is given input expenditure $x_i = \rho_i\,y_i$, +The counts come from Table~8 of the ONS workbook (enterprises by SIC division +and turnover sizeband, United Kingdom); the employment-band targets below come +from Table~3 of the same workbook (enterprises by employment sizeband), so both +ONS margins count the same statistical unit.\footnote{An earlier build of the +2023--24 population used the local-unit table (Table~18, individual sites: +3.17 million against 2.72 million enterprises) for the employment margin, so +its employment and population targets could not be met by the same weights. +The processed tables are now produced by an executable extract from the raw +workbooks and tested for agreement to disclosure rounding.} + +\paragraph{Two universes.} The model distinguishes two populations and +declares which one each calibration target describes. The \emph{frame} is the +ONS register of enterprises operating VAT and/or PAYE---about 2.72 million in +2024---which the population, employment, and near-threshold shape targets +describe. The \emph{registered subset} is the population of HMRC VAT +traders---about 2.18 million in 2023--24---which the HMRC counts by turnover +band and by trade sector and the net-liability totals describe. The two differ +in both directions. Above the threshold only about two-thirds of frame +enterprises are VAT traders (from 51\% in the \pounds150{,}000--\pounds300{,}000 +band to 100\% in the \pounds1m--\pounds10m band): the remainder are PAYE-only +employers and businesses in exempt sectors (finance, health, education, and +similar) outside the VAT net. The share is 64\% in the open +\pounds10m-plus band, but there the shortfall is largely a construction +artefact: the ONS ``\pounds5m and over'' band has no published top, and the +uniform draw to \pounds50m places more enterprises above \pounds10m than HMRC +counts; the band's liability total is calibrated regardless, so reform +costings near the threshold are unaffected, but the top band's in-scope draw +is a seeded selection and contributes most of the seed dispersion of the base +(Appendix~\ref{app:inference}). Below the threshold about 89\% of frame +enterprises are registered, largely because small businesses enter the +register \emph{through} VAT registration. HMRC also reports about 216{,}500 +traders with negative or zero turnover who are not identifiable in the ONS +turnover bands; I append them as an out-of-frame stratum before calibration. +Neither universe is the full business population of roughly 5.6 million in the +DBT business population estimates; the difference is concentrated below the +threshold and matters for threshold \emph{cuts}, which I flag where relevant. +Each firm is given input expenditure $x_i = \rho_i\,y_i$, with the input--output ratio $\rho_i$ drawn from a rescaled Beta distribution with sector-specific shifts, clamped to $\rho_i \in [0.1, 0.95]$ (mean about 0.6, so value added averages about 40\% of turnover, in line with the UK @@ -66,11 +97,18 @@ \section{Data} \[ \widehat{T}_k(\theta) \;=\; \sum_i a_{ki}\, w_i , \] -where $a_{ki}$ is firm $i$'s contribution to target $k$: an indicator of band or -employment membership for a count target, the expected registration propensity -times a sector indicator for HMRC registered-sector counts, and the firm's net -liability $v_i$ for a VAT-liability target. Registration propensity equals one -above the threshold and the HMRC voluntary-registration share below it. I choose +where $a_{ki}$ is firm $i$'s contribution to target $k$ on the universe the +target describes. For the frame targets---the ONS enterprise total and the +employment bands---$a_{ki}$ is a membership indicator for frame firms and zero +for the appended HMRC traders. For the registered-subset targets---HMRC trader +counts by turnover band and by sector, and net liability by band---$a_{ki}$ is +the firm's \emph{registration propensity} $p_{b(i)}$ times the band or sector +indicator (times $v_i$ for a liability target), where $p_b$ is the HMRC trader +count in turnover band $b$ divided by the frame's mass in that band; appended +traders carry $p=1$. At unit weights the registered-subset rows therefore +reproduce the HMRC counts by construction and the frame rows reproduce the ONS +counts by construction; the optimiser's work is to fit the sector composition, +the liability totals, and the near-threshold shape while holding both. I choose $\theta$ to minimise a symmetric relative-error loss with per-target importance weights $\lambda_k$, \[ @@ -79,9 +117,13 @@ \section{Data} \;+\; \frac{\alpha}{N}\sum_i \lvert \theta_i \rvert , \] the final term a mean absolute penalty on the log-weights. The targets are the -HMRC VAT-registered counts by turnover band and by sector, the HMRC net VAT -liability by turnover band, the ONS employment-band totals, and---for the -2023--24 vintage---the OBR's published near-threshold profile. Chart~C of the +HMRC VAT-registered counts by turnover band (all eight bands, including the +negative-or-zero band) and by trade sector, the HMRC net VAT liability by +turnover band, the ONS enterprise total and employment-band totals, and---for +the 2023--24 vintage---the OBR's published near-threshold profile. HMRC's +sector table includes traders whose turnover band is ``Unknown'' +(159{,}300 in 2024--25, none in 2023--24); I allocate them proportionally so +the sector and band rows count the same registered population. Chart~C of the March 2023 \emph{Economic and Fiscal Outlook} reports HMRC business counts in \pounds1{,}000 turnover bands from \pounds65{,}000 to \pounds90{,}000 \citep{obr2023vat}. I interpolate these counts to 2023--24 between the @@ -125,51 +167,70 @@ \section{Data} solved by gradient descent, and the symmetric loss keeps targets of very different magnitudes on a common scale. -A firm is registered with certainty when its data-year turnover exceeds the -threshold $T^{*}$. This is an approximation to the statutory test, which is a +\paragraph{Scope and registration.} After calibration each firm receives two +flags. \emph{Scope} marks firms in the VAT net: above the threshold, a seeded +weighted selection within each HMRC turnover band whose cumulative weight +reaches that band's HMRC trader count (the remainder are out-of-scope +PAYE-only or exempt-sector enterprises, which remit nothing under any schedule +considered here); below the threshold, every frame enterprise is treated as +registrable, a maintained assumption since HMRC publishes only the registered +count there. \emph{Registration} marks in-scope firms above the threshold and, +below it, a seeded weighted selection reaching the HMRC \pounds1--threshold +count; appended negative-or-zero traders are in scope and registered. +Registration above the threshold is thus certain for in-scope firms. This is an +approximation to the statutory test, which is a rolling 12-month test on \emph{taxable} turnover with a roughly two-month registration lag, a forward-looking 30-day test, and an exception for temporary -breaches (Section~\ref{sec:background}); the model also registers firms whose -turnover is largely exempt, whom the law would not require to register. For a -single-year cross-section calibrated to registered-population totals these +breaches (Section~\ref{sec:background}); exempt-sector businesses are +represented only through the out-of-scope share, not identified individually. +For a single-year cross-section calibrated to registered-population totals these timing and composition wedges are second order for the aggregates, but they are worth naming: the HMRC band tables classify traders by total declared outputs (VAT-return Box 6), and the ONS turnover measure is likewise total turnover, so -the running variable throughout is total rather than taxable turnover. Below -the threshold, candidate firms are placed in a seeded random order and selected -until their cumulative calibration weight reaches the HMRC count $M$ of -registered firms in the \pounds1--threshold band. This matches the weighted -total to within one firm's weight but does not identify which firms register -voluntarily; their composition remains a modelling assumption. The resulting -dataset comprises approximately $2.94$~million firm -records per vintage, weighted to represent roughly $2.5$~million registered -businesses. Checked against the official sources along every calibrated -dimension, the vintages attain overall calibration accuracies of 92.9\% -(2023--24) and 93.5\% (2024--25), with calibrated-dimension scores ranging -from 88.3\% to 97.3\%; Appendix~\ref{app:data} reports -the full table. The headline is the mean of five calibrated-dimension scores, -each clipped at zero as $\max\{0,1-\lvert \widehat{T}-T\rvert/\lvert T\rvert\}$; -it excludes the VAT-liability-by-sector diagnostic described above. +the running variable throughout is total rather than taxable turnover. The +seeded selections match each weighted total to within one firm's weight but do +not identify \emph{which} firms are in scope or register voluntarily; their +composition remains a modelling assumption. The resulting dataset comprises +approximately $2.94$~million firm records per vintage (2.72 million frame +draws plus 0.22 million appended traders), weighted to represent +about 2.72 million frame enterprises in 2023--24, of which about 2.18 million are +VAT-registered traders. Checked against the official sources along every +calibrated dimension, the vintages attain overall calibration accuracies of +98.0\% (2023--24) and 98.0\% (2024--25); +Appendix~\ref{app:data} reports the full table. The headline is the mean of +five calibrated-dimension scores, each clipped at zero as +$\max\{0,1-\lvert \widehat{T}-T\rvert/\lvert T\rvert\}$, HMRC dimensions +scored on registered rows and ONS dimensions on frame rows; it excludes the +VAT-liability-by-sector diagnostic described above. Because the turnover-band +rows hold by construction, the informative residuals are the sector +composition (93.3\% and 92.2\%) and the liability totals by band +(97.1\% and 98.0\%). Aggregate fit does not imply record-level representativeness. Kish effective -sample sizes are 92{,}326 (3.1\% of rows) and 188{,}612 (6.4\%), while maximum -weights are about 2{,}598 and 1{,}775. Synthetic rows should therefore be read +sample sizes are 137{,}054 (4.7\% of rows) and 192{,}917 +(6.5\%), while maximum weights are about 2{,}446 and +1{,}830; the median weight is one, and the concentration sits in the +large-turnover bands that carry the liability totals. Synthetic rows should therefore be read as weighted accounting support points, not independent firm observations; local and sub-band results can be sensitive to a small number of high-weight records. I report rounded scenario estimates and expose the full weight diagnostics in the generated calibration report. \paragraph{The registration threshold in the data.} My primary data year is -2023--24, when the statutory threshold was \pounds85{,}000. The HMRC band +2023--24, when the statutory threshold was \pounds85{,}000 +(Figure~\ref{fig:turnover_dist} shows the calibrated distribution). The HMRC band targets step down across the threshold---678{,}350 firms in the \pounds1--\pounds85{,}000 band against 305{,}320 in \pounds85{,}000--\pounds150{,}000, or roughly $8{,}000$ against $4{,}700$ firms -per \pounds1{,}000 of band width---and the calibrated population necessarily -inherits that band-average step. The \emph{local} shape within a few thousand -pounds of the threshold is pinned by the OBR near-threshold targets described -above: the calibrated density rises into the threshold (about $13{,}800$ firms -per \pounds1{,}000 in the \pounds84{,}000--\pounds85{,}000 bin), steps down -across it (about $10{,}300$ just above), and declines through the band---the +per \pounds1{,}000 of band width---and the registered subset of the calibrated +population inherits that band-average step through the registration +propensity; the frame density itself steps only from about $8{,}900$ to +$7{,}000$ enterprises per \pounds1{,}000 across the same bands. The +\emph{local} shape within a few thousand pounds of the threshold is pinned by +the OBR near-threshold targets described above: the calibrated frame density +rises into the threshold (about $14{,}600$ firms per \pounds1{,}000 in the +\pounds84{,}000--\pounds85{,}000 bin), steps down across it (about $11{,}600$ +just above), and declines through the pinned window---the administratively observed bunching geometry \citep{obr2023vat,liuetal2021}, present in the file because it is targeted, not because any synthetic firm chooses it. In an earlier build without these targets the local shape was an @@ -197,11 +258,12 @@ \section{Data} VAT/PAYE-registered frame: unregistered businesses---most of the below-threshold mass, and most of the cross-threshold cliff, in all-business administrative charts---are outside it, so level comparisons to such charts -are not like-for-like. The steps at \pounds90{,}000 (about $9{,}100$ firms -per \pounds1{,}000 in the last OBR-pinned bin against about $6{,}800$ in the -first unpinned bin) and \pounds150{,}000 are calibration-target -boundaries---the top of the OBR fine-band window and an HMRC band edge -respectively---not features of the underlying firm distribution. The +are not like-for-like. The steps at \pounds90{,}000 (about $10{,}200$ firms +per \pounds1{,}000 in the last OBR-pinned bin against about $14{,}400$ in the +first unpinned bin, where the density reverts to the frame's coarse-band +level) and \pounds150{,}000 are calibration-target boundaries---the top of the +OBR fine-band window and an HMRC band edge respectively---not features of the +underlying firm distribution. The \pounds90{,}000 boundary lies inside the bunching estimator's fit range; Appendix~\ref{app:inference} measures its effect on the headline statistics.} \label{fig:turnover_dist} diff --git a/paper/Sections/intro.tex b/paper/Sections/intro.tex index 9359a1d..d76dc1a 100644 --- a/paper/Sections/intro.tex +++ b/paper/Sections/intro.tex @@ -4,12 +4,11 @@ \section{Introduction}\label{sec:intro} \begin{figure}[htbp] \centering -\subfigure[VAT- and/or PAYE-registered businesses (ONS, 2023--24)]{ - \includegraphics[width=0.48\textwidth]{figures/firms_by_turnover_band_85k.png} -} -\hfill -\subfigure[VAT-registered firms (HMRC, 2023--24)]{ - \includegraphics[width=0.48\textwidth]{figures/vat_firms_by_turnover_band_85k.png} +\subfigure[VAT- and/or PAYE-registered businesses (ONS, 2023--24)]{% + \includegraphics[width=0.48\textwidth]{figures/firms_by_turnover_band_85k.png}% +}\hfill% +\subfigure[VAT-registered firms (HMRC, 2023--24)]{% + \includegraphics[width=0.48\textwidth]{figures/vat_firms_by_turnover_band_85k.png}% } \caption{The UK firm-size distribution and VAT registration, 2023--24 (threshold \pounds 85{,}000). Panel (a): number of businesses by turnover band in the ONS @@ -19,8 +18,8 @@ \section{Introduction}\label{sec:intro} Panel (b): number of VAT-registered firms by turnover band. Band widths are unequal, so cross-band count comparisons should be read per \pounds1{,}000 of turnover. Source: panel (a), ONS, -\href{https://www.ons.gov.uk/businessindustryandtrade/business/activitysizeandlocation/bulletins/ukbusinessactivitysizeandlocation/2023}{\emph{UK Business: Activity, Size and Location}} -(business counts by turnover band); panel (b), HMRC, +\href{https://www.ons.gov.uk/businessindustryandtrade/business/activitysizeandlocation/bulletins/ukbusinessactivitysizeandlocation/2024}{\emph{UK Business: Activity, Size and Location}}, +2024 edition (Table~8, enterprises by turnover sizeband); panel (b), HMRC, \href{https://www.gov.uk/government/statistics/value-added-tax-vat-annual-statistics}{\emph{Value Added Tax (VAT) Annual Statistics}} (VAT-registered population by turnover band).} \label{fig:dist} @@ -51,13 +50,13 @@ \section{Introduction}\label{sec:intro} On these data I do four things. First, I cost level reforms and schedule reforms---a graduated taper and a reduced-rate band---statically, by applying each counterfactual schedule mechanically to the firm distribution. Second, I characterise the notch's distortion through its \emph{dominated region}---the exact, elasticity-free range of turnover above the threshold in which no firm profits from locating---and how each reform changes it. Third, I price the level and rate reforms behaviourally: each firm re-optimises its turnover within its schedule region, conditional on an assumed turnover elasticity $e$ that the data do not identify, reported as an $e$-sensitivity range that nests the static costing exactly in its $e\to0$ limit. Fourth, I show that the synthetic data's near-threshold structure is target-inherited and cannot support bunching inference: the \pounds85{,}000 profile is calibrated to the OBR's published \pounds1{,}000-band data and the generator contains no location-choice mechanism. -Four findings follow. First, the static pipeline prices all reforms on a common basis, but its anchor is a consistency check rather than a validation. The mechanical 2025--26 cost of the April 2024 increase from \pounds85{,}000 to \pounds90{,}000 is \pounds364m if every released firm deregisters and \pounds207m if 43\% of released-firm liability is retained, against HMRC's published \pounds185m \citep{hmt_spring_budget_2024}. Both conventions reproduce the official profile's narrowing and its 2028--29 sign change, but the level gap shows that registration dynamics and population scope matter (Section~\ref{sec:static}). Direct mechanical moves from the \pounds90{,}000 base range from a gain of about \pounds1.46bn at \pounds70{,}000 to a cost of about \pounds1.55bn at \pounds120{,}000, against a base of about \pounds190.8bn in 2025--26. +Four findings follow. First, the static pipeline prices all reforms on a common basis, but its anchor is a consistency check rather than a validation. The mechanical 2025--26 cost of the April 2024 increase from \pounds85{,}000 to \pounds90{,}000 is \pounds234m if every firm released below the \pounds88{,}000 deregistration threshold deregisters and \pounds134m if 43\% of released-firm liability is retained, bracketing HMRC's published \pounds185m \citep{hmt_spring_budget_2024}. Both conventions reproduce the official profile's sign change in 2028--29; the level depends on the registration behaviour of just-released firms, which the data do not identify (Section~\ref{sec:static}). Direct mechanical moves from the \pounds90{,}000 base range from a gain of about \pounds187m at \pounds70{,}000---small, because most below-threshold enterprises in the ONS frame are already registered---to a cost of about \pounds1.19bn at \pounds120{,}000, against an in-scope base of about \pounds183.7bn in 2025--26. -Second, the shape and rate reforms act on the notch's distortion in a way that a change in level does not. The baseline notch creates a \pounds21{,}250 dominated region above the threshold, containing about 157{,}000 weighted firms in the synthetic population. Raising the threshold relocates this region; a banded reduced rate splits it across two notches; only the continuous taper removes it. On the common \pounds85{,}000 data-year base, raising the threshold to \pounds100{,}000 costs \pounds876m, the taper costs \pounds1{,}525m over its \pounds85{,}000--\pounds141{,}667 band---removing the dominated region requires relief that wide (Section~\ref{sec:model})---and the 10\% and 15\% reduced rates cost \pounds515m and \pounds258m. These comparisons index the model-implied location distortion and omit compliance-cost savings (Section~\ref{sec:conclusion}). +Second, the shape and rate reforms act on the notch's distortion in a way that a change in level does not. The baseline notch creates a \pounds21{,}250 dominated region above the threshold, containing about 143{,}000 weighted in-scope firms in the synthetic population. Raising the threshold relocates this region and widens it in proportion (to \pounds25{,}000 at \pounds100{,}000); a banded reduced rate splits it across two notches; only a continuous taper removes it. On the common \pounds85{,}000 data-year base, raising the threshold to \pounds100{,}000 costs \pounds742m; a taper over the \pounds85{,}000--\pounds141{,}667 band costs \pounds1{,}406m with a linear marginal-rate phase-in and \pounds968m with a constant 50\% marginal rate---the band top is a property of the schedule's shape, not a requirement of removing the notch (Section~\ref{sec:model})---and the 10\% and 15\% reduced rates cost \pounds446m and \pounds223m. These comparisons index the model-implied location distortion and omit compliance-cost savings (Section~\ref{sec:conclusion}). -Third, conditional on the assumed elasticity, intensive-margin responses barely move these costings. A pure level rise has exactly zero intensive-margin revenue offset in this model. At $e=0.17$, the 10\% band's cost falls from \pounds515m to \pounds505m and the 15\% band's from \pounds258m to \pounds250m. The important uncertainty is registration and location choice at the notch, exactly the margin the synthetic data cannot identify. +Third, conditional on the assumed elasticity, intensive-margin responses barely move these costings. A pure level rise has zero intensive-margin revenue offset under the model's region-confined convention. At $e=0.17$, the 10\% band's cost falls from \pounds446m to \pounds437m and the 15\% band's from \pounds223m to \pounds216m. The important uncertainty is registration and location choice at the notch, exactly the margin the synthetic data cannot identify. -Fourth, the open data carry a transparency lesson about bunching. The 2023--24 population is calibrated to the OBR's published \pounds1{,}000-band profile \citep{obr2023vat}, so its baseline excess mass ($E=10{,}418$) is target inheritance, not behavioural evidence. The estimate is highly sensitive to degree and window, and the fitted missing mass is zero in the headline specification. The synthetic data therefore cannot validate the estimator or identify behaviour. The behavioural evidence remains the administrative-data work of \citet{liuetal2021}. +Fourth, the open data carry a transparency lesson about bunching. The 2023--24 population is calibrated to the OBR's published \pounds1{,}000-band profile \citep{obr2023vat}, so its baseline excess mass ($E=5{,}498$) is target inheritance, not behavioural evidence; the 2024--25 population, calibrated to coarse bands only, produces a larger readback ($E=10{,}273$) from nothing more than the step between two HMRC bands. The estimates are highly sensitive to degree and window, and the fitted missing mass is zero in the headline specification. The synthetic data therefore cannot validate the estimator or identify behaviour. The behavioural evidence remains the administrative-data work of \citet{liuetal2021}. \subsection*{Related literature} @@ -69,9 +68,16 @@ \subsection*{Related literature} \citet{blomquistetal2021} show that it is not non-parametrically identified at a kink or notch without strong functional-form assumptions; \citet{bertanhaetal2023} develop the notch case. I therefore do not infer an elasticity from the synthetic -data. The fourth strand is the structural literature on size-based thresholds -\citep{garicano2016,gourioroys2014}; the present paper makes no comparable -structural claim. +data. The fourth strand is the theory of size-based tax thresholds: the optimal +VAT threshold of \citet{keenmintz2004}, thresholds as partitions of compliance +in \citet{kanburkeen2014}, and optimal remittance with firm-level +administrative costs in \citet{dharmapala2011}, together with the structural +literature on size-dependent policy \citep{garicano2016,gourioroys2014}; the +Mirrlees Review's VAT chapter frames the UK debate \citep{crawfordkeensmith2010}. +The present paper makes no comparable structural claim. Firm-level tax +microsimulation has precedents in corporate-tax models such as the German +BizTax model \citep{fossenbach2008}; to my knowledge no open firm-level model +of the UK VAT threshold exists, which is the gap this paper fills. This paper makes three contributions. First, it provides an open and auditable firm-level accounting microsimulation for conditional VAT-threshold scenarios, diff --git a/paper/Sections/model.tex b/paper/Sections/model.tex index 169a6f2..4775eb1 100644 --- a/paper/Sections/model.tex +++ b/paper/Sections/model.tex @@ -59,6 +59,11 @@ \subsection{The firm's problem} \emph{marginal} rate changes at the threshold while average liability moves continuously---is the defining feature of a notch: it makes locating just below the cutoff strictly attractive and creates the empty band derived next. +Throughout, the firm is assumed to bear the full statutory rate on its value +added---no pass-through to prices---so the notch geometry is stated in terms +of the firm's own net revenue; with partial pass-through the cash size of the +jump, and the width derived next, would shrink in proportion to the share +borne by the firm. \subsection{The dominated region} \label{ssec:dominated} @@ -107,19 +112,19 @@ \subsection{The dominated region} (the Scope paragraph above). Second, the region is an order of magnitude wider than the $\sim\pounds1$k region implied by a smoothed \emph{kink} approximation, and it spans a populous part of the firm distribution: about -$157{,}000$ weighted firms in the calibrated synthetic population lie in the -$\pounds85{,}000$--$\pounds106{,}250$ band -(\texttt{results/dominated\_region\_mass.txt}). Because the generator has no -location choice, this counts the firms the data \emph{place} in the -band---indexing the size of the population exposed to the distortion---rather -than firms that optimally locate there; the count is conditional on the -uniform within-band fill inside the coarse ONS bands (Section~\ref{sec:data}) -and would shift under a different within-band shape. It is the width $a$, not -the count, that is exact. +142{,}900 weighted in-scope firms in the calibrated synthetic population lie +in the $\pounds85{,}000$--$\pounds106{,}250$ band +(\texttt{results/dominated\_region\_mass.txt}; out-of-scope enterprises face +no notch and are excluded). Because the generator has no location choice, this +counts the firms the data \emph{place} in the band---indexing the size of the +population exposed to the distortion---rather than firms that optimally locate +there; the count is conditional on the uniform within-band fill inside the +coarse ONS bands (Section~\ref{sec:data}) and would shift under a different +within-band shape. It is the width $a$, not the count, that is exact. \begin{figure}[htbp]\centering \includegraphics[width=0.96\textwidth]{figures/dynamic_notch_fit_e017.png} -\caption{The VAT notch and the firm distribution. Panel (a) shows net revenue before production costs: immediately above \pounds85{,}000, VAT reduces it from $y$ to $0.8y$, making turnover in $(\pounds85{,}000,\,\pounds106{,}250)$ strictly dominated. Panel (b) shows the three-bin moving mean of weighted 2023--24 synthetic firm counts, with the same interval shaded. The distribution is descriptive and target-inherited, not the outcome of behavioural optimisation; its step at \pounds100{,}000 reflects the boundary between coarse ONS source bands, not an empty turnover interval. The exact dominated width is $a=\pounds21{,}250$.} +\caption{The VAT notch and the firm distribution. Panel (a) shows net revenue before production costs: immediately above \pounds85{,}000, VAT reduces it from $y$ to $0.8y$, making turnover in $(\pounds85{,}000,\,\pounds106{,}250)$ strictly dominated. Panel (b) shows the three-bin moving mean of weighted 2023--24 in-scope synthetic firm counts, with the same interval shaded. The distribution is descriptive and target-inherited, not the outcome of behavioural optimisation; its step at \pounds100{,}000 reflects the boundary between coarse ONS source bands, not an empty turnover interval. The exact dominated width is $a=\pounds21{,}250$.} \label{fig:notch_fit} \end{figure} @@ -130,8 +135,11 @@ \subsection{The dominated region} of~\eqref{eq:dominated} and~\eqref{eq:dominated-secondary}, with no dependence on the cost specification beyond monotonicity, any behavioural elasticity, or any simulated re-optimisation. Raising the threshold to \pounds100{,}000 leaves the rate -unchanged and therefore \emph{relocates} the region without altering its width: it -is the same \pounds21{,}250 band, transported to the new cutoff. A banded reduced +unchanged and therefore \emph{relocates} the region---and, because the width is +proportional to the threshold, \emph{widens} it: at $T^{*}=\pounds100{,}000$, +$a=\pounds100{,}000\times0.20/0.80=\pounds25{,}000$, so the dominated region +becomes $(\pounds100{,}000,\ \pounds125{,}000)$. Only the proportional width +$a/T^{*}=25\%$ is unchanged; the absolute band grows by \pounds3{,}750. A banded reduced rate does \emph{not} remove the distortion. It reduces the \emph{largest single} notch---the jump at $T^{*}$ falls from $\tau T^{*}=\pounds17{,}000$ to $r\,T^{*}=\pounds12{,}750$ at a $15\%$ band rate---but, because the band reverts to @@ -149,33 +157,44 @@ \subsection{The dominated region} \end{equation} giving $\pounds105{,}000\times0.05/0.80=\pounds6{,}562$ at $15\%$ and $\pounds105{,}000\times0.10/0.80=\pounds13{,}125$ at $10\%$. The \emph{total} -dominated-turnover width is therefore $a+a'=\pounds21{,}562$ at $15\%$ and +dominated-turnover width is therefore $a+a'=\pounds21{,}563$ at $15\%$ and $\pounds22{,}569$ at $10\%$---both marginally \emph{above} the $\pounds21{,}250$ baseline, not below it. The reduced rate splits one wide empty band into two narrower ones rather than shrinking the total: it lowers the largest single notch (to $\pounds12{,}750$ at $15\%$) at the price of adding a secondary notch of $(\tau-r)\,T_{1}=\pounds5{,}250$. The firm counts confirm this (\texttt{results/dominated\_region\_mass.txt}): the primary regions hold about -$111{,}900$ ($15\%$) and $77{,}300$ ($10\%$) weighted firms, the secondary -regions add about $34{,}200$ and $65{,}500$, and the totals---$146{,}100$ and -$142{,}900$---sit within $3.3\%$ of the $147{,}700$-firm baseline band. A -graduated taper, by contrast, can remove the dominated region entirely---provided -its schedule keeps net revenue monotone, and that condition is restrictive. Any -band-confined schedule whose effective rate reaches the full $\tau$ by -\pounds105{,}000 leaves net revenue +117{,}900 ($15\%$) and 74{,}400 ($10\%$) weighted in-scope firms, the +secondary regions add about 26{,}300 and 51{,}400, and the +totals---144{,}200 and 125{,}800---lie 0.9\% above and 11.9\% +below the 142{,}900-firm baseline band. The total dominated \emph{width} is +larger under both reduced rates; the firm count is lower at $10\%$ only because +the density declines across the wider secondary band, not because any part of +the distortion is removed. A graduated taper, by contrast, can remove the +dominated region entirely---provided its schedule keeps net revenue monotone, +and that condition is restrictive. Any band-confined schedule whose effective +rate reaches the full $\tau$ by \pounds105{,}000 leaves net revenue $(1-\tau)\times\pounds105{,}000=\pounds84{,}000$ at the band top, below the \pounds85{,}000 available at the threshold itself, so net revenue must fall over some range inside the band---a smoothly dominated interval in place of the -cliff. The taper costed here therefore phases the \emph{marginal} remittance -rate linearly from $0$ to $100\%$ over -$[\pounds85{,}000,\,T^{*}/(1-2\tau)]=[\pounds85{,}000,\pounds141{,}667]$: -liability is continuous at both band edges, net revenue is non-decreasing -throughout---its slope $1-m(y)$ vanishes only at the band top---and the band of -dominated turnover, with the discrete incentive to suppress turnover, is removed -entirely. The price of removal is a wide band and a marginal remittance rate -that reaches $100\%$ at its top. On the dominated-region metric, +cliff. Monotonicity requires the marginal remittance rate $m(y)$ to stay below +one, and band confinement requires liability to meet the standard-rate line +$\tau y$ at the band top $U$. For a constant marginal rate $m>\tau$ these two +conditions pin $U(m)=mT^{*}/(m-\tau)$: the band top is a property of the +schedule's \emph{shape}, falling from \pounds141{,}667 at $m=50\%$ towards +the infimum $T^{*}/(1-\tau)=\pounds106{,}250$ as $m\to100\%$, where net +revenue is flat across the band and the region is only weakly dominated. The +linear taper costed in Section~\ref{ssec:schedule_costs} phases $m$ from $0$ to +$100\%$ over $[\pounds85{,}000,\,T^{*}/(1-2\tau)]=[\pounds85{,}000,\pounds141{,}667]$; +the constant-rate taper costed alongside it holds $m=50\%$ over the same band. +Both leave liability continuous at both band edges and net revenue +non-decreasing throughout, so the band of dominated turnover, with the discrete +incentive to suppress turnover, is removed entirely. The price of removal is a +wide band and, for the linear design, a marginal remittance rate that reaches +$100\%$ at its top; the constant-rate design caps the marginal rate at $50\%$ +and costs a third less. On the dominated-region metric, then, only the taper removes the distortion; the reduced rate splits it and the -level move relocates it. The metric indexes the misallocation distortion +level move relocates and widens it. The metric indexes the misallocation distortion only---what it omits, and how the reforms compare on cost, is taken up in Section~\ref{sec:conclusion}. The dominated region is the paper's exact, elasticity-free measure of the distortion the notch imposes, and the analytic diff --git a/paper/Sections/static.tex b/paper/Sections/static.tex index 62ed9a7..8caec62 100644 --- a/paper/Sections/static.tex +++ b/paper/Sections/static.tex @@ -4,49 +4,59 @@ \section{Static costing of threshold reforms} A static costing measures the first-order revenue effect of a reform by re-applying the tax rule to a fixed firm population, with no behavioural response. Let firm $i$ have turnover $y_i$, calibrated net VAT liability $v_i = \tau(y_i - x_i)$ -(Section~\ref{sec:data}), and weight $w_i$. A firm remits VAT only once registered, -which the model equates with turnover above the threshold $T$, so total VAT revenue -at threshold $T$ is the weighted sum of liabilities over registered firms, +(Section~\ref{sec:data}), weight $w_i$, scope flag $s_i$ (one for firms in +the VAT net), data-year registration flag $q_i$, and baseline +voluntary-registration flag $r_i$ (one for firms registered with data-year +turnover at or below the data-year threshold $T_0$). A firm remits VAT only +once registered. Under a counterfactual threshold $T$, evaluated on turnover +$y_i$ aged to the fiscal year, an in-scope firm is registered when its turnover +is at or above $T$; a firm registered at the data year stays registered until +its turnover falls below the deregistration threshold $T-g$, where +$g=\pounds2{,}000$ is the statutory gap; a voluntary registrant is unchanged +where it was never required to register ($y_i\tau$ meets the standard-rate line at +$U(m)=mT^{*}/(m-\tau)$, which is \pounds141{,}667 at $m=50\%$ and falls +towards \pounds106{,}250 as $m\to100\%$, while a taper reaching the full rate at +\pounds105{,}000 would push net revenue inside the band below its value at the +threshold, recreating a dominated interval smoothly rather than at a cliff. I +cost two designs on the same \pounds85{,}000--\pounds141{,}667 band: the +\emph{linear} taper, whose marginal rate rises from $0\%$ to $100\%$ across the +band, and a \emph{constant} $50\%$ marginal-rate taper. Firms in the band remit +the tapered schedule on their (fixed) turnover; mechanically the linear design +reduces revenue by \(-\)\pounds1{,}406m and the constant-rate design by \(-\)\pounds968m, with +about 275{,}800 in-scope firms in the band facing a lower effective rate. +The linear design gives more relief low in the band at the price of a marginal +remittance rate that reaches $100\%$ at the top; the constant-rate design caps +the marginal rate at $50\%$ and costs a third less. \paragraph{Banded reduced rate.} A banded reduced rate charges firms in the \pounds85{,}000--\pounds105{,}000 band a reduced rate $\tau_{\text{low}}\in\{10\%,15\%\}$ in place of the standard $20\%$, with $20\%$ -retained above \pounds105{,}000. Statically this reduces revenue by \(-\)\pounds515m -at $\tau_{\text{low}}=10\%$ and by \(-\)\pounds258m at $\tau_{\text{low}}=15\%$, -again with about $151{,}600$ firms in the band facing the lower rate. +retained above \pounds105{,}000. Statically this reduces revenue by \(-\)\pounds446m +at $\tau_{\text{low}}=10\%$ and by \(-\)\pounds223m at $\tau_{\text{low}}=15\%$, +with about 138{,}000 in-scope firms in the band facing the lower rate. \begin{table}[htbp] \centering -\caption{Static, behaviour-free revenue effect of the four reforms relative to the -common \pounds85{,}000-notch / \pounds175.4bn 2023--24 baseline. Each reform is -costed mechanically by re-applying its schedule to the fixed turnover distribution, -with no behavioural response.} +\caption{Static, behaviour-free revenue effect of the reforms relative to the +common \pounds85{,}000-notch / \pounds175.7bn 2023--24 baseline of in-scope +firms. Each reform is costed mechanically by re-applying its schedule to the +fixed turnover distribution, with no behavioural response.} \label{tab:schedule_costs} -\begin{tabular}{llr} +\small +\begin{tabularx}{\textwidth}{@{}Xlr@{}} \toprule -Reform & Lever & Static revenue (\pounds m) \\ +Reform & Lever & Static (\pounds m) \\ \midrule -Raise threshold to \pounds100{,}000 & Location & $-876$ \\ -Graduated taper $[\pounds85\text{k},\pounds141.7\text{k}]$ & Shape (phase-in) & $-1{,}525$ \\ -Reduced rate $10\%$ $[\pounds85\text{k},\pounds105\text{k}]$ & Rate (step) & $-515$ \\ -Reduced rate $15\%$ $[\pounds85\text{k},\pounds105\text{k}]$ & Rate (step) & $-258$ \\ +Raise threshold to \pounds100{,}000 & Location & $-742$ \\ +Linear taper, \pounds85k--\pounds141.7k & Shape (marginal rate $0\to100\%$) & $-1{,}406$ \\ +Constant $50\%$ marginal taper, \pounds85k--\pounds141.7k & Shape (marginal rate $50\%$) & $-968$ \\ +Reduced rate $10\%$, \pounds85k--\pounds105k & Rate (step) & $-446$ \\ +Reduced rate $15\%$, \pounds85k--\pounds105k & Rate (step) & $-223$ \\ \bottomrule -\end{tabular} +\end{tabularx} \end{table} These static figures bound the first-order fiscal effect of each reform but -abstract from the behavioural response of Section~\ref{sec:behavioural}. One feature +abstract from the behavioural response of Section~\ref{sec:behavioural}. Two +maintained conventions carry through every costing: the firm bears the full +statutory rate on its value added (no pass-through to prices), and per-firm +liability is the standard rate on value added, so Flat Rate Scheme users---who +are concentrated in exactly the \pounds85{,}000--\pounds150{,}000 bands---are +represented only through the calibrated band totals, not individually. One feature of the shape reforms is not visible in a static costing: unlike a threshold move, which transports the notch to a new cutoff, the taper and the reduced rate \emph{change the size of the notch itself}---the discrete drop in net revenue at the diff --git a/paper/YGTemplate.sty b/paper/YGTemplate.sty index 89325ce..a8f65f2 100644 --- a/paper/YGTemplate.sty +++ b/paper/YGTemplate.sty @@ -8,6 +8,7 @@ \usepackage[usenames,dvipsnames]{xcolor} \usepackage{graphicx} \usepackage{tabularx} +\usepackage{array} \usepackage{subfigure} \usepackage{float} \usepackage{caption} diff --git a/paper/figures/bunching_analysis_85k.png b/paper/figures/bunching_analysis_85k.png index ece5b32..56ef187 100644 Binary files a/paper/figures/bunching_analysis_85k.png and b/paper/figures/bunching_analysis_85k.png differ diff --git a/paper/figures/bunching_analysis_90k.png b/paper/figures/bunching_analysis_90k.png index 825676f..32cb2f6 100644 Binary files a/paper/figures/bunching_analysis_90k.png and b/paper/figures/bunching_analysis_90k.png differ diff --git a/paper/figures/dynamic_cost_vs_elasticity.png b/paper/figures/dynamic_cost_vs_elasticity.png index 9bf1d73..27f8165 100644 Binary files a/paper/figures/dynamic_cost_vs_elasticity.png and b/paper/figures/dynamic_cost_vs_elasticity.png differ diff --git a/paper/figures/dynamic_notch_fit_e005.png b/paper/figures/dynamic_notch_fit_e005.png index c22053d..d27348b 100644 Binary files a/paper/figures/dynamic_notch_fit_e005.png and b/paper/figures/dynamic_notch_fit_e005.png differ diff --git a/paper/figures/dynamic_notch_fit_e017.png b/paper/figures/dynamic_notch_fit_e017.png index c22053d..d27348b 100644 Binary files a/paper/figures/dynamic_notch_fit_e017.png and b/paper/figures/dynamic_notch_fit_e017.png differ diff --git a/paper/figures/dynamic_notch_fit_e032.png b/paper/figures/dynamic_notch_fit_e032.png index c22053d..d27348b 100644 Binary files a/paper/figures/dynamic_notch_fit_e032.png and b/paper/figures/dynamic_notch_fit_e032.png differ diff --git a/paper/figures/dynamic_reform_distribution_raisethresholdto100k_e017.png b/paper/figures/dynamic_reform_distribution_raisethresholdto100k_e017.png index a09f6df..c5bc0b2 100644 Binary files a/paper/figures/dynamic_reform_distribution_raisethresholdto100k_e017.png and b/paper/figures/dynamic_reform_distribution_raisethresholdto100k_e017.png differ diff --git a/paper/figures/dynamic_reform_distribution_reducedrate1085k105k_e017.png b/paper/figures/dynamic_reform_distribution_reducedrate1085k105k_e017.png index f4c55cd..a04ca5a 100644 Binary files a/paper/figures/dynamic_reform_distribution_reducedrate1085k105k_e017.png and b/paper/figures/dynamic_reform_distribution_reducedrate1085k105k_e017.png differ diff --git a/paper/figures/dynamic_reform_distribution_reducedrate1585k105k_e017.png b/paper/figures/dynamic_reform_distribution_reducedrate1585k105k_e017.png index 392cb4b..ed6cb27 100644 Binary files a/paper/figures/dynamic_reform_distribution_reducedrate1585k105k_e017.png and b/paper/figures/dynamic_reform_distribution_reducedrate1585k105k_e017.png differ diff --git a/paper/figures/firms_impact_2025_26.png b/paper/figures/firms_impact_2025_26.png index 7ed1b0b..24dce0c 100644 Binary files a/paper/figures/firms_impact_2025_26.png and b/paper/figures/firms_impact_2025_26.png differ diff --git a/paper/figures/revenue_impact_2025_26.png b/paper/figures/revenue_impact_2025_26.png index 2459d4e..b35d44f 100644 Binary files a/paper/figures/revenue_impact_2025_26.png and b/paper/figures/revenue_impact_2025_26.png differ diff --git a/paper/figures/turnover_distribution_85k.png b/paper/figures/turnover_distribution_85k.png index 6204b10..4c6bd43 100644 Binary files a/paper/figures/turnover_distribution_85k.png and b/paper/figures/turnover_distribution_85k.png differ diff --git a/paper/figures/vat_threshold_revenue_impact.png b/paper/figures/vat_threshold_revenue_impact.png index 87fc9ca..c2cca82 100644 Binary files a/paper/figures/vat_threshold_revenue_impact.png and b/paper/figures/vat_threshold_revenue_impact.png differ diff --git a/paper/main.pdf b/paper/main.pdf index ab42c63..5f1a412 100644 Binary files a/paper/main.pdf and b/paper/main.pdf differ diff --git a/paper/references.bib b/paper/references.bib index 78a89eb..bb9a2b6 100644 --- a/paper/references.bib +++ b/paper/references.bib @@ -244,6 +244,13 @@ @misc{eudirective2020285 year = {2020} } +@misc{hmt_spring_budget_2024, + author = {{HM Treasury}}, + title = {Spring Budget 2024: Policy Costings}, + howpublished = {HM Treasury, March 2024, \url{https://assets.publishing.service.gov.uk/media/65e7920c08eef600155a5617/Published_Costing_Document_Spring_Budget_2024_Final.pdf}}, + year = {2024} +} + @article{keenmintz2004, author = {Keen, Michael and Mintz, Jack}, title = {The Optimal Threshold for a Value-Added Tax}, @@ -255,9 +262,43 @@ @article{keenmintz2004 url = {https://doi.org/10.1016/S0047-2727(02)00165-2} } -@misc{hmt_spring_budget_2024, - author = {{HM Treasury}}, - title = {Spring Budget 2024: Policy Costings}, - howpublished = {HM Treasury, March 2024, \url{https://assets.publishing.service.gov.uk/media/65e7920c08eef600155a5617/Published_Costing_Document_Spring_Budget_2024_Final.pdf}}, - year = {2024} +@article{kanburkeen2014, + author = {Kanbur, Ravi and Keen, Michael}, + title = {Thresholds, Informality, and Partitions of Compliance}, + journal = {International Tax and Public Finance}, + volume = {21}, + number = {4}, + pages = {536--559}, + year = {2014} +} + +@article{dharmapala2011, + author = {Dharmapala, Dhammika and Slemrod, Joel and Wilson, John Douglas}, + title = {Tax Policy and the Missing Middle: Optimal Tax Remittance with Firm-Level Administrative Costs}, + journal = {Journal of Public Economics}, + volume = {95}, + number = {9--10}, + pages = {1036--1047}, + year = {2011} +} + +@incollection{crawfordkeensmith2010, + author = {Crawford, Ian and Keen, Michael and Smith, Stephen}, + title = {Value Added Tax and Excises}, + booktitle = {Dimensions of Tax Design: The Mirrlees Review}, + editor = {Adam, Stuart and Besley, Timothy and Blundell, Richard and Bond, Stephen and Chote, Robert and Gammie, Malcolm and Johnson, Paul and Myles, Gareth and Poterba, James}, + publisher = {Oxford University Press}, + address = {Oxford}, + pages = {275--362}, + year = {2010} +} + +@article{fossenbach2008, + author = {Fossen, Frank M. and Bach, Stefan}, + title = {Reforming the German Local Business Tax---Lessons from an International Comparison and a Microsimulation Model}, + journal = {FinanzArchiv / Public Finance Analysis}, + volume = {64}, + number = {2}, + pages = {245--272}, + year = {2008} } diff --git a/results/bunching_analysis_85k.png b/results/bunching_analysis_85k.png index ece5b32..56ef187 100644 Binary files a/results/bunching_analysis_85k.png and b/results/bunching_analysis_85k.png differ diff --git a/results/bunching_analysis_90k.png b/results/bunching_analysis_90k.png index 825676f..32cb2f6 100644 Binary files a/results/bunching_analysis_90k.png and b/results/bunching_analysis_90k.png differ diff --git a/results/bunching_inference.txt b/results/bunching_inference.txt index 32925c8..f38430e 100644 --- a/results/bunching_inference.txt +++ b/results/bunching_inference.txt @@ -5,45 +5,45 @@ results/seed_sensitivity.txt for generator-seed dispersion. ========================================================================== --- Vintage 2023-24 (threshold GBP 85k) --- -E = 10,418 | Delta_R = 0 | b_llat = 5.127 | b = 0.0027 | y_R = 100.00 +E = 5,498 | Delta_R = 0 | b_llat = 5.357 | b = -0.0340 | y_R = 100.00 Degree x window sensitivity (point estimates): degree window b b_llat E - 5 10.0 0.079882 2.866198 9750.100754 - 5 15.0 0.048467 4.349707 11650.970478 - 5 20.0 -0.004138 3.384365 9164.917298 - 5 25.0 0.010881 3.209816 12032.094833 - 6 10.0 0.025509 2.120250 6880.561112 - 6 15.0 -0.038539 2.773461 6480.495885 - 6 20.0 -0.168191 0.466625 2396.641232 - 6 25.0 -0.257101 0.000000 0.000000 - 7 10.0 0.036373 2.536807 7693.799763 - 7 15.0 0.002709 5.127411 10417.683518 - 7 20.0 -0.118663 3.290933 5460.608730 - 7 25.0 -0.191798 2.423907 2543.353901 - 8 10.0 0.141743 3.951786 13692.975879 - 8 15.0 0.185809 8.983101 25095.937444 - 8 20.0 0.020342 6.149543 15112.418293 - 8 25.0 -0.119614 3.026539 5276.011563 + 5 10.0 0.002257 2.696852 4858.733251 + 5 15.0 0.047139 5.964198 8968.633823 + 5 20.0 0.061287 6.393184 13227.968183 + 5 25.0 0.070427 6.978502 17796.560304 + 6 10.0 -0.071034 1.926292 3083.123056 + 6 15.0 -0.055711 4.238768 4225.108314 + 6 20.0 -0.094656 3.196188 2787.874606 + 6 25.0 -0.191206 0.057251 559.000049 + 7 10.0 -0.076979 1.747737 2805.556451 + 7 15.0 -0.033953 5.356673 5497.569032 + 7 20.0 -0.067862 4.538012 4185.857987 + 7 25.0 -0.154326 3.085122 1789.778254 + 8 10.0 -0.028454 2.079282 3922.400388 + 8 15.0 0.070561 7.087564 11722.064404 + 8 20.0 0.108166 8.399882 20751.886320 + 8 25.0 -0.014974 5.367707 7108.684820 --- Vintage 2024-25 (threshold GBP 90k) --- -E = 25,227 | Delta_R = 5,961 | b_llat = 4.507 | b = 0.1642 | y_R = 105.00 +E = 10,273 | Delta_R = 4,890 | b_llat = 4.971 | b = 0.0715 | y_R = 105.00 Degree x window sensitivity (point estimates): degree window b b_llat E - 5 10.0 0.225324 3.374602 21885.459405 - 5 15.0 0.179098 3.810178 27164.034349 - 5 20.0 0.137885 3.777217 28931.443999 - 5 25.0 0.089961 2.838452 24643.900538 - 6 10.0 0.138787 2.284703 14504.599637 - 6 15.0 0.078827 1.909708 13197.384372 - 6 20.0 0.011648 0.958756 7668.804952 - 6 25.0 -0.083108 0.099154 1283.700208 - 7 10.0 0.204912 3.810282 20240.029376 - 7 15.0 0.164232 4.507400 25227.367185 - 7 20.0 0.131471 5.178809 28184.013948 - 7 25.0 0.079078 5.365649 26638.818183 - 8 10.0 0.224645 3.932580 21831.560472 - 8 15.0 0.211836 5.101957 31261.533636 - 8 20.0 0.209662 6.389781 41381.556229 - 8 25.0 0.162985 6.154408 42191.230543 + 5 10.0 0.112471 4.456726 10305.294222 + 5 15.0 0.090476 4.662462 12701.146708 + 5 20.0 0.065563 4.436122 12640.014728 + 5 25.0 0.046132 3.955023 11368.794427 + 6 10.0 0.056280 3.656349 5430.984593 + 6 15.0 0.016812 3.199315 3816.197789 + 6 20.0 -0.039323 2.413070 1370.053558 + 6 25.0 -0.114371 0.000000 0.000000 + 7 10.0 0.095467 4.686727 8883.034460 + 7 15.0 0.071476 4.970983 10273.292276 + 7 20.0 0.038103 5.046449 9781.774956 + 7 25.0 -0.000405 5.323075 8843.110715 + 8 10.0 0.130339 4.916060 11753.683094 + 8 15.0 0.131950 5.688830 17844.708802 + 8 20.0 0.123932 6.095205 22532.133191 + 8 25.0 0.090477 5.536228 21484.568265 diff --git a/results/calibration_accuracy.txt b/results/calibration_accuracy.txt index d3e2dc0..aa6ec35 100644 --- a/results/calibration_accuracy.txt +++ b/results/calibration_accuracy.txt @@ -8,52 +8,52 @@ dimensions and excludes the VAT-liability-by-sector diagnostic. Vintage 2023-24 | threshold £85k ================================================================ rows (firm types): 2,941,275 - weighted population: 2,407,267 firms + weighted ONS-frame population: 2,717,625 firms (registered traders 2,178,286) ---------------------------------------------------------------- Dimension accuracy error ---------------------------------------------------------------- - HMRC Turnover Bands 97.3% 2.7% - ONS Population 88.3% 11.7% - Employment Bands 92.1% 7.9% - Sector Distribution 90.1% 9.9% - VAT Liability by Band 96.5% 3.5% + HMRC Turnover Bands 100.0% 0.0% + ONS Population 99.7% 0.3% + Employment Bands 99.8% 0.2% + Sector Distribution 93.3% 6.7% + VAT Liability by Band 97.1% 2.9% ---------------------------------------------------------------- - Overall (5 calibrated dims) 92.9% 7.1% + Overall (5 calibrated dims) 98.0% 2.0% ---------------------------------------------------------------- Informational diagnostics (not calibration targets): - VAT Liability by Sector 44.0% 56.0% - VAT Liability below Thresh. 34.7% 65.3% + VAT Liability by Sector 42.2% 57.8% + VAT Liability below Thresh. 31.2% 68.8% ================================================================ Calibration-weight diagnostics: - effective sample size: 93,887 (3.2% of rows) - min / p50 / p90 / p99 / max: 0.065 / 0.711 / 1.000 / 3.630 / 2905.395 - coefficient of variation: 5.507 + effective sample size: 137,054 (4.7% of rows) + min / p50 / p90 / p99 / max: 0.082 / 0.999 / 1.002 / 2.588 / 2446.253 + coefficient of variation: 4.523 ================================================================ ================================================================ Vintage 2024-25 | threshold £90k ================================================================ rows (firm types): 2,946,015 - weighted population: 2,490,407 firms + weighted ONS-frame population: 2,729,522 firms (registered traders 2,170,779) ---------------------------------------------------------------- Dimension accuracy error ---------------------------------------------------------------- - HMRC Turnover Bands 96.6% 3.4% - ONS Population 91.1% 8.9% - Employment Bands 92.3% 7.7% - Sector Distribution 90.4% 9.6% - VAT Liability by Band 97.1% 2.9% + HMRC Turnover Bands 100.0% 0.0% + ONS Population 99.8% 0.2% + Employment Bands 99.9% 0.1% + Sector Distribution 92.2% 7.8% + VAT Liability by Band 98.0% 2.0% ---------------------------------------------------------------- - Overall (5 calibrated dims) 93.5% 6.5% + Overall (5 calibrated dims) 98.0% 2.0% ---------------------------------------------------------------- Informational diagnostics (not calibration targets): - VAT Liability by Sector 45.2% 54.8% + VAT Liability by Sector 45.7% 54.3% VAT Liability below Thresh. 0.0% 100.0% ================================================================ Calibration-weight diagnostics: - effective sample size: 275,186 (9.3% of rows) - min / p50 / p90 / p99 / max: 0.076 / 0.779 / 1.001 / 3.160 / 879.987 - coefficient of variation: 3.115 + effective sample size: 192,917 (6.5% of rows) + min / p50 / p90 / p99 / max: 0.085 / 1.000 / 1.001 / 2.924 / 1830.364 + coefficient of variation: 3.778 ================================================================ Done: 2/2 vintage(s) reported. diff --git a/results/dominated_region_mass.txt b/results/dominated_region_mass.txt index 81f5c89..3e86523 100644 --- a/results/dominated_region_mass.txt +++ b/results/dominated_region_mass.txt @@ -4,28 +4,29 @@ vintage = 2023-24 T* = GBP 85,000 tau = 0.20 ========================================================================== Dominated region (Kleven-Waseem): a = T* * tau/(1-tau) -Bands measured on the WEIGHTED synthetic firm population. +Bands measured on the WEIGHTED in-scope (VAT-liable) synthetic firms; + out-of-scope PAYE-only/exempt enterprises face no notch and are excluded. OBS uses exact band masks on the microdata; CF integrates the binned counterfactual with fractional edge-bin overlap. OBS = observed weighted firms in band CF = weighted firms the mass-conserving no-bunching counterfactual density places in band (total smooth density across the band) - NET = CF - OBS = net MISSING (notch-displaced) mass in the band; this - is the firms the notch removed from the dominated region -- the - quantity with empirical bite, consistent with excess mass E. + NET = CF - OBS: positive = mass the counterfactual places in the band + beyond what is observed (missing/displaced); NEGATIVE = observed + surplus over the smooth counterfactual (no displacement read). rate / band width a (GBP) band (GBP k) OBS CF NET disp. ------------------------------------------------------------------------------------------ -20% (baseline notch) 21,250 [85, 106.250) 156,747 131,072 -25,675 -15% band 15,000 [85, 100.000) 130,991 100,171 -30,820 -10% band 9,444 [85, 94.444) 88,767 68,779 -19,987 +20% (baseline notch) 21,250 [85, 106.250) 142,860 131,749 -11,111 +15% band 15,000 [85, 100.000) 117,879 101,373 -16,506 +10% band 9,444 [85, 94.444) 74,403 69,584 -4,819 ------------------------------------------------------------------------------------------ INTERPRETATION (baseline 20% notch, band [85,000, 106,250)): - Observed firms in dominated region ........ 156,747 - Counterfactual total density in band (CF).. 131,072 - NET displaced mass in band (CF - OBS) ..... -25,675 - CF mass as share of firms near threshold .. 0.414 + Observed firms in dominated region ........ 142,860 + Counterfactual total density in band (CF).. 131,749 + NET displaced mass in band (CF - OBS) ..... -11,111 + CF mass as share of firms near threshold .. 0.435 Note: the headline displaced-mass number is NET (CF - OBS), not the total CF density. The band is wide, so most CF firms in it would @@ -33,23 +34,25 @@ INTERPRETATION (baseline 20% notch, band [85,000, 106,250)): NET deficit is the mass the notch evacuates from the dominated region. REFORM-SHRUNK BANDS (net displaced mass): - 15% -> band width GBP 15,000: net displaced = -30,820 firms (CF total 100,171) - 10% -> band width GBP 9,444: net displaced = -19,987 firms (CF total 68,779) + 15% -> band width GBP 15,000: net displaced = -16,506 firms (CF total 101,373) + 10% -> band width GBP 9,444: net displaced = -4,819 firms (CF total 69,584) SECONDARY NOTCH at the reduced-rate band top (T1 = GBP 105,000): A banded reduced rate reverts to tau=20% at T1, adding a SECOND dominated region a' = T1*(tau-r)/(1-tau). Total dominated turnover and mass = primary [T*, T*+a] + secondary [T1, T1+a']. - 15% band secondary [105, 111.562) width GBP 6,563 OBS = 26,682 - primary OBS 130,991 + secondary OBS 26,682 = TOTAL 157,674 (baseline 156,747, +0.6%) - 10% band secondary [105, 118.125) width GBP 13,125 OBS = 52,517 - primary OBS 88,767 + secondary OBS 52,517 = TOTAL 141,284 (baseline 156,747, -9.9%) - -REDUCED-FORM BUNCHING on this population (context for the masses above): - excess mass below T* E = 10,418 firms - missing mass above T* Delta_R = 0 firms + 15% band secondary [105, 111.562) width GBP 6,563 OBS = 26,309 + primary OBS 117,879 + secondary OBS 26,309 = TOTAL 144,188 (baseline 142,860, +0.9%) + 10% band secondary [105, 118.125) width GBP 13,125 OBS = 51,423 + primary OBS 74,403 + secondary OBS 51,423 = TOTAL 125,826 (baseline 142,860, -11.9%) + +REDUCED-FORM BUNCHING on the IN-SCOPE density (context for the masses above; + NOT the Section 6 headline, which runs on the full ONS-frame density): + frame-density headline (Section 6): E = 5,498, Delta_R = 0, y_R = GBP 100,000 + in-scope excess mass below T* E = 11,364 firms + missing mass above T* Delta_R = 2,666 firms marginal buncher y_R = GBP 100,000 - NET displaced mass in 20% dominated band = -25,675 firms + NET displaced mass in 20% dominated band = -11,111 firms By mass conservation the excess mass E that bunches just below T* is the mass that, absent the notch, would have spread into the @@ -58,6 +61,6 @@ REDUCED-FORM BUNCHING on this population (context for the masses above): (The TOTAL CF density in the band is NOT the displaced mass and should not be compared to E -- that would be a category error.) - total observed weighted mass on [20,140] est. range = 959,170 + total observed weighted mass on [20,140] est. range = 872,014 script: firm-microsim-dominated-region diff --git a/results/dynamic_cost_vs_elasticity.png b/results/dynamic_cost_vs_elasticity.png index 9bf1d73..27f8165 100644 Binary files a/results/dynamic_cost_vs_elasticity.png and b/results/dynamic_cost_vs_elasticity.png differ diff --git a/results/dynamic_notch_fit_e005.png b/results/dynamic_notch_fit_e005.png index c22053d..d27348b 100644 Binary files a/results/dynamic_notch_fit_e005.png and b/results/dynamic_notch_fit_e005.png differ diff --git a/results/dynamic_notch_fit_e017.png b/results/dynamic_notch_fit_e017.png index c22053d..d27348b 100644 Binary files a/results/dynamic_notch_fit_e017.png and b/results/dynamic_notch_fit_e017.png differ diff --git a/results/dynamic_notch_fit_e032.png b/results/dynamic_notch_fit_e032.png index c22053d..d27348b 100644 Binary files a/results/dynamic_notch_fit_e032.png and b/results/dynamic_notch_fit_e032.png differ diff --git a/results/dynamic_reform_distribution_raisethresholdto100k_e017.png b/results/dynamic_reform_distribution_raisethresholdto100k_e017.png index a09f6df..c5bc0b2 100644 Binary files a/results/dynamic_reform_distribution_raisethresholdto100k_e017.png and b/results/dynamic_reform_distribution_raisethresholdto100k_e017.png differ diff --git a/results/dynamic_reform_distribution_reducedrate1085k105k_e017.png b/results/dynamic_reform_distribution_reducedrate1085k105k_e017.png index f4c55cd..a04ca5a 100644 Binary files a/results/dynamic_reform_distribution_reducedrate1085k105k_e017.png and b/results/dynamic_reform_distribution_reducedrate1085k105k_e017.png differ diff --git a/results/dynamic_reform_distribution_reducedrate1585k105k_e017.png b/results/dynamic_reform_distribution_reducedrate1585k105k_e017.png index 392cb4b..ed6cb27 100644 Binary files a/results/dynamic_reform_distribution_reducedrate1585k105k_e017.png and b/results/dynamic_reform_distribution_reducedrate1585k105k_e017.png differ diff --git a/results/dynamic_reform_results.txt b/results/dynamic_reform_results.txt index 307cd21..92cc928 100644 --- a/results/dynamic_reform_results.txt +++ b/results/dynamic_reform_results.txt @@ -1,30 +1,30 @@ Iso-elastic (Kleven-Waseem) dynamic VAT-notch simulator — reform results vintage=2023-24 T*=£85,000 tau_max=0.20 reduced-rate band top=£105,000 taper top=£141,667 -Repo-generated baseline registered VAT base = £175.355bn. +Repo-generated baseline registered VAT base = £175.692bn. A SINGLE turnover elasticity e governs the response. The behavioural cost is now a defensible e-SENSITIVITY RANGE over e in {0.05,0.17,0.32} (headline 0.17). Ability n is an accounting anchor recovered under the £85k notch given e; e is -NOT identified from the synthetic data (see the placebo) — read results as -CONDITIONAL on the assumed e. +NOT identified from the synthetic data — read results as CONDITIONAL on the assumed e. +Liabilities are in-scope only: out-of-scope (PAYE-only/exempt) enterprises remit nothing (issue #37). ============================================================================== -Marginal buncher n_H(e) [analytic; £]: - e=0.05 n_H=£ 112.8k dy*=£ 27.8k - e=0.17 n_H=£ 127.4k dy*=£ 42.4k - e=0.32 n_H=£ 143.5k dy*=£ 58.5k +Marginal buncher n_H(e, delta) [analytic; £]. delta = deductible-input +share: it scales revenue but not own-factor cost, so n_H rises with delta; +delta=0 is the turnover-tax solve, 0.6 the population mean input share. + e=0.05 n_H(delta=0)=£ 112.8k dy*=£ 27.8k n_H(delta=0.4)=£ 115.7k n_H(delta=0.6)=£ 118.1k + e=0.17 n_H(delta=0)=£ 127.4k dy*=£ 42.4k n_H(delta=0.4)=£ 138.9k n_H(delta=0.6)=£ 148.9k + e=0.32 n_H(delta=0)=£ 143.5k dy*=£ 58.5k n_H(delta=0.4)=£ 169.0k n_H(delta=0.6)=£ 192.4k Reform costs (change vs £85k-notch baseline): reform static e=0.05 e=0.17 e=0.32 #reopt(e=0.17) near-thr Δ - Raise threshold to £100k £ -876m £ -876m £ -876m £ -876m 127,261 +0 - Graduated taper (£85k→£141.7k, monotone)£ -1525m — — — - Reduced rate 10% (£85k–£105k) £ -515m £ -512m £ -505m £ -497m 149,581 +0 - Reduced rate 15% (£85k–£105k) £ -258m £ -255m £ -250m £ -243m 149,581 +0 + Raise threshold to £100k £ -742m £ -742m £ -742m £ -742m 114,452 +0 + Graduated taper (£85k→£141.7k, monotone)£ -1406m — — — + Flat 50% marginal taper (£85k→£141.7k)£ -968m — — — + Reduced rate 10% (£85k–£105k) £ -446m £ -443m £ -437m £ -430m 136,112 +0 + Reduced rate 15% (£85k–£105k) £ -223m £ -221m £ -216m £ -210m 136,112 +0 -Direction: a LARGER e makes every reform CHEAPER (smaller revenue loss). Each -reform lowers the effective rate on the band, so the iso-elastic response scales -turnover UP toward the firm's frictionless optimum (y* = y_obs*((1-tau_reform)/(1-tau_base))^e > y_obs), widening the taxed base; -the larger e, the larger this base-broadening offset to the static cost. -NOTE: the taper figure captures this intensive base-broadening only; it omits the -taper's marginal-rate distortion (the iso-elastic cost cannot credibly price the -downward response a taper induces), so the taper's behavioural cost is a lower -bound on its true cost. See Section 7 of the paper. +Reading: the raise row is flat in e by construction of the region-confined solve +(released firms leave the base; no other firm's rate changes). Reduced-rate bands +get cheaper as e rises: band firms scale turnover up by ((1-tau f1)/(1-tau f0))^e, +widening the taxed base. The tapers have no behavioural row: their rate varies +continuously with turnover, outside the region-confined flat-rate solve. diff --git a/results/firms_impact_2025_26.png b/results/firms_impact_2025_26.png index 7ed1b0b..24dce0c 100644 Binary files a/results/firms_impact_2025_26.png and b/results/firms_impact_2025_26.png differ diff --git a/results/notch_model_fit_85k.png b/results/notch_model_fit_85k.png index 5656b92..7e7c9c2 100644 Binary files a/results/notch_model_fit_85k.png and b/results/notch_model_fit_85k.png differ diff --git a/results/notch_model_fit_90k.png b/results/notch_model_fit_90k.png index 11de793..8da0b62 100644 Binary files a/results/notch_model_fit_90k.png and b/results/notch_model_fit_90k.png differ diff --git a/results/recovery_bunching.txt b/results/recovery_bunching.txt new file mode 100644 index 0000000..5ef0984 --- /dev/null +++ b/results/recovery_bunching.txt @@ -0,0 +1,39 @@ +RECOVERY / POWER TEST — £85k UK VAT bunching estimator (vintage 2023-24) +============================================================================== + +Design +------ +1. Baseline: the £85k synthetic firms under the Placebo-A reweighting + (smooth log-quadratic density across £85k), a step-free null world: + b=-0.1050, E=0, y_R=84.00. +2. Injection (Kleven–Waseem-consistent): firms in the donor region + (85, 112] relocate to [80, 85) with probability + declining linearly from the threshold to zero at the true marginal + buncher y_R = 112 — so the missing-mass region extends + 12k BEYOND the ±15k exclusion window, as with + genuine bunching. Mass is conserved. +3. Score with the HEADLINE estimator outputs only (floored excess mass + E, endogenous y_R): no side statistics that require knowing the null. + +Results +------- + E_true E_hat recovery y_R_hat b_hat b_llat +---------------------------------------------------------- + 2,000 0 0.0% 84.00 -0.0934 0.000 + 5,000 1,381 27.6% 87.06 -0.0760 0.155 + 8,000 3,593 44.9% 91.22 -0.0584 0.421 + +Baseline (null, no injection): b=-0.1050 E=0 y_R=84.00 +True marginal buncher: y_R = 112. Mean recovery: 24.2% (monotone in E_true: True). + +Reading +------- +Recovery below 100% is genuine estimator attenuation, now measured +rather than asserted: the counterfactual is fitted on bins that include +the depressed (100, 112] missing-mass region outside the exclusion +window, pulling the fitted counterfactual down and truncating the +measured missing mass; the mass-conservation search then places y_R_hat +accordingly. The estimator under-states a true response and does not +over-state one; together with the placebo's null result (no false +positive on a step-free world) this characterises the estimator as +specific and, with quantified attenuation, sensitive. diff --git a/results/reform_menu_common_base.txt b/results/reform_menu_common_base.txt index 8e886d2..a4b8f85 100644 --- a/results/reform_menu_common_base.txt +++ b/results/reform_menu_common_base.txt @@ -2,33 +2,35 @@ REFORM MENU ON A SINGLE COMMON BASE (tab:schedule_costs) Base: GBP85k notch, 2023-24 microdata, UNAGED ======================================================================== -Dataset : data/synthetic/synthetic_firms_2023-24.csv -VAT base (>=85k) : GBP 175.355 bn -Firms in [85k,105k): 151,636 +Dataset : data/synthetic/synthetic_firms_2023-24.csv (in-scope VAT firms only) +VAT base (>=85k) : GBP 175.692 bn +Firms in [85k,105k): 138,031 --- Threshold relocation GBP85k -> GBP100k (recomputed) ------------- - (A) DIRECT band-sum [85k,100k) : -876.5 m firms -131.0 (000s) - as % of VAT base : -0.500% - (B) SMOOTH counterfactual : -699.8 m firms -102.4 (000s) - HEADLINE (adopt A) : -876 m firms -131.0 (000s) + (A) DIRECT band-sum [85k,100k) : -742.2 m firms -117.9 (000s) + as % of VAT base : -0.422% + (B) SMOOTH counterfactual : -615.8 m firms -96.5 (000s) + HEADLINE (adopt A) : -742 m firms -117.9 (000s) --- Schedule reforms (computed on the SAME repo-generated base) ----- - Graduated taper [85k,141.7k] : -1525.0 m affected firms -290.6 (000s) - Reduced rate 10% [85k,105k] : -515.3 m affected firms -151.6 (000s) - Reduced rate 15% [85k,105k] : -257.6 m affected firms -151.6 (000s) + Graduated taper [85k,141.7k] : -1406.1 m affected firms -275.8 (000s) + Flat 50% marginal taper [85k,141.7k]: -968.0 m affected firms -275.8 (000s) + Reduced rate 10% [85k,105k] : -446.1 m affected firms -138.0 (000s) + Reduced rate 15% [85k,105k] : -223.1 m affected firms -138.0 (000s) ======================================================================== CORRECTED tab:schedule_costs (repo-generated GBP85k, 2023-24 unaged) ======================================================================== -Reform Lever Static (GBPm) --------------------------------------------------------------------------- -Raise threshold to GBP100,000 Location -876 -Graduated taper [85k,141.7k] Shape (phase-in) -1525 -Reduced rate 10% [85k,105k] Rate (step) -515 -Reduced rate 15% [85k,105k] Rate (step) -258 +Reform Lever Static (GBPm) +---------------------------------------------------------------------------------- +Raise threshold to GBP100,000 Location -742 +Graduated taper [85k,141.7k] Shape (phase-in) -1406 +Flat 50% marginal taper [85k,141.7k] Shape (constant marginal) -968 +Reduced rate 10% [85k,105k] Rate (step) -446 +Reduced rate 15% [85k,105k] Rate (step) -223 ======================================================================== Firm-in-band counts: - Raise-to-100k band [85k,100k): 130,991 firms released - Reduced-rate band [85k,105k): 151,636 firms - Taper band [85k,141.7k): 290,582 firms + Raise-to-100k band [85k,100k): 117,879 firms released + Reduced-rate band [85k,105k): 138,031 firms + Taper band [85k,141.7k): 275,805 firms diff --git a/results/revenue_impact_2025_26.png b/results/revenue_impact_2025_26.png index 2459d4e..b35d44f 100644 Binary files a/results/revenue_impact_2025_26.png and b/results/revenue_impact_2025_26.png differ diff --git a/results/seed_sensitivity.txt b/results/seed_sensitivity.txt index 50dc38f..3fd6b4d 100644 --- a/results/seed_sensitivity.txt +++ b/results/seed_sensitivity.txt @@ -1,10 +1,10 @@ GENERATOR-SEED SENSITIVITY (full-size 2023-24 builds; seeds 42/7/99) -definitive build: OBR near-threshold targets, side-consistent frame -scaling, no below-threshold liability calibration +two-universe build (#37): ONS-frame rows + HMRC registered-subset rows via +per-band registration propensity; in-scope liabilities only; scope is a seeded draw ====================================================================== run E b_llat raise £m taper £m base £bn -seed 42 (paper) 10,418 5.13 876.5 1525.0 175.4 -seed 7 10,196 4.98 874.2 1524.6 175.4 -seed 99 10,338 5.13 876.9 1525.1 175.3 +seed 42 (paper) 5,498 5.36 742.2 1406.1 175.7 +seed 7 5,227 5.05 743.2 1407.6 175.1 +seed 99 5,165 4.93 744.4 1408.2 176.0 -half-range across seeds: E ±111 | b_llat ±0.07 | raise ±£1.4m | taper ±£0.3m | base ±£0.03bn +half-range across seeds: E ±166 | b_llat ±0.21 | raise ±£1.1m | taper ±£1.1m | base ±£0.45bn diff --git a/results/static_sweep.txt b/results/static_sweep.txt index 3f1413c..e294e53 100644 --- a/results/static_sweep.txt +++ b/results/static_sweep.txt @@ -6,44 +6,73 @@ model vs HMRC published costing (Spring Budget 2024), by fiscal year; baseline column = counterfactual threshold path (85/85/87/89/92k). -------------------------------------------------------------------------- year hmrc_impact_m policyengine_impact_m -2024-25 -150.0 -356.8 -2025-26 -185.0 -363.9 -2026-27 -125.0 -223.6 -2027-28 -50.0 -77.8 -2028-29 65.0 118.7 +2024-25 -150.0 -249.8 +2025-26 -185.0 -234.3 +2026-27 -125.0 -200.1 +2027-28 -50.0 -65.1 +2028-29 65.0 122.9 + +Release decomposition (headline; liability GBPm of firms registered under the +baseline threshold but not under GBP90k, by data-year status): + 2024-25: released 36,307 firms; voluntary-at-data-year 207.2m, mandatory-at-data-year 18.3m, not-registered-at-data-year 24.3m + 2025-26: released 33,786 firms; voluntary-at-data-year 196.8m, mandatory-at-data-year 0.0m, not-registered-at-data-year 37.5m + 2026-27: released 28,913 firms; voluntary-at-data-year 176.5m, mandatory-at-data-year 0.0m, not-registered-at-data-year 23.5m + 2027-28: released 9,284 firms; voluntary-at-data-year 58.2m, mandatory-at-data-year 0.0m, not-registered-at-data-year 6.9m + 2028-29: released 0 firms; voluntary-at-data-year 0.0m, mandatory-at-data-year 0.0m, not-registered-at-data-year 0.0m + +Deregistration-threshold sensitivity: the headline releases registered +firms only below the GBP88k deregistration threshold ([85k, 88k) in the +raise years). Releasing the whole [baseline, 90k) band instead (gap = 0): + 2024-25: whole-band release -346.3m + 2025-26: whole-band release -382.3m + 2026-27: whole-band release -217.7m + 2027-28: whole-band release -65.1m + 2028-29: whole-band release +130.0m Voluntary-retention sensitivity (anchor, per year): headline assumes every released firm deregisters (full liability lost). If the Liu et al. (2021) voluntary share (43%) of released-firm liability is retained, the impact scales accordingly: - 2024-25: headline -356.8m -> retention-adjusted -203.4m (HMRC -150m) - 2025-26: headline -363.9m -> retention-adjusted -207.4m (HMRC -185m) - 2026-27: headline -223.6m -> retention-adjusted -127.5m (HMRC -125m) - 2027-28: headline -77.8m -> retention-adjusted -44.3m (HMRC -50m) - 2028-29: headline +118.7m -> retention-adjusted +67.7m (HMRC +65m) + 2024-25: headline -249.8m -> retention-adjusted -142.4m (HMRC -150m) + 2025-26: headline -234.3m -> retention-adjusted -133.6m (HMRC -185m) + 2026-27: headline -200.1m -> retention-adjusted -114.0m (HMRC -125m) + 2027-28: headline -65.1m -> retention-adjusted -37.1m (HMRC -50m) + 2028-29: headline +122.9m -> retention-adjusted +122.9m (HMRC +65m) + +Fixed-preference sensitivity: baseline voluntary registrants keep their +registration wherever the threshold moves (treats the frame's ~89% below- +threshold registered share as revealed preference; frame-selection bias +makes this a lower bound on the revenue loss): + 2024-25: fixed-preference -42.6m + 2025-26: fixed-preference -37.5m + 2026-27: fixed-preference -23.5m + 2027-28: fixed-preference -6.9m + 2028-29: fixed-preference +13.5m Threshold sweep (2024-25 vintage, GBP 90k baseline, 2025-26 fiscal year) -method: direct mechanical reclassification on the calibrated population +method: direct mechanical reclassification of in-scope VAT firms; voluntary +registrants below the data-year threshold held registered, firms aged across +it released unless gap-protected; turnover and liability aged together -------------------------------------------------------------------------- threshold_k revenue_change_m firms_change_k - 70.0 1456.512684 227.115249 - 75.0 1125.766861 170.113640 - 80.0 769.424905 112.982479 - 85.0 394.226572 56.199379 + 70.0 186.833777 29.365142 + 75.0 143.660781 21.890836 + 80.0 97.797747 14.459392 + 85.0 50.397430 7.251324 90.0 0.000000 0.000000 - 95.0 -413.832846 -55.935628 - 100.0 -708.211123 -96.197488 - 105.0 -1007.362098 -135.289361 - 110.0 -1187.581099 -157.800797 - 115.0 -1369.082526 -179.569095 - 120.0 -1552.806767 -200.811780 + 95.0 -190.870272 -27.362336 + 100.0 -445.099691 -65.892554 + 105.0 -687.659992 -99.825650 + 110.0 -847.865458 -120.831333 + 115.0 -1014.218110 -140.979245 + 120.0 -1190.180203 -161.645866 -Total VAT revenue at GBP 90k, 2025-26: 190.8bn -Total VAT revenue at GBP 90k, 2026-27: 195.7bn +Total VAT revenue at GBP 90k, 2025-26: 183.7bn +Total VAT revenue at GBP 90k, 2026-27: 188.4bn Bases and calibration-target comparison -------------------------------------------------------------------------- - 2023-24: registered base (>= threshold, unaged) = 175.4bn - 2024-25: registered base (>= threshold, unaged) = 181.0bn + 2023-24: in-scope base (>= threshold, unaged) = 175.7bn; model-implied voluntary below-threshold remittance = 2.5bn (not calibrated) + 2024-25: in-scope base (>= threshold, unaged) = 180.0bn; model-implied voluntary below-threshold remittance = 2.6bn (not calibrated) Sweep thresholds: [70000, 75000, 80000, 85000, 90000, 95000, 100000, 105000, 110000, 115000, 120000] diff --git a/results/turnover_distribution_85k.png b/results/turnover_distribution_85k.png index 6204b10..4c6bd43 100644 Binary files a/results/turnover_distribution_85k.png and b/results/turnover_distribution_85k.png differ diff --git a/results/turnover_distribution_90k.png b/results/turnover_distribution_90k.png index 97ceabc..f518654 100644 Binary files a/results/turnover_distribution_90k.png and b/results/turnover_distribution_90k.png differ diff --git a/results/vat_threshold_revenue_impact.png b/results/vat_threshold_revenue_impact.png index 87fc9ca..c2cca82 100644 Binary files a/results/vat_threshold_revenue_impact.png and b/results/vat_threshold_revenue_impact.png differ diff --git a/scripts/etl_ons_tables.py b/scripts/etl_ons_tables.py new file mode 100644 index 0000000..7407a21 --- /dev/null +++ b/scripts/etl_ons_tables.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""ETL: ONS *UK Business: Activity, Size and Location* workbooks -> processed CSVs. + +Rebuilds, for each vintage, the two ONS input tables the generator reads: + +* ``ons_firm_turnover.csv`` from **Table 8** — VAT and/or PAYE based + *enterprises* within region by SIC division and turnover sizeband (£000s), + United Kingdom columns. +* ``ons_firm_employment.csv`` from **Table 3** — VAT and/or PAYE based + *enterprises* within region by SIC division and employment sizeband, + United Kingdom columns. + +Both tables count the same statistical unit (enterprises), so their per-SIC +totals agree to ONS disclosure rounding. Table 18 (local units by employment +sizeband) is NOT used: a local unit is an individual site, and the 2023-24 +employment CSV shipped before this script reproduced Table 18 by mistake +(issue #37). + +Source layout (both editions): row 4 holds the geography header +(``K02000001 United Kingdom`` in column B), row 5 the sizeband labels, rows +6-93 the 88 SIC divisions ``"01 : Description"`` ... ``"99 : ..."`` and row 94 +the ``Total`` row. Columns B-I are the UK sizebands plus ``Total``. + +Usage:: + + python scripts/etl_ons_tables.py # rebuild both vintages + python scripts/etl_ons_tables.py --check # exit 1 if any CSV would change +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import openpyxl + +REPO = Path(__file__).resolve().parents[1] +RAW = REPO / "data" / "raw" / "ons" +PROCESSED = REPO / "data" / "processed" + +VINTAGES = { + "2023-24": RAW / "ukbusinessworkbook2024.xlsx", + "2024-25": RAW / "ukbusinessworkbook2025new.xlsx", +} +TABLES = { + "ons_firm_turnover.csv": ("Table 8", ["0-49", "50-99", "100-249", "250-499", "500-999", "1000-4999", "5000+"]), + "ons_firm_employment.csv": ("Table 3", ["0-4", "5-9", "10-19", "20-49", "50-99", "100-249", "250+"]), +} +UK_HEADER_ROW = 4 +LABEL_ROW = 5 +FIRST_DATA_ROW = 6 + + +def extract(workbook: Path, sheet: str, bands: list[str]) -> list[list[str]]: + wb = openpyxl.load_workbook(workbook, read_only=True, data_only=True) + ws = wb[sheet] + rows = list(ws.iter_rows(values_only=True)) + uk = str(rows[UK_HEADER_ROW - 1][1]) + if "United Kingdom" not in uk: + raise ValueError(f"{workbook.name} {sheet}: expected UK header in B4, got {uk!r}") + labels = [str(v).strip() for v in rows[LABEL_ROW - 1][1:9]] + if labels != bands + ["Total"]: + raise ValueError(f"{workbook.name} {sheet}: sizeband labels {labels} != {bands + ['Total']}") + out = [["SIC Code", "Description"] + bands + ["Total"]] + for row in rows[FIRST_DATA_ROW - 1:]: + label = row[0] + if label is None: + continue + label = str(label).strip() + vals = [int(v) for v in row[1:9]] + if label == "Total": + out.append(["", "Total"] + [str(v) for v in vals]) + break + code, _, desc = label.partition(" : ") + if not code.isdigit(): + raise ValueError(f"{workbook.name} {sheet}: unexpected row label {label!r}") + out.append([code.zfill(2), desc.replace(",", ";"), *[str(v) for v in vals]]) + if len(out) != 1 + 88 + 1: + raise ValueError(f"{workbook.name} {sheet}: expected 88 SIC rows + Total, got {len(out) - 1}") + return out + + +def write_csv(path: Path, table: list[list[str]]) -> str: + text = "\n".join(",".join(r) for r in table) + "\n" + path.write_text(text) + return text + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--check", action="store_true", help="do not write; exit 1 on any difference") + args = ap.parse_args(argv) + changed = 0 + for vintage, workbook in VINTAGES.items(): + for fname, (sheet, bands) in TABLES.items(): + table = extract(workbook, sheet, bands) + text = "\n".join(",".join(r) for r in table) + "\n" + target = PROCESSED / vintage / fname + current = target.read_text() if target.exists() else None + same = current == text + if same: + print(f"unchanged {target.relative_to(REPO)} ({sheet})") + continue + changed += 1 + if args.check: + print(f"DIFFERS {target.relative_to(REPO)} ({sheet})") + else: + target.write_text(text) + print(f"rewrote {target.relative_to(REPO)} ({sheet})") + return 1 if (args.check and changed) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/seed_sensitivity.py b/scripts/seed_sensitivity.py index a9bc2b5..bd64b4f 100644 --- a/scripts/seed_sensitivity.py +++ b/scripts/seed_sensitivity.py @@ -24,6 +24,7 @@ import argparse +import numpy as np import pandas as pd from firm_microsim.bunching.model import RANGE_HI, RANGE_LO, _run_estimator @@ -35,14 +36,16 @@ T_STAR = 85_000.0 T_NEW = 100_000.0 OUT = RESULTS_DIR / "seed_sensitivity.txt" -COLS = ["annual_turnover_k", "vat_liability_k", "weight"] +COLS = ["annual_turnover_k", "vat_liability_k", "weight", "vat_scope"] def headline_stats(df: pd.DataFrame) -> dict: """Compute the paper's headline 2023-24 statistics on one build.""" tk = df["annual_turnover_k"].to_numpy() t = tk * 1000.0 - liab = df["vat_liability_k"].to_numpy() * 1000.0 + scope = df["vat_scope"].to_numpy(dtype=bool) + # In-scope liabilities only (issue #37); bunching runs on the frame density. + liab = np.where(scope, df["vat_liability_k"].to_numpy() * 1000.0, 0.0) w = df["weight"].to_numpy() # Same population filter as BunchingEstimator.__init__ — the headline @@ -102,8 +105,8 @@ def half_range(key: str) -> float: lines = [ "GENERATOR-SEED SENSITIVITY (full-size 2023-24 builds; " f"seeds {'/'.join(str(s) for s in args.seeds)})", - "definitive build: OBR near-threshold targets, side-consistent frame", - "scaling, no below-threshold liability calibration", + "two-universe build (#37): ONS-frame rows + HMRC registered-subset rows via", + "per-band registration propensity; in-scope liabilities only; scope is a seeded draw", "=" * 70, f"{'run':<18}{'E':>8}{'b_llat':>8}{'raise £m':>10}{'taper £m':>10}{'base £bn':>10}", ] diff --git a/src/firm_microsim/analysis/dominated_region_mass.py b/src/firm_microsim/analysis/dominated_region_mass.py index fe8d75b..e2813a5 100644 --- a/src/firm_microsim/analysis/dominated_region_mass.py +++ b/src/firm_microsim/analysis/dominated_region_mass.py @@ -83,8 +83,10 @@ def main() -> None: t_star = float(NotchModel(VINTAGE).t_star) # 85.0 (£k), from VINTAGES config # --- Build observed + mass-conserving counterfactual densities ---------- - est = BunchingEstimator(VINTAGE) - res = est.estimate() # full reduced-form bunching solve (E, y_R, ...) + # Exposure counts are for firms that face the notch: in-scope VAT firms + # (issue #37). Out-of-scope enterprises are not liable at any turnover. + est = BunchingEstimator(VINTAGE, scope_only=True) + res = est.estimate() # reduced-form solve on the in-scope density centres = res["centres"] f_obs = res["f_obs"] f_cf = res["f_cf"] @@ -163,15 +165,16 @@ def main() -> None: W("=" * 74) W("") W("Dominated region (Kleven-Waseem): a = T* * tau/(1-tau)") - W("Bands measured on the WEIGHTED synthetic firm population.") + W("Bands measured on the WEIGHTED in-scope (VAT-liable) synthetic firms;") + W(" out-of-scope PAYE-only/exempt enterprises face no notch and are excluded.") W(" OBS uses exact band masks on the microdata; CF integrates the binned") W(" counterfactual with fractional edge-bin overlap.") W(" OBS = observed weighted firms in band") W(" CF = weighted firms the mass-conserving no-bunching counterfactual") W(" density places in band (total smooth density across the band)") - W(" NET = CF - OBS = net MISSING (notch-displaced) mass in the band; this") - W(" is the firms the notch removed from the dominated region -- the") - W(" quantity with empirical bite, consistent with excess mass E.") + W(" NET = CF - OBS: positive = mass the counterfactual places in the band") + W(" beyond what is observed (missing/displaced); NEGATIVE = observed") + W(" surplus over the smooth counterfactual (no displacement read).") W("") hdr = (f"{'rate / band':<22}{'width a (GBP)':>14}{'band (GBP k)':>18}" f"{'OBS':>12}{'CF':>12}{'NET disp.':>12}") @@ -211,8 +214,12 @@ def main() -> None: f" = TOTAL {s['total_obs']:,.0f} (baseline {base['obs']:,.0f}," f" {100*(s['total_obs']/base['obs']-1):+.1f}%)") W("") - W("REDUCED-FORM BUNCHING on this population (context for the masses above):") - W(f" excess mass below T* E = {E:,.0f} firms") + W("REDUCED-FORM BUNCHING on the IN-SCOPE density (context for the masses above;") + W(" NOT the Section 6 headline, which runs on the full ONS-frame density):") + frame_res = BunchingEstimator(VINTAGE).estimate() + W(f" frame-density headline (Section 6): E = {frame_res['E']:,.0f}, " + f"Delta_R = {frame_res['Delta_R']:,.0f}, y_R = GBP {frame_res['y_R']*1000:,.0f}") + W(f" in-scope excess mass below T* E = {E:,.0f} firms") W(f" missing mass above T* Delta_R = {Delta_R:,.0f} firms") W(f" marginal buncher y_R = GBP {y_R*1000:,.0f}") W(f" NET displaced mass in 20% dominated band = {missing_in_band:,.0f} firms") diff --git a/src/firm_microsim/analysis/reform_menu_common_base.py b/src/firm_microsim/analysis/reform_menu_common_base.py index 2c398b5..2d9797f 100644 --- a/src/firm_microsim/analysis/reform_menu_common_base.py +++ b/src/firm_microsim/analysis/reform_menu_common_base.py @@ -60,12 +60,14 @@ TABLE_LABELS = { "raise100k": "Raise threshold to GBP100,000", "taper": "Graduated taper [85k,141.7k]", + "taper_flat50": "Flat 50% marginal taper [85k,141.7k]", "rate10": "Reduced rate 10% [85k,105k]", "rate15": "Reduced rate 15% [85k,105k]", } LEVERS = { "raise100k": "Location", "taper": "Shape (phase-in)", + "taper_flat50": "Shape (constant marginal)", "rate10": "Rate (step)", "rate15": "Rate (step)", } @@ -107,7 +109,11 @@ def main(): " firm-microsim --vintage 2023-24 --output synthetic_firms_2023-24.csv" ) - df = pd.read_csv(data, usecols=["annual_turnover_k", "vat_liability_k", "weight"]) + df = pd.read_csv( + data, usecols=["annual_turnover_k", "vat_liability_k", "weight", "vat_scope"] + ) + # In-scope firms only (issue #37): out-of-scope enterprises never remit. + df = df[df["vat_scope"].astype(bool)].reset_index(drop=True) tk = df["annual_turnover_k"].to_numpy() # GBP thousand t = tk * 1000.0 # GBP liab = df["vat_liability_k"].to_numpy() * 1000.0 # GBP @@ -151,7 +157,7 @@ def main(): p("REFORM MENU ON A SINGLE COMMON BASE (tab:schedule_costs)") p("Base: GBP85k notch, 2023-24 microdata, UNAGED") p("=" * 72) - p(f"Dataset : {display_path(data)}") + p(f"Dataset : {display_path(data)} (in-scope VAT firms only)") p(f"VAT base (>=85k) : GBP {base_bn:.3f} bn") p(f"Firms in [85k,105k): {firms_band_85_105:,.0f}") p("") @@ -165,7 +171,7 @@ def main(): f"{rev_direct_m:+.0f} m firms {firms_direct_k:+.1f} (000s)") p("") p("--- Schedule reforms (computed on the SAME repo-generated base) -----") - for key in ["taper", "rate10", "rate15"]: + for key in ["taper", "taper_flat50", "rate10", "rate15"]: row = static_rows[key] p(f" {TABLE_LABELS[key]:<33}: {row['cost_m']:+.1f} m " f"affected firms {row['firms_k']:+.1f} (000s)") @@ -173,10 +179,10 @@ def main(): p("=" * 72) p("CORRECTED tab:schedule_costs (repo-generated GBP85k, 2023-24 unaged)") p("=" * 72) - p(f"{'Reform':<42}{'Lever':<18}{'Static (GBPm)':>14}") - p("-" * 74) - for key in ["raise100k", "taper", "rate10", "rate15"]: - p(f"{TABLE_LABELS[key]:<42}{LEVERS[key]:<18}" + p(f"{'Reform':<42}{'Lever':<26}{'Static (GBPm)':>14}") + p("-" * 82) + for key in ["raise100k", "taper", "taper_flat50", "rate10", "rate15"]: + p(f"{TABLE_LABELS[key]:<42}{LEVERS[key]:<26}" f"{round(static_rows[key]['cost_m']):>14d}") p("=" * 72) p("") diff --git a/src/firm_microsim/bunching/model.py b/src/firm_microsim/bunching/model.py index 1cb5e4d..1a552b9 100644 --- a/src/firm_microsim/bunching/model.py +++ b/src/firm_microsim/bunching/model.py @@ -307,12 +307,15 @@ class BunchingEstimator: :meth:`estimate`, :meth:`bootstrap`, :meth:`summary`, or :meth:`sensitivity`. """ - def __init__(self, vintage: str = "2023-24") -> None: + def __init__(self, vintage: str = "2023-24", *, scope_only: bool = False) -> None: """Load the synthetic population for ``vintage``. The threshold ``t_star`` is taken from ``firm_microsim.config.VINTAGES[vintage]["threshold"]`` -- never hardcoded. The population is filtered to ``[RANGE_LO, RANGE_HI]``. + With ``scope_only`` the density is that of in-scope VAT firms + (``vat_scope``); by default it is the whole ONS-frame density, the + universe on which the near-threshold shape targets are applied. """ self.vintage = vintage if vintage not in VINTAGES: @@ -327,7 +330,11 @@ def __init__(self, vintage: str = "2023-24") -> None: f" python -m firm_microsim --vintage {vintage} " f"--output synthetic_firms_{vintage}.csv" ) - firms = pd.read_csv(path, usecols=["annual_turnover_k", "weight"]) + cols = ["annual_turnover_k", "weight"] + (["vat_scope"] if scope_only else []) + firms = pd.read_csv(path, usecols=cols) + if scope_only: + firms = firms[firms["vat_scope"].astype(bool)] + self.scope_only = scope_only firms = firms[ (firms["annual_turnover_k"] >= RANGE_LO) & (firms["annual_turnover_k"] <= RANGE_HI) diff --git a/src/firm_microsim/calibration.py b/src/firm_microsim/calibration.py index 3c9f852..7d628ad 100644 --- a/src/firm_microsim/calibration.py +++ b/src/firm_microsim/calibration.py @@ -5,9 +5,20 @@ 1. Map firms to HMRC turnover bands (edges driven by the single configurable VAT threshold). - 2. Build a sparse target matrix ``A`` where ``A[i, j]`` is firm ``j``'s - contribution to target ``i`` (turnover bands, HMRC sectors, ONS - employment bands, VAT-liability by sector, VAT-liability by band). + 2. Build a target matrix ``A`` where ``A[i, j]`` is firm ``j``'s + contribution to target ``i``. Every row is declared on ONE of two + universes (issue #37): + + * the **ONS VAT/PAYE enterprise frame** (population row, employment + rows, near-threshold shape rows): contribution 1 for frame rows, + 0 for the appended HMRC negative/zero-turnover traders; + * the **HMRC VAT-registered subset** (turnover-band rows, sector + rows, liability-by-band rows): contribution equals the firm's + *registration propensity* ``p_b`` -- the HMRC count in its turnover + band divided by the frame's base-weighted mass in that band -- so + expected registered totals match HMRC by construction at unit + weights, while the frame margins are matched by the frame rows. + Appended negative/zero traders carry ``p = 1``. 3. Optimize per-firm log-weights with Adam under a *symmetric relative error* loss, so targets of very different scales are balanced. Turnover bands carry ~5x importance; VAT-liability-by-band carry 2x. @@ -138,8 +149,10 @@ class TargetSpec: """ def __init__(self, n_sectors: int, n_vat_sectors: int, n_near: int = 0) -> None: - self.n_turnover = 7 + self.n_turnover = 8 # Negative_or_Zero + 7 positive-turnover bands self.n_population = 1 + self.propensity: Tensor | None = None # per-firm registration propensity + self.frame_mask: Tensor | None = None # True for ONS-frame rows self.n_sectors = n_sectors self.n_employment = len(EMPLOYMENT_BANDS) self.n_vat_sectors = n_vat_sectors @@ -156,6 +169,54 @@ def __init__(self, n_sectors: int, n_vat_sectors: int, n_near: int = 0) -> None: self.n_targets = self.near_start + self.n_near +def registration_propensity( + band_indices: Tensor, + frame_mask: Tensor, + base_weights: Tensor, + hmrc_bands: dict, +) -> Tensor: + """Per-firm probability of being a VAT-registered trader, by HMRC band. + + For each positive-turnover band ``b`` the propensity is the HMRC + registered count divided by the frame's base-weighted mass in that band, + clamped to one (a warning is logged if the frame cannot contain the HMRC + population). Appended negative/zero-turnover rows (``frame_mask`` False) + are HMRC traders by construction and receive propensity one. + + Below the threshold the propensity is the registered share of the frame + (voluntary registrants plus firms registered under the rolling test); + above it, the share of frame enterprises that are in the VAT net at all + (the remainder are PAYE-only or exempt-sector businesses). + """ + prop = torch.zeros_like(base_weights) + band_names = [ + "Negative_or_Zero", + "£1_to_Threshold", + "£Threshold_to_£150k", + "£150k_to_£300k", + "£300k_to_£500k", + "£500k_to_£1m", + "£1m_to_£10m", + "Greater_than_£10m", + ] + for b in range(1, 8): + mask = frame_mask & (band_indices == b) + frame_mass = float(base_weights[mask].sum().item()) + target = float(hmrc_bands[band_names[b]]) + if frame_mass <= 0: + continue + p = target / frame_mass + if p > 1.0 + 1e-6: + logger.warning( + "Band %s: HMRC count %.0f exceeds frame mass %.0f (propensity %.3f clamped to 1)", + band_names[b], target, frame_mass, p, + ) + prop[mask] = min(1.0, p) + logger.info("Registration propensity %-22s %.3f", band_names[b], min(1.0, p)) + prop[~frame_mask] = 1.0 + return prop + + def build_target_matrix( config: Config, turnover_values: Tensor, @@ -170,30 +231,41 @@ def build_target_matrix( vat_liability_bands: dict, near_threshold_bins: pd.DataFrame | None = None, base_weights: Tensor | None = None, + frame_mask: Tensor | None = None, ) -> Tuple[Tensor, Tensor, TargetSpec]: """Construct the calibration target matrix and target vector. Args: config: Run configuration (threshold + device). - turnover_values: Per-firm turnover (£k). + turnover_values: Per-firm turnover (£k); appended HMRC negative/zero + traders carry exactly 0. sic_codes: Per-firm integer SIC sector codes. input_values: Per-firm input expenditure (£k). employment_band_indices: Per-firm ONS employment band index (0-6). - hmrc_bands: Latest HMRC VAT firm counts by turnover band. - ons_total: ONS total firms in the registered-business frame. + hmrc_bands: Latest HMRC VAT trader counts by turnover band (all 8). + ons_total: ONS enterprise count in the VAT/PAYE frame. hmrc_sector_df: HMRC VAT population by sector. - ons_employment_df: ONS employment-band table. + ons_employment_df: ONS enterprise counts by employment band. vat_liability_sector_df: HMRC VAT liability by sector (£m). vat_liability_bands: Latest VAT liability by turnover band (£m). + near_threshold_bins: OBR £1k-bin shape targets (frame rows). + base_weights: Per-row base weights (stratified builds); default ones. + frame_mask: True for ONS-frame rows, False for appended HMRC + negative/zero traders; default all True. Returns: Tuple of (target_matrix [n_targets x n_firms], target_values, spec). + ``spec.propensity`` holds the per-firm registration propensity and + ``spec.frame_mask`` the frame indicator used to build the rows. """ device = config.device threshold = config.vat_threshold n_firms = len(turnover_values) if base_weights is None: base_weights = torch.ones_like(turnover_values) + if frame_mask is None: + frame_mask = torch.ones(n_firms, dtype=torch.bool, device=device) + frame_f = frame_mask.float() sector_rows = hmrc_sector_df[hmrc_sector_df["Trade_Sector"] != "Total"].copy() if config.calibrate_vat_liability_sector: @@ -210,86 +282,62 @@ def build_target_matrix( target_matrix = torch.zeros(spec.n_targets, n_firms, device=device) band_indices = map_to_hmrc_bands(turnover_values, threshold) + propensity = registration_propensity(band_indices, frame_mask, base_weights, hmrc_bands) + spec.propensity = propensity + spec.frame_mask = frame_mask - # Expected registration status used by registered-population targets. - # Mandatory firms contribute one. Below-threshold firms contribute the - # HMRC voluntary-registration share of the base-weighted ONS frame. The - # final microdata use a seeded weighted selection with the same total. - below_mask = band_indices == 1 - below_mass = float(base_weights[below_mask].sum().item()) - voluntary_share = ( - min(1.0, float(hmrc_bands["£1_to_Threshold"]) / below_mass) - if below_mass > 0 - else 0.0 - ) - registration_propensity = torch.zeros_like(turnover_values) - registration_propensity[below_mask] = voluntary_share - registration_propensity[turnover_values > threshold] = 1.0 - - # Rows 0-6: turnover bands (band index 1..7 -> rows 0..6). - for row, band_idx in enumerate(range(1, 8)): - target_matrix[row, band_indices == band_idx] = 1.0 - - # Negative/zero-turnover firms are appended after calibration with unit - # weights. Target the positive-turnover base at the residual needed for - # the final population to equal the ONS registered-business total. - target_matrix[spec.population_start, :] = 1.0 - - # Sector targets count VAT-registered firms, not the full ONS frame. Use - # expected registration during differentiable calibration; generate.py - # realizes the same below-threshold total after weights are optimized. + # ---- HMRC registered-subset rows ----------------------------------- + # Rows 0-7: trader counts by turnover band (band index b -> row b). + for b in range(8): + mask = band_indices == b + target_matrix[spec.turnover_start + b, mask] = propensity[mask] + + # Sector rows count VAT-registered traders. for offset, (_, sector_row) in enumerate(sector_rows.iterrows()): sic_code = int(sector_row["Trade_Sector"]) mask = sic_codes == sic_code - target_matrix[spec.sector_start + offset, mask] = registration_propensity[mask] + target_matrix[spec.sector_start + offset, mask] = propensity[mask] - # Employment-band targets. - for band_idx in range(spec.n_employment): - row = spec.employment_start + band_idx - target_matrix[row, employment_band_indices == band_idx] = 1.0 - - # Net VAT liability (£k) per firm = standard rate * value added - # = STANDARD_VAT_RATE * (turnover - input). This matches the HMRC - # net-VAT-liability targets, which are net of input reclaim. + # Net VAT liability (£k) per firm = standard rate * value added. vat_liability_values = STANDARD_VAT_RATE * (turnover_values - input_values) - # VAT-liability-by-sector targets (weight firms by their liability). for offset, (_, vat_row) in enumerate(vat_liability_sector_rows.iterrows()): row = spec.vat_sector_start + offset sic_code = int(vat_row["Trade_Sector"]) mask = sic_codes == sic_code - target_matrix[row, mask] = ( - vat_liability_values[mask] * registration_propensity[mask] - ) + target_matrix[row, mask] = vat_liability_values[mask] * propensity[mask] - # VAT-liability-by-band targets. for offset, band_name in enumerate(VAT_LIABILITY_BANDS_CALIBRATED): row = spec.vat_band_start + offset mask = _band_membership_mask(turnover_values, band_name, threshold) - target_matrix[row, mask] = vat_liability_values[mask] + target_matrix[row, mask] = vat_liability_values[mask] * propensity[mask] + + # ---- ONS frame rows -------------------------------------------------- + target_matrix[spec.population_start, :] = frame_f + + for band_idx in range(spec.n_employment): + row = spec.employment_start + band_idx + mask = (employment_band_indices == band_idx) & frame_mask + target_matrix[row, mask] = 1.0 # Near-threshold £1k-bin membership rows (bins are (lo, lo+1], matching - # the coarse-band edge conventions). Target values are assembled below. + # the coarse-band edge conventions). SHAPE-ONLY targets on BOTH sides of + # the threshold: each side takes the OBR profile's within-side shape, + # scaled to the synthetic frame's own base-weighted mass on that side. + # The OBR chart counts HMRC traders (a different unit and, below the + # threshold, a different universe than the ONS business frame), so its + # LEVELS are not imported on either side; mixing direct counts on one + # side with frame-scaled shape on the other inverted the cross-threshold + # ordering. With side-consistent scaling the cross-threshold ratio is the + # frame's own, and the OBR data supply only the within-side profile. near_targets: list[float] = [] if n_near: - # SHAPE-ONLY targets on BOTH sides of the threshold: each side takes - # the OBR profile's within-side shape, scaled to the synthetic frame's - # own base-weighted mass on that side. The OBR chart counts HMRC - # traders (a different unit and, below the threshold, a different - # universe than the ONS business frame), so its LEVELS are not - # imported on either side; mixing direct counts on one side with - # frame-scaled shape on the other inverted the cross-threshold - # ordering (more mass just above than just below - economically - # backwards for a liability notch). With side-consistent scaling the - # cross-threshold ratio is the frame's own, and the OBR data supply - # only the within-side profile (the rise into the threshold and the - # decline beyond it). below = near_threshold_bins[near_threshold_bins["bin_lo_k"] < threshold] above = near_threshold_bins[near_threshold_bins["bin_lo_k"] >= threshold] below_lo = float(below["bin_lo_k"].min()) above_hi = float(above["bin_lo_k"].max()) + 1.0 - below_mask = (turnover_values > below_lo) & (turnover_values <= threshold) - above_mask = (turnover_values > threshold) & (turnover_values <= above_hi) + below_mask = (turnover_values > below_lo) & (turnover_values <= threshold) & frame_mask + above_mask = (turnover_values > threshold) & (turnover_values <= above_hi) & frame_mask below_rows = float(base_weights[below_mask].sum().item()) above_rows = float(base_weights[above_mask].sum().item()) below_total = float(below["count"].sum()) @@ -297,38 +345,41 @@ def build_target_matrix( for offset, (_, bin_row) in enumerate(near_threshold_bins.iterrows()): lo = float(bin_row["bin_lo_k"]) row = spec.near_start + offset - mask = (turnover_values > lo) & (turnover_values <= lo + 1.0) + mask = (turnover_values > lo) & (turnover_values <= lo + 1.0) & frame_mask target_matrix[row, mask] = 1.0 if lo < threshold: - near_targets.append( - float(bin_row["count"]) / below_total * below_rows - ) + near_targets.append(float(bin_row["count"]) / below_total * below_rows) else: - near_targets.append( - float(bin_row["count"]) / above_total * above_rows - ) + near_targets.append(float(bin_row["count"]) / above_total * above_rows) # ---- Target values -------------------------------------------------- - # £1_to_Threshold keeps the ONS structure (the base-weighted synthetic - # count, which equals the row count on unsampled builds); all higher - # bands match HMRC. - ons_threshold_count = float(base_weights[band_indices == 1].sum().item()) turnover_targets = [ - ons_threshold_count, - hmrc_bands["£Threshold_to_£150k"], - hmrc_bands["£150k_to_£300k"], - hmrc_bands["£300k_to_£500k"], - hmrc_bands["£500k_to_£1m"], - hmrc_bands["£1m_to_£10m"], - hmrc_bands["Greater_than_£10m"], + float(hmrc_bands["Negative_or_Zero"]), + float(hmrc_bands["£1_to_Threshold"]), + float(hmrc_bands["£Threshold_to_£150k"]), + float(hmrc_bands["£150k_to_£300k"]), + float(hmrc_bands["£300k_to_£500k"]), + float(hmrc_bands["£500k_to_£1m"]), + float(hmrc_bands["£1m_to_£10m"]), + float(hmrc_bands["Greater_than_£10m"]), ] - population_target = max( - 0.0, float(ons_total) - float(hmrc_bands["Negative_or_Zero"]) - ) + population_target = float(ons_total) + + # HMRC sector counts include traders whose turnover band is "Unknown" + # (159,300 in 2024-25; none in 2023-24), who cannot enter the band rows. + # Allocate them proportionally so the sector rows and the band rows count + # the same registered universe. + sector_raw = [float(r.iloc[-1]) for _, r in sector_rows.iterrows()] + sector_scale = sum(turnover_targets) / sum(sector_raw) if sum(sector_raw) > 0 else 1.0 + if abs(sector_scale - 1.0) > 1e-3: + logger.info( + "Sector targets rescaled by %.4f to the turnover-band total " + "(HMRC 'Unknown' turnover band allocated proportionally)", sector_scale, + ) + sector_targets = [v * sector_scale for v in sector_raw] # Value column is the (single) year column, always the last column — # year-agnostic so the 2023-24 / 2024-25 vintages both work. - sector_targets = [float(r.iloc[-1]) for _, r in sector_rows.iterrows()] ons_emp_rows = ons_employment_df[ ~ons_employment_df["Description"].str.contains("Total", na=False) @@ -339,7 +390,6 @@ def build_target_matrix( ] # VAT liability targets are in £m in the source; convert to £k. - # Value column is the (single) year column, always last — year-agnostic. vat_liability_sector_targets = [ float(r.iloc[-1]) * 1000.0 for _, r in vat_liability_sector_rows.iterrows() ] @@ -360,7 +410,7 @@ def build_target_matrix( logger.info("Target matrix shape: %s", tuple(target_matrix.shape)) logger.info( - "Targets: 7 turnover + 1 population + %d sector + %d employment + %d VAT-liability sector " + "Targets: 8 turnover + 1 population + %d sector + %d employment + %d VAT-liability sector " "+ %d VAT-liability band + %d near-threshold = %d", spec.n_sectors, spec.n_employment, @@ -369,6 +419,12 @@ def build_target_matrix( spec.n_near, spec.n_targets, ) + logger.info( + "Universe check: employment targets sum %.0f vs population target %.0f; " + "HMRC band targets sum %.0f vs sector targets sum %.0f", + sum(employment_targets), population_target, + sum(turnover_targets), sum(sector_targets), + ) return target_matrix, target_values, spec @@ -403,7 +459,7 @@ def optimize_weights( """Optimize per-firm weights to match all targets simultaneously. Minimizes a mean symmetric-relative-error loss with per-target importance - weights, Adam, dropout regularization, a mean absolute log-weight penalty, + weights, Adam, inverted dropout regularization, a mean absolute log-weight penalty, gradient clipping, and early stopping. Weights are parameterized as ``exp(log_w)`` to remain strictly positive. @@ -489,6 +545,7 @@ def optimize_weights( logger.info("Optimization complete. Turnover-band fit:") band_names = [ + "Negative_or_Zero", "£1_to_Threshold", "£Threshold_to_£150k", "£150k_to_£300k", diff --git a/src/firm_microsim/dynamic/__main__.py b/src/firm_microsim/dynamic/__main__.py index 8fec2e6..1b0a363 100644 --- a/src/firm_microsim/dynamic/__main__.py +++ b/src/firm_microsim/dynamic/__main__.py @@ -31,6 +31,7 @@ crosscheck, load_reform_data, marginal_buncher, + marginal_buncher_iso, reform_revenue, ) @@ -83,16 +84,23 @@ def main(argv=None): "(headline 0.17).") lines.append("Ability n is an accounting anchor recovered under the £85k notch " "given e; e is") - lines.append("NOT identified from the synthetic data (see the placebo) — read " - "results as") - lines.append("CONDITIONAL on the assumed e.") + lines.append("NOT identified from the synthetic data — read results as " + "CONDITIONAL on the assumed e.") + lines.append("Liabilities are in-scope only: out-of-scope (PAYE-only/exempt) " + "enterprises remit nothing (issue #37).") lines.append("=" * 78) # --- Analytic marginal buncher n_H(e). ---------------------------------- - lines.append("\nMarginal buncher n_H(e) [analytic; £]:") + lines.append("\nMarginal buncher n_H(e, delta) [analytic; £]. delta = deductible-input") + lines.append("share: it scales revenue but not own-factor cost, so n_H rises with delta;") + lines.append("delta=0 is the turnover-tax solve, 0.6 the population mean input share.") for e in elasticities: nH, dy = marginal_buncher(e) - row = f" e={e:<5} n_H=£{nH/1000:7.1f}k dy*=£{dy/1000:6.1f}k" + parts = [f" e={e:<5} n_H(delta=0)=£{nH/1000:6.1f}k dy*=£{dy/1000:5.1f}k"] + for d in (0.4, 0.6): + nH_d, _ = marginal_buncher_iso(e, delta=d) + parts.append(f"n_H(delta={d})=£{nH_d/1000:6.1f}k") + row = " ".join(parts) print(row) lines.append(row) @@ -131,24 +139,18 @@ def main(argv=None): if args.behavioural and E_HEADLINE in beh: fig_reform_distribution(df, beh[E_HEADLINE], label, E_HEADLINE) - # Direction-of-e note. + # Direction-of-e note, derived from the table above rather than asserted. lines.append("") - lines.append("Direction: a LARGER e makes every reform CHEAPER (smaller " - "revenue loss). Each") - lines.append("reform lowers the effective rate on the band, so the " - "iso-elastic response scales") - lines.append("turnover UP toward the firm's frictionless optimum " - "(y* = y_obs*((1-tau_reform)/(1-tau_base))^e > y_obs), widening " - "the taxed base;") - lines.append("the larger e, the larger this base-broadening offset to the " - "static cost.") - lines.append("NOTE: the taper figure captures this intensive base-broadening " - "only; it omits the") - lines.append("taper's marginal-rate distortion (the iso-elastic cost cannot " - "credibly price the") - lines.append("downward response a taper induces), so the taper's behavioural " - "cost is a lower") - lines.append("bound on its true cost. See Section 7 of the paper.") + lines.append("Reading: the raise row is flat in e by construction of the " + "region-confined solve") + lines.append("(released firms leave the base; no other firm's rate changes). " + "Reduced-rate bands") + lines.append("get cheaper as e rises: band firms scale turnover up by " + "((1-tau f1)/(1-tau f0))^e,") + lines.append("widening the taxed base. The tapers have no behavioural row: " + "their rate varies") + lines.append("continuously with turnover, outside the region-confined " + "flat-rate solve.") # Notch-fit figure (observed density + dominated region + n_H) per e. for e in elasticities: diff --git a/src/firm_microsim/dynamic/model.py b/src/firm_microsim/dynamic/model.py index 6312c1c..c128be1 100644 --- a/src/firm_microsim/dynamic/model.py +++ b/src/firm_microsim/dynamic/model.py @@ -309,6 +309,51 @@ def schedule_taper(y, T=T_STAR, top=TAPER_WIDE_TOP, tau_max=TAU_MAX): schedule_taper.regions = None +def taper_band_top(m: float, T=T_STAR, tau_max=TAU_MAX) -> float: + """Band top ``U(m) = m T / (m - tau_max)`` of a band-confined taper with a + CONSTANT marginal remittance rate ``m`` on ``[T, U]``. + + Continuity with the standard-rate line requires ``m (U - T) = tau_max U``. + ``m`` must exceed ``tau_max``; ``m -> 1`` gives the infimum ``T/(1-tau_max)`` + (= £106,250: net revenue flat across the band, weakly dominated), and + ``m = 0.5`` gives £141,667 -- the same band top the linear 0->100% design + reaches, at half its peak marginal rate. The band top is therefore a + property of the chosen shape, not a requirement of removing the notch. + """ + if m <= tau_max: + raise ValueError("constant marginal rate must exceed the standard rate") + return m * T / (m - tau_max) + + +def make_schedule_taper_flat(m: float = 0.5, T=T_STAR, tau_max=TAU_MAX): + """Band-confined taper with a CONSTANT marginal remittance rate ``m``. + + Liability ``L(y) = m (y - T)`` on ``[T, U(m)]`` and ``tau_max y`` above; + the average-rate fraction is ``f = m (y - T) / (tau_max y)`` in the band. + Net revenue has slope ``1 - m > 0`` throughout, so no dominated interval + exists and the peak marginal rate is ``m`` rather than the 100% the linear + design reaches at its top. Default ``m = 0.5`` shares the shipped linear + taper's band top (£141,667) for a like-for-like cost comparison. + """ + top = taper_band_top(m, T, tau_max) + + def sched(y): + y = np.asarray(y, dtype=float) + T_ = float(T) + band = (y >= T_) & (y <= top) + above = y > top + f = np.zeros_like(y, dtype=float) + safe_y = np.where(y > 0, y, 1.0) + f = np.where(band, m * (y - T_) / (tau_max * safe_y), f) + f = np.where(above, 1.0, f) + return f + + sched.regions = None + sched.band_top = top + sched.marginal_rate = m + return sched + + def make_schedule_reduced_rate(tau_low, T=T_STAR, top=TAPER_TOP, tau_std=TAU_MAX): """Banded reduced rate: fraction ``tau_low/tau_std`` in [T, top], 1 above.""" frac_low = tau_low / tau_std @@ -430,23 +475,38 @@ def marginal_buncher(e, vintage="2023-24"): return nH_k * 1000.0, dy_k * 1000.0 -def marginal_buncher_iso(e, T=T_STAR, tau=TAU_MAX): - """Independent iso-elastic indifference solve for the marginal buncher (£). +def marginal_buncher_iso(e, T=T_STAR, tau=TAU_MAX, delta=0.0): + """Iso-elastic indifference solve for the marginal buncher (£). - Solves ``pi_bunch(T*; n) = pi_register(y1; n)`` for ``n``, where the firm - bunches at ``T*`` (unregistered) or registers at ``y1 = n*(1-tau)**e``. - Used only to cross-check :func:`marginal_buncher`. + Solves ``pi_bunch(T*; n) = pi_register(y1; n)`` for ``n`` under + formulation A with deductible-input share ``delta``: bunching yields + ``(1-delta) T* - C(T*)`` and registering yields + ``(1-delta)(1-tau) y1 - C(y1)`` at ``y1 = n[(1-delta)(1-tau)]**e``. The + deductible share does NOT cancel here (it scales revenue but not the + own-factor cost), so ``n_H`` rises with ``delta``; ``delta = 0`` reproduces + the turnover-tax solve in :mod:`firm_microsim.notch.model`. """ from scipy.optimize import brentq + va = 1.0 - delta + net_reg = va * (1.0 - tau) + def gap(n): - u_bunch = iso_profit(T, n, e, net=1.0) - y1 = n * (1.0 - tau) ** e - u_tax = iso_profit(y1, n, e, net=1.0 - tau) + # Best unregistered choice: the untaxed optimum, capped at the + # threshold (a firm cannot stay unregistered above T*). + y_u = min(T, n * va ** e) + u_bunch = iso_profit(y_u, n, e, net=va) + # Registered optimum; registering with y1 < T* is never chosen (it is + # dominated by staying unregistered at y1), so the branch starts at T*. + y1 = max(T, n * net_reg ** e) + u_tax = iso_profit(y1, n, e, net=net_reg) return float(u_bunch - u_tax) - lo, hi = T * (1.0 + 1e-6), T * 5.0 - while gap(lo) * gap(hi) > 0 and hi < T * 50: + # At n_lo the registered optimum is exactly T*, where bunching strictly + # wins (same turnover, no tax), so gap(lo) > 0 and the root lies above. + lo = T / net_reg ** e * (1.0 + 1e-9) + hi = lo * 2.0 + while gap(hi) > 0 and hi < T * 1e3: hi *= 1.5 n_H = brentq(gap, lo, hi) return float(n_H), float(n_H - T) @@ -581,7 +641,7 @@ def wmass(arr): def load_reform_data(path=REFORM_DATA): - """Load the reform-costing dataset (£ units). Ability is recovered per-e.""" + """Load the reform-costing dataset (£ units), in-scope liabilities only.""" path = Path(path) if not path.exists(): raise FileNotFoundError( @@ -590,8 +650,12 @@ def load_reform_data(path=REFORM_DATA): ) df = pd.read_csv( path, - usecols=["annual_turnover_k", "vat_liability_k", "weight", "vat_registered"], + usecols=["annual_turnover_k", "vat_liability_k", "weight", "vat_scope", "vat_registered"], ) + # Out-of-scope enterprises (PAYE-only / exempt-sector, issue #37) face no + # VAT schedule: drop them so every reform is priced, and every firm + # counted, on the in-scope population only. + df = df[df["vat_scope"].astype(bool)].reset_index(drop=True) out = pd.DataFrame() out["turnover"] = df["annual_turnover_k"].to_numpy(dtype=float) * 1000.0 out["liab"] = df["vat_liability_k"].to_numpy(dtype=float) * 1000.0 @@ -608,6 +672,8 @@ def build_reforms(): "raise100k": (make_schedule_raise(100_000.0), "Raise threshold to £100k"), "taper": (schedule_taper, "Graduated taper (£85k→£141.7k, monotone)"), + "taper_flat50": (make_schedule_taper_flat(0.5), + "Flat 50% marginal taper (£85k→£141.7k)"), "rate10": (make_schedule_reduced_rate(0.10), "Reduced rate 10% (£85k–£105k)"), "rate15": (make_schedule_reduced_rate(0.15), diff --git a/src/firm_microsim/generate.py b/src/firm_microsim/generate.py index f2d29fa..ea4f424 100644 --- a/src/firm_microsim/generate.py +++ b/src/firm_microsim/generate.py @@ -9,25 +9,29 @@ source bands. Draw input expenditure (Beta-distributed input/output ratios with sector-specific shifts) and sector-conditional employment. - Stage 2 — Calibrate per-firm weights via multi-objective optimization so - the *weighted* population simultaneously matches **HMRC VAT Annual - Statistics** (firm counts by turnover band and by sector, net VAT liability - by band and by sector) and ONS employment-band counts. Turnover bands are - weighted ~5x; VAT-liability-by-band 2x. A symmetric-relative-error loss - balances targets across scales. Below-threshold zero/negative-turnover - firms are added manually from the HMRC Negative_or_Zero target. VAT - liability by sector is reported as an informational diagnostic rather than - optimized by default because the input/output tax structure is not yet a - calibrated target. - -VAT registration is then assigned: mandatory above the (single, configurable) -VAT threshold, plus a seeded weighted selection below it that matches HMRC's -£1_to_Threshold registered count to within one calibration weight. + Stage 2 — Calibrate per-firm weights via multi-objective optimization on + TWO declared universes (issue #37): the ONS VAT/PAYE enterprise frame + (population, employment bands, near-threshold shape) and the HMRC + VAT-registered subset (trader counts by turnover band and by sector, net + VAT liability by band), the latter entering through a per-band + registration propensity. HMRC negative/zero-turnover traders are appended + before calibration as an out-of-frame stratum. Turnover bands are weighted + ~5x; VAT-liability-by-band 2x. A symmetric-relative-error loss balances + targets across scales. VAT liability by sector is reported as an + informational diagnostic rather than optimized by default because the + input/output tax structure is not yet a calibrated target. + +VAT scope and registration are then assigned by seeded weighted selection per +HMRC band so registered totals match HMRC to within one calibration weight +(see :func:`assign_vat_flags`). Output: ~2.94M rows with calibration weights, written to ``data/synthetic/synthetic_firms.csv`` with columns ``sic_code, annual_turnover_k, annual_input_k, vat_liability_k, employment, -weight, vat_registered``. (A ``productivity`` column is added downstream.) +weight, vat_scope, vat_registered, in_frame``. ``in_frame`` marks ONS-frame +enterprises (False for the appended HMRC negative/zero-turnover traders); +``vat_scope`` marks firms in the VAT net (registered above the threshold, or +registrable below it); ``vat_registered`` marks HMRC-count-matched traders. Sources: * ONS Business Structure Database (firm counts by turnover & employment). @@ -339,63 +343,110 @@ def stratified_thin( return keep_idx, base_kept +def _select_to_weighted_target( + candidate_idx: Tensor, weights: Tensor, target: float, device: str +) -> Tensor: + """Seeded random order of ``candidate_idx`` until cumulative weight >= target.""" + if len(candidate_idx) == 0 or target <= 0: + return candidate_idx[:0] + order = candidate_idx[torch.randperm(len(candidate_idx), device=device)] + cumulative = torch.cumsum(weights[order], dim=0) + cutoff = int(torch.searchsorted(cumulative, torch.tensor(float(target), device=device)).item()) + cutoff = min(cutoff, len(order) - 1) + return order[: cutoff + 1] + + def assign_vat_flags( turnover_values: Tensor, hmrc_bands: Dict[str, float], config: Config, calibration_weights: Tensor | None = None, -) -> Tensor: - """Assign VAT registration flags. - - Mandatory above the configurable threshold; voluntary below it. When final - calibration weights are supplied, below-threshold firms are selected in a - seeded random order until their cumulative weight reaches the HMRC target. - - Args: - turnover_values: Per-firm turnover (£k). - hmrc_bands: HMRC band targets (provides the £1_to_Threshold count). - config: Run configuration (threshold + device). - - Returns: - Boolean tensor of VAT-registration status. + frame_mask: Tensor | None = None, +) -> tuple[Tensor, Tensor]: + """Assign VAT scope and registration flags to match HMRC trader counts. + + Two universes (issue #37). Every ONS-frame row is an enterprise; only a + subset are VAT traders. ``vat_scope`` marks firms in the VAT net: + + * above the threshold, a seeded weighted selection per HMRC turnover band + whose cumulative calibration weight reaches that band's HMRC count (the + remainder are PAYE-only or exempt-sector enterprises, out of scope); + * below the threshold, every frame enterprise is treated as registrable + (scope = True) — a maintained assumption, since HMRC publishes only the + registered count there — and a seeded weighted selection reaching the + HMRC ``£1_to_Threshold`` count is flagged registered (voluntary, or + registered under the rolling test); + * appended negative/zero-turnover rows (``frame_mask`` False) are HMRC + traders by construction: in scope and registered. + + ``vat_registered`` = (scope and turnover > threshold) or selected below or + appended. Weighted totals match HMRC to within one calibration weight per + band. + + Returns ``(vat_scope, vat_registered)`` as boolean tensors. """ - logger.info("Assigning VAT registration flags...") + logger.info("Assigning VAT scope and registration flags...") threshold = config.vat_threshold device = config.device - - below = (turnover_values > 0) & (turnover_values <= threshold) + n = len(turnover_values) weights = ( calibration_weights if calibration_weights is not None else torch.ones_like(turnover_values) ) + if frame_mask is None: + frame_mask = torch.ones(n, dtype=torch.bool, device=device) + + from .calibration import map_to_hmrc_bands + + band_indices = map_to_hmrc_bands(turnover_values, threshold) + band_names = [ + "Negative_or_Zero", "£1_to_Threshold", "£Threshold_to_£150k", + "£150k_to_£300k", "£300k_to_£500k", "£500k_to_£1m", "£1m_to_£10m", + "Greater_than_£10m", + ] + + scope = torch.zeros(n, dtype=torch.bool, device=device) + registered = torch.zeros(n, dtype=torch.bool, device=device) + + # Appended HMRC traders: in scope, registered. + scope[~frame_mask] = True + registered[~frame_mask] = True + + # Below the threshold: all frame rows registrable; HMRC count registered. + below = frame_mask & (band_indices == 1) + scope[below] = True + target_below = float(hmrc_bands.get("£1_to_Threshold", 0.0)) + chosen = _select_to_weighted_target(torch.where(below)[0], weights, target_below, device) + registered[chosen] = True n_below = float(weights[below].sum().item()) - target_below = float(hmrc_bands["£1_to_Threshold"]) - voluntary_rate = target_below / n_below if n_below > 0 else 0.15 logger.info( - "Voluntary VAT weighted rate: %.3f (target %s / weighted rows %s)", - voluntary_rate, - f"{target_below:,.0f}", - f"{n_below:,.0f}", + "Below-threshold registered share: %.3f (HMRC %s / weighted frame %s)", + target_below / n_below if n_below > 0 else float("nan"), + f"{target_below:,.0f}", f"{n_below:,.0f}", ) - mandatory = turnover_values > threshold - voluntary = torch.zeros_like(below) - candidate_idx = torch.where(below)[0] - if len(candidate_idx) > 0: - order = candidate_idx[torch.randperm(len(candidate_idx), device=device)] - cumulative = torch.cumsum(weights[order], dim=0) - cutoff = int(torch.searchsorted(cumulative, target_below).item()) - cutoff = min(cutoff, len(order) - 1) - voluntary[order[: cutoff + 1]] = True - vat_registered = mandatory | voluntary + # Above the threshold: in-scope selection per band to the HMRC count. + for b in range(2, 8): + cand = frame_mask & (band_indices == b) + target = float(hmrc_bands.get(band_names[b], 0.0)) + chosen = _select_to_weighted_target(torch.where(cand)[0], weights, target, device) + scope[chosen] = True + registered[chosen] = True + mass = float(weights[cand].sum().item()) + logger.info( + " %-22s in-scope %s of %s weighted frame firms (%.3f)", + band_names[b], f"{target:,.0f}", f"{mass:,.0f}", + target / mass if mass > 0 else float("nan"), + ) + logger.info( - "VAT: %d mandatory + %d voluntary = %d registered", - int(mandatory.sum().item()), - int(voluntary.sum().item()), - int(vat_registered.sum().item()), + "VAT: %s in scope, %s registered (weighted %s)", + f"{int(scope.sum().item()):,}", + f"{int(registered.sum().item()):,}", + f"{float(weights[registered].sum().item()):,.0f}", ) - return vat_registered + return scope, registered def _add_zero_turnover_firms( @@ -406,7 +457,13 @@ def _add_zero_turnover_firms( hmrc_bands: Dict[str, float], device: str, ) -> tuple[Tensor, Tensor, Tensor, Tensor]: - """Append HMRC Negative_or_Zero firms, allocated by sector share.""" + """Append HMRC Negative_or_Zero traders, allocated by sector share. + + Appended BEFORE calibration (issue #37): the rows enter the optimiser with + unit base weights, contribute to the HMRC registered-subset rows with + propensity one, and are excluded from the ONS-frame rows (population, + employment, near-threshold shape) through ``frame_mask``. + """ target = int(hmrc_bands["Negative_or_Zero"]) if target <= 0: return sic_codes, turnover, input_values, weights @@ -523,19 +580,31 @@ def generate( else: base_weights = torch.ones_like(base_turnover) - # Temporary per-firm employment-band assignment for the target matrix. - tmp_emp = assign_employment(base_sic, data.ons_employment, cfg.device) + # Append the HMRC negative/zero-turnover traders BEFORE calibration so + # every target row sees the same rows (issue #37). They are outside the + # ONS frame (frame_mask False) and inside the HMRC registered subset. + n_frame = len(base_turnover) + final_sic, final_turnover, final_input, final_base_weights = _add_zero_turnover_firms( + base_sic, base_turnover, base_input, base_weights, data.hmrc_bands, cfg.device + ) + frame_mask = torch.zeros(len(final_turnover), dtype=torch.bool, device=cfg.device) + frame_mask[:n_frame] = True + + # Per-firm employment assignment used by the target matrix AND retained in + # the output. Earlier versions redrew it after optimisation, invalidating + # the employment target rows. + employment = assign_employment(final_sic, data.ons_employment, cfg.device) emp_band_idx = torch.tensor( - [_employment_band_index(e.item()) for e in tmp_emp], + [_employment_band_index(e.item()) for e in employment], dtype=torch.long, device=cfg.device, ) target_matrix, target_values, spec = build_target_matrix( cfg, - base_turnover, - base_sic, - base_input, + final_turnover, + final_sic, + final_input, emp_band_idx, data.hmrc_bands, data.ons_total, @@ -544,24 +613,17 @@ def generate( data.hmrc_liability_sector, data.vat_liability_bands, near_threshold_bins=getattr(data, "near_threshold_bins", None), - base_weights=base_weights, - ) - - weights = optimize_weights( - cfg, target_matrix, target_values, spec, base_weights=base_weights + base_weights=final_base_weights, + frame_mask=frame_mask, ) - final_sic, final_turnover, final_input, final_weights = _add_zero_turnover_firms( - base_sic, base_turnover, base_input, weights, data.hmrc_bands, cfg.device + final_weights = optimize_weights( + cfg, target_matrix, target_values, spec, base_weights=final_base_weights ) - # Preserve the exact employment assignments used by calibration. Earlier - # versions redrew them here, invalidating the employment target matrix. - n_extra = len(final_turnover) - len(base_turnover) - extra_emp = assign_employment(final_sic[-n_extra:], data.ons_employment, cfg.device) - employment = torch.cat([tmp_emp, extra_emp]) if n_extra else tmp_emp - vat_flags = assign_vat_flags( - final_turnover, data.hmrc_bands, cfg, calibration_weights=final_weights + vat_scope, vat_flags = assign_vat_flags( + final_turnover, data.hmrc_bands, cfg, + calibration_weights=final_weights, frame_mask=frame_mask, ) logger.info("Assembling final DataFrame...") @@ -577,7 +639,9 @@ def generate( "vat_liability_k": STANDARD_VAT_RATE * (turnover_np - input_np), "employment": employment.cpu().numpy().astype(int), "weight": final_weights.cpu().numpy(), + "vat_scope": vat_scope.cpu().numpy().astype(bool), "vat_registered": vat_flags.cpu().numpy().astype(bool), + "in_frame": frame_mask.cpu().numpy().astype(bool), } ) logger.info( diff --git a/src/firm_microsim/report.py b/src/firm_microsim/report.py index b7b9fad..991d312 100644 --- a/src/firm_microsim/report.py +++ b/src/firm_microsim/report.py @@ -53,7 +53,8 @@ def _vintage_lines(vintage: str) -> list[str]: out += [ f" rows (firm types): {len(df):,}", - f" weighted population: {float(df['weight'].sum()):,.0f} firms", + f" weighted ONS-frame population: {rep.total_population:,.0f} firms" + f" (registered traders {float(df.loc[df['vat_registered'].astype(bool), 'weight'].sum()):,.0f})", "-" * 64, f" {'Dimension':<26}{'accuracy':>12}{'error':>12}", "-" * 64, diff --git a/src/firm_microsim/static/model.py b/src/firm_microsim/static/model.py index fa3b6fe..dccc8a1 100644 --- a/src/firm_microsim/static/model.py +++ b/src/firm_microsim/static/model.py @@ -16,11 +16,13 @@ import numpy as np import pandas as pd -from firm_microsim.config import SYNTHETIC_DATA_DIR +from firm_microsim.config import SYNTHETIC_DATA_DIR, VINTAGES -# Fiscal-year aging factors (cumulative from the base-year microdata) plus the -# April-2024 anchor reform: a frozen £85k baseline, RPI-uprated with a two-year -# lag, against the £90k policy, alongside HMRC's published costing (£m). +# Fiscal-year ageing factors (cumulative nominal growth from the 2023-24 data +# year) plus the April-2024 anchor reform: a frozen £85k baseline, RPI-uprated +# with a two-year lag, against the £90k policy, alongside HMRC's published +# costing (£m). Both turnover and liability are aged by the same factor against +# fixed nominal thresholds (the fiscal-drag convention); see ``_growth``. FISCAL_YEARS = [ {"year": "2024-25", "baseline": 85000, "policy": 90000, "hmrc_impact": -150, "firm_growth": 1.0310}, {"year": "2025-26", "baseline": 85000, "policy": 90000, "hmrc_impact": -185, "firm_growth": 1.0516}, @@ -28,6 +30,17 @@ {"year": "2027-28", "baseline": 89000, "policy": 90000, "hmrc_impact": -50, "firm_growth": 1.1102}, {"year": "2028-29", "baseline": 92000, "policy": 90000, "hmrc_impact": 65, "firm_growth": 1.1424}, ] +# Cumulative factor already embodied in each vintage's data year, so a +# 2024-25 build is not aged by the 2023-24 -> 2024-25 step a second time. +VINTAGE_BASE_GROWTH = {"2023-24": 1.0, "2024-25": 1.0310} + +# The deregistration threshold sits £2,000 below the registration threshold +# (£83k/£85k to March 2024; £88k/£90k from April 2024). A registered firm may +# deregister only if its turnover falls below the deregistration threshold, so +# when the threshold rises the mechanically "released" band is +# [old threshold, new threshold - gap): firms in the top £2,000 of the raised +# band stay registered. +DEREGISTRATION_GAP = 2_000.0 # Current statutory threshold (£) — the baseline for the threshold sweep. POLICY_THRESHOLD = 90000 @@ -51,6 +64,9 @@ class StaticVATModel: def __init__(self, vintage: str = "2024-25") -> None: """Load the synthetic population for ``vintage`` (default the £90k year).""" self.vintage = vintage + if vintage not in VINTAGES: + raise ValueError(f"Unknown vintage {vintage!r}; choose from {sorted(VINTAGES)}") + self.data_threshold = float(VINTAGES[vintage]["threshold"]) * 1000.0 path = SYNTHETIC_DATA_DIR / f"synthetic_firms_{vintage}.csv" if not path.exists(): raise FileNotFoundError( @@ -58,32 +74,102 @@ def __init__(self, vintage: str = "2024-25") -> None: f" python -m firm_microsim --vintage {vintage} " f"--output synthetic_firms_{vintage}.csv" ) - self.firms = pd.read_csv( - path, usecols=["annual_turnover_k", "vat_liability_k", "weight"] - ) + cols = ["annual_turnover_k", "vat_liability_k", "weight", "vat_scope", "vat_registered"] + self.firms = pd.read_csv(path, usecols=cols) + # Baseline voluntary registrants: registered with data-year turnover at + # or below the data-year threshold. Their status is held fixed under + # every counterfactual threshold (registration is not modelled). + t_k = self.firms["annual_turnover_k"] + registered = self.firms["vat_registered"].astype(bool) + self.firms["voluntary"] = registered & (t_k * 1000.0 <= self.data_threshold) + # Mandatory registrants at the data-year threshold: the population that + # a threshold rise can release, subject to the deregistration threshold. + self.firms["mandatory"] = registered & (t_k * 1000.0 > self.data_threshold) # -- core mechanics ---------------------------------------------------- + def _growth(self, year: str) -> float: + """Nominal-growth factor from this vintage's data year to ``year``.""" + return _fiscal_year(year)["firm_growth"] / VINTAGE_BASE_GROWTH[self.vintage] + def _aged(self, growth: float) -> pd.DataFrame: - """Return turnover (£) and net VAT liability (£), aged by ``growth``.""" + """Return turnover (£) and net VAT liability (£), both aged by ``growth``. + + Turnover and liability grow together against fixed nominal thresholds + (the fiscal-drag convention the paper's institutional section relies + on), so band membership in a later fiscal year is evaluated on aged + turnover. + """ df = self.firms return pd.DataFrame( { "turnover": df["annual_turnover_k"] * 1000.0 * growth, "liab": df["vat_liability_k"] * 1000.0 * growth, "weight": df["weight"], + "scope": df["vat_scope"].astype(bool), + "voluntary": df["voluntary"].astype(bool), + "mandatory": df["mandatory"].astype(bool), } ) - @staticmethod - def _revenue(df: pd.DataFrame, threshold: float) -> float: + def _registered( + self, + df: pd.DataFrame, + threshold: float, + gap: float = DEREGISTRATION_GAP, + retain_voluntary: bool = False, + ) -> pd.Series: + """Registration under a counterfactual ``threshold`` (£). + + With aged turnover ``y`` and the data-year threshold ``T0``, an + in-scope firm is registered if + + * ``y >= threshold`` (required under the counterfactual); or + * it was registered at the data year and ``y >= threshold - gap`` + (cannot deregister: turnover is above the deregistration + threshold); or + * it is a baseline voluntary registrant with ``y < T0`` (never + required under either regime, so its status is unchanged); or + * ``retain_voluntary`` and it is a baseline voluntary registrant + (fixed-preference convention: a firm that chose registration below + ``T0`` keeps it wherever the threshold moves). + + Out-of-scope (PAYE-only / exempt-sector) enterprises never remit. At + ``threshold == T0`` the rule reproduces the baseline registered set. + Voluntary registrants that ageing carries into a band whose + requirement the move removes are therefore RELEASED by default; the + ONS frame's high below-threshold registration share (about 89%) + partly reflects that small firms enter the frame through VAT + registration, so treating it as revealed preference + (``retain_voluntary=True``) is reported as a sensitivity, not the + headline. + """ + y = df["turnover"] + registered0 = df["mandatory"] | df["voluntary"] + reg = (y >= threshold) | (registered0 & (y >= threshold - gap)) + reg = reg | (df["voluntary"] & (y < self.data_threshold)) + if retain_voluntary: + reg = reg | df["voluntary"] + return df["scope"] & reg + + def _revenue(self, df: pd.DataFrame, threshold: float, **kw) -> float: """Total weighted net VAT (£) from firms registered at ``threshold``.""" - registered = df["turnover"] >= threshold + registered = self._registered(df, threshold, **kw) return float((df["liab"].where(registered, 0.0) * df["weight"]).sum()) @staticmethod - def _vat_paying_firms(df: pd.DataFrame, threshold: float) -> float: + def _mandatory_base(df: pd.DataFrame, threshold: float) -> float: + """Weighted net VAT (£) of in-scope firms at or above ``threshold``. + + Excludes below-threshold voluntary remittances, whose model liability + is not calibrated, so the figure is comparable to HMRC's + above-threshold liability bands. + """ + mask = df["scope"] & (df["turnover"] >= threshold) + return float((df["liab"].where(mask, 0.0) * df["weight"]).sum()) + + def _vat_paying_firms(self, df: pd.DataFrame, threshold: float, **kw) -> float: """Weighted count of VAT-paying firms (registered & net-positive).""" - mask = (df["turnover"] >= threshold) & (df["liab"] > 0) + mask = self._registered(df, threshold, **kw) & (df["liab"] > 0) return float(df.loc[mask, "weight"].sum()) # -- smooth counterfactual density ------------------------------------ @@ -144,14 +230,14 @@ def threshold_sweep( ) -> pd.DataFrame: """Revenue (£m) and VAT-paying-firm (000s) changes vs ``baseline``. - Each row costs moving the threshold to a new location, holding turnover - fixed. Lowering the threshold draws firms in (positive); raising it - loses them (negative). Liability is scaled to the fiscal year by the - nominal-growth factor; firm counts are year-invariant. + Each row costs moving the threshold to a new location. Lowering the + threshold draws in-scope, not-yet-registered firms in (positive); + raising it releases in-scope registered firms (negative). Turnover and + liability are aged to the fiscal year by the same nominal-growth + factor, so membership is evaluated on aged turnover. """ thresholds = thresholds or SWEEP_THRESHOLDS - growth = _fiscal_year(year)["firm_growth"] - df = self._aged(growth) + df = self._aged(self._growth(year)) base_revenue = self._revenue(df, baseline) base_firms = self._vat_paying_firms(df, baseline) @@ -166,28 +252,48 @@ def threshold_sweep( ) return pd.DataFrame(rows) - def anchor_reform(self) -> pd.DataFrame: + def anchor_reform( + self, + gap: float = DEREGISTRATION_GAP, + retention: float = 0.0, + retain_voluntary: bool = False, + ) -> pd.DataFrame: """£85k→£90k anchor-reform impact (£m) per year: model vs HMRC. + ``gap`` is the registration-minus-deregistration threshold distance + (£2,000 by statute; pass 0 for the naive whole-band release). + ``retention`` scales the released liability by ``1 - retention`` (a + share of released firms assumed to stay registered voluntarily, e.g. + the Liu et al. 43%). ``retain_voluntary`` applies the fixed-preference + convention of :meth:`_registered`. + Simple band-sum on the loaded vintage. Use the £85k (2023-24) vintage — the basis HMRC actually had at the 6 March 2024 costing (the threshold was still £85k until 1 April 2024). There the affected [baseline, £90k) firms sit ABOVE the £85k registration threshold, so the band is cleanly - populated with registered firms (no de-bunching needed). For each year - the affected firms lie between the (uprated, frozen) £85k baseline and - the £90k policy; liability is scaled to the year by the growth factor. + populated with in-scope registered firms. Each year's impact is the + revenue under the £90k policy minus revenue under that year's + counterfactual baseline threshold, both evaluated on turnover and + liability aged to the year with the same registration rule. When fiscal drag lifts the baseline above £90k (2028-29) the band flips and the reform adds firms (a revenue gain). """ - tk = self.firms["annual_turnover_k"].to_numpy() - liab = self.firms["vat_liability_k"].to_numpy() * 1000.0 - w = self.firms["weight"].to_numpy() rows = [] for fy in FISCAL_YEARS: - lo_k, hi_k = sorted((fy["baseline"] / 1000.0, fy["policy"] / 1000.0)) - band = (tk >= lo_k) & (tk < hi_k) - mass_m = float((liab[band] * w[band]).sum()) * fy["firm_growth"] / 1e6 - pe_impact = mass_m if fy["baseline"] > fy["policy"] else -mass_m + df = self._aged(self._growth(fy["year"])) + base_t, pol_t = float(fy["baseline"]), float(fy["policy"]) + # Revenue under each regime from the same registration rule: + # in-scope firms at/above the regime's threshold, data-year + # registrants retained down to its deregistration threshold, + # voluntary registrants throughout. Differencing the two regimes + # handles both the release years and the 2028-29 sign flip. + liab_w = df["liab"] * df["weight"] + kw = dict(gap=gap, retain_voluntary=retain_voluntary) + r_policy = float(liab_w[self._registered(df, pol_t, **kw)].sum()) + r_base = float(liab_w[self._registered(df, base_t, **kw)].sum()) + pe_impact = (r_policy - r_base) / 1e6 + if pe_impact < 0: + pe_impact *= 1.0 - retention rows.append( { "year": fy["year"], @@ -198,6 +304,10 @@ def anchor_reform(self) -> pd.DataFrame: return pd.DataFrame(rows) def total_revenue_bn(self, year: str = "2025-26", threshold: int = POLICY_THRESHOLD) -> float: - """Total VAT revenue (£bn) at ``threshold`` in ``year`` (sanity check).""" - df = self._aged(_fiscal_year(year)["firm_growth"]) - return self._revenue(df, threshold) / 1e9 + """In-scope above-threshold VAT base (£bn) at ``threshold`` in ``year``. + + Comparable to HMRC's above-threshold liability bands; excludes the + uncalibrated below-threshold voluntary remittances. + """ + df = self._aged(self._growth(year)) + return self._mandatory_base(df, threshold) / 1e9 diff --git a/src/firm_microsim/validate.py b/src/firm_microsim/validate.py index 9710ba7..755a7ff 100644 --- a/src/firm_microsim/validate.py +++ b/src/firm_microsim/validate.py @@ -1,9 +1,9 @@ """Calibration-accuracy validation against ONS + HMRC targets. -Ports the validation summary of the original generator: compares the -weighted synthetic population against HMRC turnover bands, the ONS total, -ONS employment bands, HMRC sector counts, and HMRC VAT-liability targets -(by sector and by band). Reporting uses the logging framework. +Compares the weighted synthetic population against its targets on the two +declared universes (issue #37): HMRC turnover bands, sector counts and +VAT-liability targets are scored on VAT-registered rows; the ONS total and +employment bands on ONS-frame rows. Reporting uses the logging framework. Band edges are driven by the single configurable VAT threshold. """ @@ -134,19 +134,25 @@ def validate( df["vat_liability_k"] = STANDARD_VAT_RATE * (df["annual_turnover_k"] - df["annual_input_k"]) df["sic_numeric"] = df["sic_code"].astype(int) df["weighted_liability_m"] = df["vat_liability_k"] * df["weight"] / 1000.0 - all_bands = df.groupby("hmrc_band")["weight"].sum() - - # --- HMRC turnover-band accuracy (excludes the ONS-based threshold band). + # Two universes (issue #37): HMRC margins are scored on VAT-registered + # rows; ONS margins on ONS-frame rows (appended negative/zero-turnover + # traders are outside the frame). + if "in_frame" in df.columns: + frame = df[df["in_frame"].astype(bool)] + else: # legacy files: appended rows carry exactly zero turnover + frame = df[df["annual_turnover_k"] > 0] + vat_registered = df[df["vat_registered"].astype(bool)] + reg_bands = vat_registered.groupby("hmrc_band")["weight"].sum() + + # --- HMRC turnover-band accuracy: registered traders, all 8 bands. hmrc_accs: List[float] = [] for band_name, target in data.hmrc_bands.items(): - synth = float(all_bands.get(band_name, 0.0)) - if band_name == "£1_to_Threshold": - continue # ONS-based, not an HMRC calibration target + synth = float(reg_bands.get(band_name, 0.0)) hmrc_accs.append(_accuracy(synth, float(target))) hmrc_accuracy = float(np.mean(hmrc_accs)) if hmrc_accs else 0.0 - # --- ONS total population accuracy. - total_weighted = float(df["weight"].sum()) + # --- ONS frame population accuracy. + total_weighted = float(frame["weight"].sum()) ons_accuracy = _accuracy(total_weighted, float(data.ons_total)) # --- Employment-band accuracy. @@ -159,8 +165,8 @@ def validate( else 0.0 for band in EMPLOYMENT_BANDS } - df["employment_band"] = df["employment"].apply(_employment_band_name) - synth_emp = df.groupby("employment_band")["weight"].sum() + frame = frame.assign(employment_band=frame["employment"].apply(_employment_band_name)) + synth_emp = frame.groupby("employment_band")["weight"].sum() emp_accs = [ _accuracy(float(synth_emp.get(band, 0.0)), emp_targets[band]) for band in EMPLOYMENT_BANDS @@ -171,11 +177,16 @@ def validate( sector_rows = data.hmrc_population_sector[ data.hmrc_population_sector["Trade_Sector"] != "Total" ] - vat_registered = df[df["vat_registered"]] synth_sector = vat_registered.groupby("sic_numeric")["weight"].sum() + # Same universe as calibration: HMRC 'Unknown'-band traders allocated + # proportionally so sector totals equal the band total. + band_total = float(sum(data.hmrc_bands.values())) + sector_total = float(sum(float(r.iloc[-1]) for _, r in sector_rows.iterrows())) + sector_scale = band_total / sector_total if sector_total > 0 else 1.0 sector_accs = [ _accuracy( - float(synth_sector.get(int(r["Trade_Sector"]), 0.0)), float(r.iloc[-1]) + float(synth_sector.get(int(r["Trade_Sector"]), 0.0)), + float(r.iloc[-1]) * sector_scale, ) for _, r in sector_rows.iterrows() ] @@ -185,7 +196,7 @@ def validate( liab_sector_rows = data.hmrc_liability_sector[ data.hmrc_liability_sector["Trade_Sector"] != "Total" ] - synth_liab_sector = df.groupby("sic_numeric")["weighted_liability_m"].sum() + synth_liab_sector = vat_registered.groupby("sic_numeric")["weighted_liability_m"].sum() liab_sector_accs = [ _accuracy( float(synth_liab_sector.get(int(r["Trade_Sector"]), 0.0)), @@ -243,6 +254,7 @@ def validate( "VAT Liability by Sector: %.1f%% [informational, NOT calibrated]", report.vat_liability_sector * 100, ) - logger.info("Total Population: %s firms", f"{total_weighted:,.0f}") + logger.info("ONS-frame population: %s firms (all rows %s)", + f"{total_weighted:,.0f}", f"{float(df['weight'].sum()):,.0f}") return report diff --git a/tests/test_calibration_targets.py b/tests/test_calibration_targets.py index fb801e8..9ffb1a5 100644 --- a/tests/test_calibration_targets.py +++ b/tests/test_calibration_targets.py @@ -128,28 +128,64 @@ def test_near_target_rows_select_their_bins(frames): assert torch.equal(matrix[spec.near_start + offset], expected) -def test_population_target_accounts_for_appended_zero_turnover_firms(frames): +def test_population_and_employment_rows_are_frame_universe(frames): + """Frame rows (population, employment) exclude appended HMRC traders and + target the full ONS total; the employment targets partition that total + (issue #37).""" cfg, data = frames turnover, sic, inputs, emp, base_weights = _tiny_firms() + # Append two zero-turnover HMRC traders outside the frame. + turnover = torch.cat([turnover, torch.zeros(2)]) + sic = torch.cat([sic, torch.tensor([47, 62])]) + inputs = torch.cat([inputs, torch.zeros(2)]) + emp = torch.cat([emp, torch.zeros(2, dtype=torch.int64)]) + base_weights = torch.cat([base_weights, torch.ones(2)]) + frame_mask = torch.tensor([True] * 6 + [False] * 2) matrix, values, spec = build_target_matrix( - cfg, - turnover, - sic, - inputs, - emp, - data.hmrc_bands, - data.ons_total, - data.hmrc_population_sector, - data.ons_employment, - data.hmrc_liability_sector, - data.vat_liability_bands, + cfg, turnover, sic, inputs, emp, data.hmrc_bands, data.ons_total, + data.hmrc_population_sector, data.ons_employment, + data.hmrc_liability_sector, data.vat_liability_bands, near_threshold_bins=data.near_threshold_bins, - base_weights=base_weights, + base_weights=base_weights, frame_mask=frame_mask, ) - assert torch.equal(matrix[spec.population_start], torch.ones(len(turnover))) - assert float(values[spec.population_start]) == pytest.approx( - data.ons_total - data.hmrc_bands["Negative_or_Zero"] + assert torch.equal(matrix[spec.population_start], frame_mask.float()) + assert float(values[spec.population_start]) == pytest.approx(data.ons_total) + emp_rows = matrix[spec.employment_start: spec.employment_start + spec.n_employment] + assert torch.equal(emp_rows.sum(0), frame_mask.float()) + emp_targets = values[spec.employment_start: spec.employment_start + spec.n_employment] + assert float(emp_targets.sum()) == pytest.approx(data.ons_total, rel=2e-5) + # Appended traders enter the HMRC rows with propensity one, and the + # Negative_or_Zero band row targets the HMRC count. + assert torch.equal(matrix[spec.turnover_start][6:], torch.ones(2)) + assert float(values[spec.turnover_start]) == pytest.approx( + data.hmrc_bands["Negative_or_Zero"] ) + assert spec.propensity is not None + assert float(spec.propensity.max()) <= 1.0 + 1e-6 + + +def test_registration_propensity_below_one_on_real_frames(frames): + """The ONS frame must contain the HMRC registered population in every band.""" + from firm_microsim.calibration import map_to_hmrc_bands, registration_propensity + from firm_microsim.generate import generate_base_firms + cfg, data = frames + torch.manual_seed(0) + sic, turnover = generate_base_firms(data.ons_turnover, "cpu") + bands = map_to_hmrc_bands(turnover, cfg.vat_threshold) + frame = torch.ones(len(turnover), dtype=torch.bool) + prop = registration_propensity(bands, frame, torch.ones(len(turnover)), data.hmrc_bands) + # Documented expectation per band (HMRC 2023-24 / uniform-draw frame mass): + # below-threshold ~0.89; above, 0.5-1.0; the >£10m band ~0.64 is a draw + # artefact of the open 5000+ band (issue #40), not a PAYE/exempt share. + expected = {1: 0.89, 2: 0.67, 3: 0.51, 4: 0.60, 5: 0.77, 6: 1.00, 7: 0.64} + for b, e_p in expected.items(): + p = float(prop[bands == b].max()) + assert abs(p - e_p) < 0.03, f"band {b}: propensity {p:.3f} != ~{e_p}" + # Expected registered count reproduces HMRC at unit weights (band 7 is + # tight because the open 5000+ band is drawn uniformly to £50m). + expected = float(prop.sum()) + hmrc = sum(v for k, v in data.hmrc_bands.items() if k not in ("Negative_or_Zero", "Unknown")) + assert expected == pytest.approx(hmrc, rel=1e-3) def test_inverse_dropout_scaling_is_unbiased() -> None: diff --git a/tests/test_data_universes.py b/tests/test_data_universes.py new file mode 100644 index 0000000..3e3b893 --- /dev/null +++ b/tests/test_data_universes.py @@ -0,0 +1,44 @@ +"""The two ONS input tables count the same statistical unit (issue #37).""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pandas as pd +import pytest + +from firm_microsim.config import PROCESSED_DATA_DIR, VINTAGES + +REPO = Path(__file__).resolve().parents[1] + + +def _sic_rows(path: Path) -> pd.DataFrame: + df = pd.read_csv(path) + return df[~df["Description"].str.contains("Total", na=False) & df["SIC Code"].notna()] + + +@pytest.mark.parametrize("vintage", sorted(VINTAGES)) +def test_employment_and_turnover_tables_agree_to_rounding(vintage: str) -> None: + emp = _sic_rows(PROCESSED_DATA_DIR / vintage / "ons_firm_employment.csv") + turn = _sic_rows(PROCESSED_DATA_DIR / vintage / "ons_firm_turnover.csv") + assert len(emp) == len(turn) == 88 + # ONS rounds cells independently to the nearest 5: per-SIC totals may + # differ by a few units, never by the local-unit/enterprise gap (~15%). + diff = (emp["Total"].to_numpy() - turn["Total"].to_numpy()) + assert abs(diff).max() <= 20, "employment table is not on the enterprise unit" + assert abs(emp["Total"].sum() - turn["Total"].sum()) <= 50 + + +@pytest.mark.skipif( + not (REPO / "data" / "raw" / "ons" / "ukbusinessworkbook2024.xlsx").exists(), + reason="raw ONS workbooks not present in this checkout", +) +def test_processed_ons_tables_match_etl() -> None: + """The checked CSVs are exactly what scripts/etl_ons_tables.py produces.""" + result = subprocess.run( + [sys.executable, str(REPO / "scripts" / "etl_ons_tables.py"), "--check"], + capture_output=True, text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/test_generation_invariants.py b/tests/test_generation_invariants.py index 4b1fb08..019b1bc 100644 --- a/tests/test_generation_invariants.py +++ b/tests/test_generation_invariants.py @@ -53,7 +53,7 @@ def test_voluntary_registration_matches_weighted_target_within_one_weight() -> N turnover = torch.full((100,), 50.0) weights = torch.linspace(0.5, 2.0, 100) target = 42.0 - flags = assign_vat_flags( + scope, flags = assign_vat_flags( turnover, {"£1_to_Threshold": target}, Config(), @@ -62,6 +62,25 @@ def test_voluntary_registration_matches_weighted_target_within_one_weight() -> N achieved = float(weights[flags].sum()) assert achieved >= target assert achieved - target <= float(weights.max()) + assert bool(scope.all()) # every below-threshold frame firm is registrable + + +def test_above_threshold_scope_matches_hmrc_band_count_within_one_weight() -> None: + torch.manual_seed(4) + turnover = torch.full((200,), 120.0) # £Threshold_to_£150k band at £85k + weights = torch.linspace(0.5, 2.0, 200) + target = 100.0 + scope, flags = assign_vat_flags( + turnover, + {"£1_to_Threshold": 0.0, "£Threshold_to_£150k": target}, + Config(), + calibration_weights=weights, + ) + assert torch.equal(scope, flags) # above the threshold, in scope == registered + achieved = float(weights[scope].sum()) + assert achieved >= target + assert achieved - target <= float(weights.max()) + assert int(scope.sum()) < 200 # out-of-scope enterprises remain def test_zero_turnover_allocation_hits_hmrc_target_exactly() -> None: diff --git a/tests/test_static_ageing.py b/tests/test_static_ageing.py new file mode 100644 index 0000000..351265f --- /dev/null +++ b/tests/test_static_ageing.py @@ -0,0 +1,89 @@ +"""Static model: one ageing convention, applied to turnover and liability.""" + +from __future__ import annotations + +import pandas as pd +import pytest + +from firm_microsim.static.model import FISCAL_YEARS, VINTAGE_BASE_GROWTH, StaticVATModel + + +def _model_with(df: pd.DataFrame, vintage: str = "2023-24") -> StaticVATModel: + m = StaticVATModel.__new__(StaticVATModel) + m.vintage = vintage + m.data_threshold = 85_000.0 + m.firms = df + return m + + +def _firms() -> pd.DataFrame: + # A firm just below £85k that ageing carries across the threshold, one + # above, one out of scope above, and one voluntary registrant below. + return pd.DataFrame( + { + "annual_turnover_k": [84.0, 100.0, 100.0, 40.0], + "vat_liability_k": [8.0, 10.0, 10.0, 4.0], + "weight": [1.0, 1.0, 1.0, 1.0], + "vat_scope": [True, True, False, True], + "vat_registered": [False, True, False, True], + "voluntary": [False, False, False, True], + "mandatory": [False, True, False, False], + } + ) + + +def test_ageing_moves_turnover_and_liability_together() -> None: + m = _model_with(_firms()) + aged = m._aged(1.05) + assert aged["turnover"].iloc[0] == pytest.approx(84_000.0 * 1.05) + assert aged["liab"].iloc[0] == pytest.approx(8_000.0 * 1.05) + # The £84k firm crosses £85k once aged: membership is on aged turnover. + assert bool(m._registered(aged, 85_000.0).iloc[0]) + assert not bool(m._registered(m._aged(1.0), 85_000.0).iloc[0]) + + +def test_out_of_scope_and_voluntary_conventions() -> None: + m = _model_with(_firms()) + df = m._aged(1.0) + reg = m._registered(df, 85_000.0) + assert list(reg) == [False, True, False, True] + # Raising the threshold to £120k releases the in-scope firm only; the + # voluntary registrant keeps remitting; out-of-scope never remits. + assert m._revenue(df, 120_000.0) == pytest.approx(4_000.0) + assert m._revenue(df, 85_000.0) == pytest.approx(14_000.0) + # The reported base excludes below-threshold voluntary remittances. + assert m._mandatory_base(df, 85_000.0) == pytest.approx(10_000.0) + + +def test_vintage_growth_is_relative_to_its_own_data_year() -> None: + m23 = _model_with(_firms(), "2023-24") + m24 = _model_with(_firms(), "2024-25") + fy = {f["year"]: f["firm_growth"] for f in FISCAL_YEARS} + assert m23._growth("2025-26") == pytest.approx(fy["2025-26"]) + assert m24._growth("2025-26") == pytest.approx(fy["2025-26"] / VINTAGE_BASE_GROWTH["2024-25"]) + assert m24._growth("2024-25") == pytest.approx(1.0) + + +def test_deregistration_gap_retains_top_of_released_band() -> None: + m = _model_with(_firms()) + df = m._aged(1.0) + # Raising the threshold to £101k: the £100k in-scope registrant sits + # inside the £2k deregistration gap and stays registered ... + assert bool(m._registered(df, 101_000.0).iloc[1]) + # ... but a raise to £103k releases it. + assert not bool(m._registered(df, 103_000.0).iloc[1]) + # With no gap the naive whole-band release applies. + assert not bool(m._registered(df, 101_000.0, gap=0.0).iloc[1]) + + +def test_voluntary_registrant_aged_across_threshold_is_released_by_a_rise() -> None: + """The documented convention: a data-year voluntary registrant that ageing + carries above T0 is released by a rise (unless gap-protected); with + retain_voluntary it keeps its registration.""" + m = _model_with(_firms()) + m.firms.loc[3, "annual_turnover_k"] = 84.0 # voluntary at £84k + df = m._aged(1.05) # -> £88.2k, above T0 = £85k + assert bool(m._registered(df, 85_000.0).iloc[3]) # baseline: registered + assert not bool(m._registered(df, 95_000.0).iloc[3]) # rise to £95k: released + assert bool(m._registered(df, 95_000.0, retain_voluntary=True).iloc[3]) + assert bool(m._registered(df, 90_000.0).iloc[3]) # £88.2k >= 90k-2k: gap-protected diff --git a/tests/test_static_model.py b/tests/test_static_model.py index 9915f8d..da00a20 100644 --- a/tests/test_static_model.py +++ b/tests/test_static_model.py @@ -5,11 +5,17 @@ def test_threshold_sweep_has_correct_revenue_sign() -> None: model = StaticVATModel.__new__(StaticVATModel) + model.vintage = "2024-25" + model.data_threshold = 90_000.0 model.firms = pd.DataFrame( { "annual_turnover_k": [80.0, 87.0, 95.0, 110.0], "vat_liability_k": [2.0, 3.0, 4.0, 5.0], "weight": [1.0, 1.0, 1.0, 1.0], + "vat_scope": [True, True, True, True], + "vat_registered": [False, False, True, True], + "voluntary": [False, False, False, False], + "mandatory": [False, False, True, True], } ) diff --git a/tests/test_taper_monotone.py b/tests/test_taper_monotone.py index 921d78d..ed44d3e 100644 --- a/tests/test_taper_monotone.py +++ b/tests/test_taper_monotone.py @@ -85,3 +85,28 @@ def test_marginal_relief_variant_is_monotone_but_leaks_above_band(): y_big = np.array([1_000_000.0]) relief = TAU_MAX * (1.0 - schedule_taper_marginal_relief(y_big)) * y_big assert abs(relief[0] - TAU_MAX * (TAPER_TOP + T_STAR) / 2.0) < 1e-6 + + +def test_flat_marginal_taper_shares_band_top_and_is_monotone(): + import numpy as np + import pytest + + from firm_microsim.dynamic.model import ( + TAPER_WIDE_TOP, + TAU_MAX, + T_STAR, + make_schedule_taper_flat, + taper_band_top, + ) + assert taper_band_top(0.5) == pytest.approx(TAPER_WIDE_TOP) + assert taper_band_top(0.999) == pytest.approx(T_STAR / (1 - TAU_MAX), rel=1e-2) + sched = make_schedule_taper_flat(0.5) + y = np.linspace(T_STAR, sched.band_top * 1.2, 20001) + net = y * (1 - TAU_MAX * sched(y)) + assert np.all(np.diff(net) > 0) # strictly monotone + assert sched(np.array([T_STAR]))[0] == pytest.approx(0.0) + assert sched(np.array([sched.band_top]))[0] == pytest.approx(1.0, abs=1e-9) + # Peak marginal rate is 50%, against 100% for the linear design. + liab = TAU_MAX * sched(y) * y + band = y <= sched.band_top + assert np.max(np.diff(liab[band]) / np.diff(y[band])) == pytest.approx(0.5, abs=1e-6)