From efc6ceaf15dc0a96daca2454b45402b8c58b47f4 Mon Sep 17 00:00:00 2001 From: Labro Bot Date: Tue, 21 Jul 2026 09:22:52 +0000 Subject: [PATCH 1/2] feat: add per-project publish gate for dashboard snapshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add publish: bool = False to ProjectConfig so the dashboard snapshot only includes runs from explicitly opted-in projects. Fails safe — new projects publish nothing until opted in. - Filter snapshot rows by published projects after VACUUM INTO - Upsert only published projects to the projects table - 5 new tests covering all filtering scenarios Fixes #51 --- src/labro/cli.py | 36 ++++++- src/labro/config/schema.py | 7 ++ tests/test_publish_db.py | 193 +++++++++++++++++++++++++++++++++++-- 3 files changed, 226 insertions(+), 10 deletions(-) diff --git a/src/labro/cli.py b/src/labro/cli.py index 53aabe9..687177f 100644 --- a/src/labro/cli.py +++ b/src/labro/cli.py @@ -1202,15 +1202,47 @@ def _cmd_publish_db(args: argparse.Namespace) -> int: try: import sqlite3 + published_names = [p.name for p in config.projects if p.publish] + conn = store_mod.open_db(db_path) try: - project_rows = [(p.name, p.name_short, p.repo) for p in config.projects] + # Only upsert projects that are opted-in to dashboard publishing. + project_rows = [(p.name, p.name_short, p.repo) for p in config.projects if p.publish] store_mod.upsert_projects(conn, project_rows) # Bound parameter avoids bandit B608 (SQL injection via f-string) conn.execute("VACUUM INTO ?", (str(snapshot_path),)) finally: conn.close() + # Filter snapshot: remove runs (and orphaned items_touched) for + # projects that are not opted-in to publishing. This is done on + # the disposable snapshot copy so the live DB is never touched. + snap_conn = sqlite3.connect(str(snapshot_path)) + try: + if published_names: + placeholders = ",".join("?" for _ in published_names) + snap_conn.execute( + f"DELETE FROM items_touched" # noqa: S608 + f" WHERE run_id NOT IN" + f" (SELECT run_id FROM runs WHERE project IN ({placeholders}))", + published_names, + ) + snap_conn.execute( + f"DELETE FROM runs WHERE project NOT IN ({placeholders})", # noqa: S608 + published_names, + ) + snap_conn.execute( + f"DELETE FROM projects WHERE name NOT IN ({placeholders})", # noqa: S608 + published_names, + ) + else: + snap_conn.execute("DELETE FROM items_touched") + snap_conn.execute("DELETE FROM runs") + snap_conn.execute("DELETE FROM projects") + snap_conn.commit() + finally: + snap_conn.close() + # Hash the snapshot for content-addressed key and manifest hasher = hashlib.sha256() size_bytes = snapshot_path.stat().st_size @@ -1223,7 +1255,7 @@ def _cmd_publish_db(args: argparse.Namespace) -> int: db_filename = f"labro-{content_hash[:16]}.db" db_key = f"{key_prefix}db/{db_filename}" - # Row count from snapshot + # Row count from filtered snapshot snap_conn = sqlite3.connect(str(snapshot_path)) try: row_count: int = snap_conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0] diff --git a/src/labro/config/schema.py b/src/labro/config/schema.py index a6f2c57..64cf21f 100644 --- a/src/labro/config/schema.py +++ b/src/labro/config/schema.py @@ -287,6 +287,13 @@ class ProjectConfig(BaseModel): repo: str cron: str enabled: bool = True + publish: bool = False + """Include this project's runs in the public dashboard snapshot. + + Defaults to ``False`` so new projects never publish until explicitly + opted in. Only projects with ``publish = true`` appear in the + snapshot uploaded by ``labro publish-db``. + """ model: ModelSlugList | None = None max_turns: int | None = None timeout_s: int | None = None diff --git a/tests/test_publish_db.py b/tests/test_publish_db.py index 6134438..7fbd15f 100644 --- a/tests/test_publish_db.py +++ b/tests/test_publish_db.py @@ -14,7 +14,13 @@ import labro.store as store_mod from labro.cli import _cmd_publish_db -from labro.config.schema import DashboardConfig, DefaultsConfig, DigestConfig, LabroConfig +from labro.config.schema import ( + DashboardConfig, + DefaultsConfig, + DigestConfig, + LabroConfig, + ProjectConfig, +) from labro.r2 import R2Credentials @@ -32,28 +38,56 @@ def _make_config( dashboard_enabled: bool = True, key_prefix: str = "", endpoint: str | None = "https://fake.r2.cloudflarestorage.com", + projects: list[ProjectConfig] | None = None, ) -> LabroConfig: dashboard = DashboardConfig( enabled=dashboard_enabled, key_prefix=key_prefix, endpoint=endpoint, ) + if projects is None: + projects = [ + ProjectConfig(name="test-project", repo="o/test", cron="0 * * * *", publish=True) + ] return LabroConfig( digest=DigestConfig(enabled=False), dashboard=dashboard, defaults=DefaultsConfig(), + projects=projects, ) -def _make_db(tmp_path: Path, *, num_rows: int = 3) -> Path: - """Create a real labro.db at *tmp_path/labro.db* with *num_rows* run rows.""" +def _make_db( + tmp_path: Path, + *, + num_rows: int = 3, + projects: dict[str, int] | None = None, +) -> Path: + """Create a real labro.db at *tmp_path/labro.db*. + + If *projects* is provided (e.g. ``{"alpha": 2, "beta": 5}``), insert + that many rows per project instead of using *num_rows*/*test-project*. + """ db_path = tmp_path / "labro.db" conn = store_mod.open_db(db_path) - for i in range(num_rows): - conn.execute( - "INSERT INTO runs (run_id, project, started_at, outcome) VALUES (?, ?, ?, ?)", - (f"run-{i}", "test-project", "2024-01-01T00:00:00Z", "success"), - ) + if projects: + for project_name, count in projects.items(): + for i in range(count): + conn.execute( + "INSERT INTO runs (run_id, project, started_at, outcome) VALUES (?, ?, ?, ?)", + ( + f"run-{project_name}-{i}", + project_name, + "2024-01-01T00:00:00Z", + "success", + ), + ) + else: + for i in range(num_rows): + conn.execute( + "INSERT INTO runs (run_id, project, started_at, outcome) VALUES (?, ?, ?, ?)", + (f"run-{i}", "test-project", "2024-01-01T00:00:00Z", "success"), + ) conn.commit() conn.close() return db_path @@ -299,3 +333,146 @@ def test_snapshot_path_kept(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> assert rc == 0 assert snapshot_path.exists() + + +# ── per-project publish gate ───────────────────────────────────────────────── + + +def _extract_row_counts(snapshot_path: Path) -> dict[str, int]: + """Return a dict of table → row count from a snapshot DB.""" + import sqlite3 + + conn = sqlite3.connect(str(snapshot_path)) + counts = {} + for table in ("runs", "items_touched", "projects"): + counts[table] = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] # noqa: S608 + conn.close() + return counts + + +def test_no_projects_published_empty_snapshot(tmp_path: Path) -> None: + """When no project has publish=true, the snapshot has zero rows.""" + db_path = _make_db(tmp_path, projects={"alpha": 3, "beta": 2}) + snapshot_path = tmp_path / "snapshot.db" + config = _make_config( + projects=[ + ProjectConfig(name="alpha", repo="o/alpha", cron="0 * * * *", publish=False), + ProjectConfig(name="beta", repo="o/beta", cron="0 * * * *", publish=False), + ] + ) + + rc, _, _ = _run(config, db_path, dry_run=True, snapshot_path=snapshot_path) + assert rc == 0 + counts = _extract_row_counts(snapshot_path) + assert counts["runs"] == 0 + assert counts["projects"] == 0 + + +def test_mixed_publish_flags_filters_correctly(tmp_path: Path) -> None: + """Only runs from published projects appear in the snapshot.""" + db_path = _make_db(tmp_path, projects={"published": 4, "private": 6}) + snapshot_path = tmp_path / "snapshot.db" + config = _make_config( + projects=[ + ProjectConfig(name="published", repo="o/pub", cron="0 * * * *", publish=True), + ProjectConfig(name="private", repo="o/priv", cron="0 * * * *", publish=False), + ] + ) + + rc, _, _ = _run(config, db_path, dry_run=True, snapshot_path=snapshot_path) + assert rc == 0 + counts = _extract_row_counts(snapshot_path) + assert counts["runs"] == 4 + assert counts["projects"] == 1 + + +def test_all_published_keeps_all_rows(tmp_path: Path) -> None: + """When all projects have publish=true, all runs are kept.""" + db_path = _make_db(tmp_path, projects={"a": 3, "b": 2}) + snapshot_path = tmp_path / "snapshot.db" + config = _make_config( + projects=[ + ProjectConfig(name="a", repo="o/a", cron="0 * * * *", publish=True), + ProjectConfig(name="b", repo="o/b", cron="0 * * * *", publish=True), + ] + ) + + rc, _, _ = _run(config, db_path, dry_run=True, snapshot_path=snapshot_path) + assert rc == 0 + counts = _extract_row_counts(snapshot_path) + assert counts["runs"] == 5 + assert counts["projects"] == 2 + + +def test_items_touched_filtered_with_runs(tmp_path: Path) -> None: + """items_touched rows for unpublished runs are removed.""" + import sqlite3 + + db_path = tmp_path / "labro.db" + conn = store_mod.open_db(db_path) + conn.execute( + "INSERT INTO runs (run_id, project, started_at, outcome) VALUES (?, ?, ?, ?)", + ("run-pub", "published", "2024-01-01T00:00:00Z", "success"), + ) + conn.execute( + "INSERT INTO runs (run_id, project, started_at, outcome) VALUES (?, ?, ?, ?)", + ("run-priv", "private", "2024-01-01T00:00:00Z", "success"), + ) + conn.execute( + "INSERT INTO items_touched (run_id, repo, item_type, item_number) VALUES (?, ?, ?, ?)", + ("run-pub", "o/pub", "issue", 1), + ) + conn.execute( + "INSERT INTO items_touched (run_id, repo, item_type, item_number) VALUES (?, ?, ?, ?)", + ("run-priv", "o/priv", "issue", 2), + ) + conn.commit() + conn.close() + + snapshot_path = tmp_path / "snapshot.db" + config = _make_config( + projects=[ + ProjectConfig(name="published", repo="o/pub", cron="0 * * * *", publish=True), + ProjectConfig(name="private", repo="o/priv", cron="0 * * * *", publish=False), + ] + ) + + rc, _, _ = _run(config, db_path, dry_run=True, snapshot_path=snapshot_path) + assert rc == 0 + + snap_conn = sqlite3.connect(str(snapshot_path)) + runs = snap_conn.execute("SELECT run_id FROM runs").fetchall() + items = snap_conn.execute("SELECT run_id FROM items_touched").fetchall() + snap_conn.close() + + assert len(runs) == 1 + assert runs[0][0] == "run-pub" + assert len(items) == 1 + assert items[0][0] == "run-pub" + + +def test_publish_gate_updates_manifest_row_count(tmp_path: Path) -> None: + """Manifest row_count reflects only published runs, not total DB rows.""" + db_path = _make_db(tmp_path, projects={"pub": 3, "priv": 7}) + config = _make_config( + projects=[ + ProjectConfig(name="pub", repo="o/pub", cron="0 * * * *", publish=True), + ProjectConfig(name="priv", repo="o/priv", cron="0 * * * *", publish=False), + ] + ) + + captured_manifest: list[bytes] = [] + + def _capture(**kwargs: object) -> None: + if kwargs.get("key") == "manifest.json": + captured_manifest.append(kwargs["body"]) # type: ignore[arg-type] + + with ( + patch("labro.r2._put_object", side_effect=_capture), + patch("labro.r2.credentials_from_env", return_value=_FAKE_CREDS), + ): + rc, _, _ = _run(config, db_path) + + assert rc == 0 + manifest = json.loads(captured_manifest[0]) + assert manifest["row_count"] == 3 From 69e7b4ffecf582ed5e77c9beff0ec188db422df2 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 24 Jul 2026 08:50:48 +0100 Subject: [PATCH 2/2] fix(publish): filter project_locks and harden snapshot against residue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The publish gate filtered runs/items_touched/projects but left the project_locks table untouched, so an unpublished project's name shipped in the snapshot as a live row whenever a lock existed (transient, during an in-flight run or a stale lock). Filter project_locks alongside the other tables — this is the substantive leak this commit closes. Also VACUUM the snapshot after the DELETEs. On the SQLite builds we ship (Debian bookworm and the dev interpreter) secure_delete is compiled in and defaults ON, so freed-page residue is already zeroed there; the VACUUM is defensive hardening that makes the no-residue guarantee hold regardless of the build's secure_delete setting, and it corrects the reported size_bytes. Fold the row-count read into the same connection instead of reopening the snapshot. Adds a byte-level regression test asserting no unpublished project name or free-text survives in the raw snapshot bytes. --- src/labro/cli.py | 40 +++++++++++++++++++++++----------- tests/test_publish_db.py | 47 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 13 deletions(-) diff --git a/src/labro/cli.py b/src/labro/cli.py index 687177f..5b68426 100644 --- a/src/labro/cli.py +++ b/src/labro/cli.py @@ -1214,32 +1214,53 @@ def _cmd_publish_db(args: argparse.Namespace) -> int: finally: conn.close() - # Filter snapshot: remove runs (and orphaned items_touched) for - # projects that are not opted-in to publishing. This is done on - # the disposable snapshot copy so the live DB is never touched. + # Filter snapshot: remove runs (and orphaned items_touched), plus + # projects and project_locks, for projects that are not opted-in to + # publishing. This is done on the disposable snapshot copy so the + # live DB is never touched. + # + # The final ``VACUUM`` is not cosmetic: SQLite's default + # ``secure_delete`` is OFF, so ``DELETE`` only marks pages free — + # the deleted rows' bytes (repo names, agent free-text) remain + # physically present in the file and are trivially recoverable from + # the uploaded snapshot. VACUUM rebuilds the file, purging the + # residue, and also corrects the reported ``size_bytes`` below. + # @author Claude Opus 4.8 Anthropic snap_conn = sqlite3.connect(str(snapshot_path)) try: if published_names: placeholders = ",".join("?" for _ in published_names) + # `placeholders` is only "?" marks; values are bound. noqa/nosec + # silence ruff/bandit's f-string SQL heuristic (no injection). snap_conn.execute( - f"DELETE FROM items_touched" # noqa: S608 + f"DELETE FROM items_touched" # noqa: S608 # nosec B608 f" WHERE run_id NOT IN" f" (SELECT run_id FROM runs WHERE project IN ({placeholders}))", published_names, ) snap_conn.execute( - f"DELETE FROM runs WHERE project NOT IN ({placeholders})", # noqa: S608 + f"DELETE FROM runs WHERE project NOT IN ({placeholders})", # noqa: S608 # nosec B608 published_names, ) snap_conn.execute( - f"DELETE FROM projects WHERE name NOT IN ({placeholders})", # noqa: S608 + f"DELETE FROM projects WHERE name NOT IN ({placeholders})", # noqa: S608 # nosec B608 + published_names, + ) + snap_conn.execute( + f"DELETE FROM project_locks WHERE project NOT IN ({placeholders})", # noqa: S608 # nosec B608 published_names, ) else: snap_conn.execute("DELETE FROM items_touched") snap_conn.execute("DELETE FROM runs") snap_conn.execute("DELETE FROM projects") + snap_conn.execute("DELETE FROM project_locks") snap_conn.commit() + # Purge deleted-row residue from free pages (see comment above). + snap_conn.execute("VACUUM") + # Read the published row count from the same connection, after + # filtering, to avoid reopening the snapshot a second time. + row_count: int = snap_conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0] finally: snap_conn.close() @@ -1255,13 +1276,6 @@ def _cmd_publish_db(args: argparse.Namespace) -> int: db_filename = f"labro-{content_hash[:16]}.db" db_key = f"{key_prefix}db/{db_filename}" - # Row count from filtered snapshot - snap_conn = sqlite3.connect(str(snapshot_path)) - try: - row_count: int = snap_conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0] - finally: - snap_conn.close() - manifest_dict: dict[str, object] = { "schema_version": 1, "db_filename": f"db/{db_filename}", diff --git a/tests/test_publish_db.py b/tests/test_publish_db.py index 7fbd15f..a900dbd 100644 --- a/tests/test_publish_db.py +++ b/tests/test_publish_db.py @@ -476,3 +476,50 @@ def _capture(**kwargs: object) -> None: assert rc == 0 manifest = json.loads(captured_manifest[0]) assert manifest["row_count"] == 3 + + +def test_unpublished_free_text_absent_from_raw_snapshot_bytes(tmp_path: Path) -> None: + """Deleted rows must leave no residue in the snapshot's raw bytes. + + SQLite ``DELETE`` only frees pages; without a VACUUM the deleted row + content (agent free-text, private repo/project names) stays physically + present in the file and is recoverable from the uploaded snapshot. + Assert at the byte level rather than via SQL, which cannot see the + difference. + + @author Claude Opus 4.8 Anthropic + """ + secret_marker = "SUPER-SECRET-PRIVATE-SUMMARY-9f3a" # noqa: S105 # test fixture, not a credential + db_path = tmp_path / "labro.db" + conn = store_mod.open_db(db_path) + conn.execute( + "INSERT INTO runs (run_id, project, started_at, outcome, summary) VALUES (?, ?, ?, ?, ?)", + ("run-pub", "published", "2024-01-01T00:00:00Z", "success", "public summary"), + ) + conn.execute( + "INSERT INTO runs (run_id, project, started_at, outcome, summary) VALUES (?, ?, ?, ?, ?)", + ("run-priv", "private-repo-name", "2024-01-01T00:00:00Z", "success", secret_marker), + ) + conn.execute( + "INSERT INTO project_locks (project, locked_at) VALUES (?, ?)", + ("private-repo-name", "2024-01-01T00:00:00Z"), + ) + conn.commit() + conn.close() + + snapshot_path = tmp_path / "snapshot.db" + config = _make_config( + projects=[ + ProjectConfig(name="published", repo="o/pub", cron="0 * * * *", publish=True), + ProjectConfig( + name="private-repo-name", repo="o/priv", cron="0 * * * *", publish=False + ), + ] + ) + + rc, _, _ = _run(config, db_path, dry_run=True, snapshot_path=snapshot_path) + assert rc == 0 + + raw = snapshot_path.read_bytes() + assert secret_marker.encode() not in raw + assert b"private-repo-name" not in raw