From 3bd5867e222d5a2125b53eb93fca996acc8972b6 Mon Sep 17 00:00:00 2001 From: Lukas Wuttke Date: Fri, 11 Sep 2026 11:47:57 +0200 Subject: [PATCH] Import the quickstart guide from start-training (curated snapshot) Fresh single-commit import of the start-training tree at its develop tip (5af8e509d404), as part of the repository restructuring. The shared caller workflows, CLAUDE.md and .cursor/BUGBOT.md are left out (added separately for this repo); the self-contained template-rules workflow is kept so the template checker runs. README and notebook links now name tracebloc/quickstart; internal ticket and design-note references are scrubbed from the imported files. Co-Authored-By: Claude Fable 5.1 --- .github/pull_request_template.md | 28 + .github/workflows/template-rules.yml | 90 ++ .gitignore | 146 ++ LICENSE | 201 +++ README.md | 75 +- notebooks/GenerateCheckWeights.ipynb | 165 ++ notebooks/templates/README.md | 294 ++++ notebooks/templates/embeddings.ipynb | 192 +++ notebooks/templates/families.json | 164 ++ notebooks/templates/nlp_finetune.ipynb | 205 +++ notebooks/templates/nlp_generative.ipynb | 205 +++ notebooks/templates/survival.ipynb | 254 +++ notebooks/templates/tabular_timeseries.ipynb | 353 +++++ notebooks/templates/verification-dev.json | 282 ++++ notebooks/templates/vision_from_scratch.ipynb | 200 +++ notebooks/traceblocTrainingGuide.ipynb | 417 +++++ scripts/check_templates.py | 1363 +++++++++++++++++ scripts/check_templates_mutations.py | 700 +++++++++ 18 files changed, 5333 insertions(+), 1 deletion(-) create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/template-rules.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 notebooks/GenerateCheckWeights.ipynb create mode 100644 notebooks/templates/README.md create mode 100644 notebooks/templates/embeddings.ipynb create mode 100644 notebooks/templates/families.json create mode 100644 notebooks/templates/nlp_finetune.ipynb create mode 100644 notebooks/templates/nlp_generative.ipynb create mode 100644 notebooks/templates/survival.ipynb create mode 100644 notebooks/templates/tabular_timeseries.ipynb create mode 100644 notebooks/templates/verification-dev.json create mode 100644 notebooks/templates/vision_from_scratch.ipynb create mode 100644 notebooks/traceblocTrainingGuide.ipynb create mode 100755 scripts/check_templates.py create mode 100755 scripts/check_templates_mutations.py diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..cea0c18 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,28 @@ +## Summary + + +## Related + + +## Type of change +- [ ] Feature +- [ ] Bug fix +- [ ] Tech-debt / refactor +- [ ] Docs +- [ ] Security / hardening +- [ ] Breaking change + +## Test plan + + +## Screenshots / recordings + + +## Deployment notes + + +## Checklist +- [ ] Tests added / updated and passing locally +- [ ] Docs updated if behavior or config changed +- [ ] No secrets / credentials in the diff +- [ ] For security-sensitive paths: appropriate reviewer requested diff --git a/.github/workflows/template-rules.yml b/.github/workflows/template-rules.yml new file mode 100644 index 0000000..ef9e4c8 --- /dev/null +++ b/.github/workflows/template-rules.yml @@ -0,0 +1,90 @@ +name: Template rules + +# Runs the D9 template rule checker. Until this existed, NOTHING executed +# `scripts/check_templates.py`: the only CI change #89 made was arming the +# shared reusable's `ruff` job, which lints changed `.py` files and says +# nothing about the templates. So every "enforced" in families.json, the +# templates README, CLAUDE.md and #89's own body meant "enforced if the author +# remembers to run it locally" -- a checker that is never invoked is +# indistinguishable from one that always passes (start-training#89, 5/10). +# +# `paths:` is deliberately ABSENT. A path filter here would be the same defect +# one level up: edit only `scripts/check_templates.py` and the job that +# validates it would not run. + +on: + pull_request: + types: [opened, reopened, synchronize, ready_for_review] + push: + branches: [develop, staging, main] + +concurrency: + group: template-rules-${{ github.workflow }}-${{ github.ref }} + # PR runs supersede each other; PUSH runs must not. Cancelling on push + # leaves the earlier commit with a cancelled run and therefore NO verdict, + # which reads the same as never having been checked. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + d9-rules: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + # The SDK's `requires-python` excludes older interpreters, and pip + # answers that by SILENTLY BACKTRACKING to an ancient release rather + # than failing: measured, `pip install tracebloc` under 3.9 resolves + # to 0.8.1, whose `_SURVIVAL_FRAMEWORKS` happens to match the mirror + # -- so a release predating these templates "confirms" them. Pinning + # the interpreter is what makes the floor check below meaningful. + python-version: '3.11' + + - name: Install the SDK for the mirror cross-check + # `--no-deps` plus the handful of light imports `tracebloc/__init__` + # needs. This deliberately does NOT pull torch: the checker only reads + # `tracebloc.training.plan._SURVIVAL_FRAMEWORKS`, so a ~20 MB install + # buys a real cross-check instead of the "SDK not importable, skipping" + # note that made rule 14b advisory everywhere it mattered. + run: | + set -euo pipefail + python -m pip install --quiet --upgrade pip + python -m pip install --quiet --no-deps tracebloc + # Derived by FOLLOWING the import chain on a clean 3.11 venv, one + # missing module at a time, not by guessing: tracebloc 1.0.7's + # `training.plan` needs exactly these six. The first version of this + # step omitted numpy and the job went red -- correctly, because + # strict mode refuses to pass a mirror it cannot verify. Note pandas + # is NOT here: pip warns that tracebloc requires it, but that is the + # declared dependency set, not what this import path touches. + python -m pip install --quiet numpy psutil requests rich termcolor tqdm + # Fail here, loudly, rather than letting the checker report a + # vaguer "not importable" one step later. + python - <<'PY' + import importlib.metadata as m + from tracebloc.training.plan import _SURVIVAL_FRAMEWORKS + print("tracebloc", m.version("tracebloc"), + sorted(getattr(f, "value", f) for f in _SURVIVAL_FRAMEWORKS)) + PY + + - name: Mutation harness β€” every rule seen to FAIL + # Runs before the checker on purpose: the checker passing tells you the + # tree is clean, the harness tells you the checker can still fail. A + # deleted rule leaves the checker green (measured: deleting rule 16 + # still printed "OK -- all D9 rules hold") and only the harness catches + # it. + run: python3 scripts/check_templates_mutations.py + + - name: D9 rule checker + # Runs from the repo root; the checker resolves its own paths. + # TRACEBLOC_CHECK_STRICT makes "cannot determine the SDK version" a + # FAILURE here rather than a printed note, because a skipped + # cross-check in the one place that gates is the whole defect. + env: + TRACEBLOC_CHECK_STRICT: '1' + run: python3 scripts/check_templates.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..34af4a1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,146 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +xrays/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# ds store +.DS_Store/* + +.idea +.DS_Store + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d5f76c2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 tracebloc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 4b3721e..05c22f6 100644 --- a/README.md +++ b/README.md @@ -1 +1,74 @@ -# quickstart \ No newline at end of file +[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tracebloc/quickstart/blob/main/notebooks/traceblocTrainingGuide.ipynb) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE) [![Platform](https://img.shields.io/badge/platform-tracebloc-00C9A7.svg)](https://ai.tracebloc.io) + +# Quickstart πŸš€ + +Launch an ML training experiment on [tracebloc](https://tracebloc.io/) in under 10 minutes. Connect your account, upload a model, link it to a dataset, configure training parameters, and start benchmarking β€” all from a single notebook. + +## Get started + +### Option A: Google Colab (recommended) + +No local setup. Click the badge above or: + +**πŸ‘‰ [Open in Google Colab](https://colab.research.google.com/github/tracebloc/quickstart/blob/main/notebooks/traceblocTrainingGuide.ipynb)** + +Copy the notebook to your Drive and start running cells. + +### Option B: Run locally + +```bash +git clone https://github.com/tracebloc/quickstart.git +cd quickstart + +# Pick the extra that matches your ML framework: +pip install "tracebloc[pytorch]>=0.14.0" # most common +# pip install "tracebloc[sklearn]>=0.14.0" # scikit-learn / boosting +# pip install "tracebloc[all]>=0.14.0" # everything + +jupyter notebook notebooks/traceblocTrainingGuide.ipynb +``` + +**Which Pythons work:** whatever the SDK's own package metadata declares β€” see +[`tracebloc` on PyPI](https://pypi.org/project/tracebloc/). This README +deliberately does not repeat the range; a copy here would go stale against the +package, which is exactly the failure this notebook was fixed for. + +If pip answers `No matching distribution found for tracebloc`, that most often +means your interpreter is outside that range rather than the package being +missing β€” but it can also mean an unreachable index or a custom `--index-url`. +The install cell prints pip's own answer either way: on failure it shows the +range pip actually read alongside the Python you are on. On macOS the default +`python3` is frequently *ahead* of the supported range, so check +`python3 --version` first when running locally. + +TensorFlow uploads were removed in SDK 1.0.0, so there is no `[tensorflow]` +extra β€” the extras are `[pytorch]`, `[sklearn]`, `[catboost]`, `[lightgbm]`, +`[xgboost]`, `[lifelines]`, `[scikit-survival]` and `[all]`. + +## What the notebook covers + +| Step | What you do | +|:---:|---| +| **1** | Connect to tracebloc with your email + password | +| **2** | Upload a model from the [model zoo](https://github.com/tracebloc/model-zoo) or your own | +| **3** | Link it to a dataset from your use case | +| **4** | Configure training β€” epochs, batch size, learning rate, augmentation | +| **5** | Start training β€” model runs inside your secure Kubernetes environment | + +Results appear on the use case leaderboard in the [tracebloc web app](https://ai.tracebloc.io/). + +## Before you start + +- A **tracebloc account** β€” [sign up free](https://ai.tracebloc.io/signup) +- An **active use case** with a dataset β€” [how to join one](https://docs.tracebloc.io/join-use-case/) +- A **model file** β€” grab one from the [model zoo](https://github.com/tracebloc/model-zoo) or [build your own](https://docs.tracebloc.io/join-use-case/model-optimization) + +## Links + +[Platform](https://ai.tracebloc.io/) Β· [Docs](https://docs.tracebloc.io/) Β· [Model zoo](https://github.com/tracebloc/model-zoo) Β· [PyPI package](https://pypi.org/project/tracebloc/) Β· [Discord](https://discord.gg/tracebloc) + +## License + +Apache 2.0 β€” see [LICENSE](LICENSE). + +**Need help?** [support@tracebloc.io](mailto:support@tracebloc.io) or [open an issue](https://github.com/tracebloc/quickstart/issues). diff --git a/notebooks/GenerateCheckWeights.ipynb b/notebooks/GenerateCheckWeights.ipynb new file mode 100644 index 0000000..0f079bc --- /dev/null +++ b/notebooks/GenerateCheckWeights.ipynb @@ -0,0 +1,165 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0e68ce9a", + "metadata": {}, + "source": [ + "# Generate & Verify Model Weights\n", + "\n", + "A small utility for producing a starting-weights file that pairs with a tracebloc model file.\n", + "\n", + "Use this when:\n", + "- You wrote a custom model and need a `_weights.pkl` to upload alongside it\n", + "- You want to confirm a weights file loads correctly before running `user.upload_model(..., weights=True)`\n", + "\n", + "**Convention:** tracebloc expects a model file `mymodel.py` defining a function `MyModel()`, and a companion weights file named `mymodel_weights.pkl` (or `.pth` for PyTorch) in the same directory.\n", + "\n", + "\ud83d\udcd6 [Model structure requirements](https://docs.tracebloc.io/join-use-case/model-optimization)" + ] + }, + { + "cell_type": "markdown", + "id": "db0877bc", + "metadata": {}, + "source": [ + "## 1. Point at your model file\n", + "\n", + "Set the path to the `.py` file that defines `MyModel()`. The weights file will be written next to it." + ] + }, + { + "cell_type": "code", + "id": "732903bb", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "import importlib.util\nimport os\n\nMODEL_PATH = \"mymodel.py\" # <-- change to your model file\n\nmodel_dir = os.path.dirname(os.path.abspath(MODEL_PATH)) or \".\"\nmodel_name = os.path.splitext(os.path.basename(MODEL_PATH))[0]\n\nspec = importlib.util.spec_from_file_location(model_name, MODEL_PATH)\nmodule = importlib.util.module_from_spec(spec)\nspec.loader.exec_module(module)" + }, + { + "cell_type": "markdown", + "id": "d7e0f3ca", + "metadata": {}, + "source": [ + "## 2. Generate weights\n", + "\n", + "Pick the cell that matches your framework. Each one builds the model via `MyModel()` and writes a weights file using the format tracebloc expects." + ] + }, + { + "cell_type": "markdown", + "id": "c55d3110", + "metadata": {}, + "source": [ + "### PyTorch\n", + "\n", + "Saves the model's `state_dict` with `torch.save` \u2014 tracebloc loads it back via `model.load_state_dict(torch.load(...))`." + ] + }, + { + "cell_type": "code", + "id": "0356466d", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "import torch\n\nweights_path = os.path.join(model_dir, f\"{model_name}_weights.pth\")\n\nmodel = module.MyModel()\ntorch.save(model.state_dict(), weights_path)\nprint(f\"Wrote {weights_path}\")" + }, + { + "cell_type": "markdown", + "id": "25068e30", + "metadata": {}, + "source": [ + "### TensorFlow / Keras\n", + "\n", + "Saves the weights as a pickled list of arrays \u2014 tracebloc loads it back via `model.set_weights(pickle.load(...))`." + ] + }, + { + "cell_type": "code", + "id": "0c902690", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "import pickle\n\nweights_path = os.path.join(model_dir, f\"{model_name}_weights.pkl\")\n\nmodel = module.MyModel()\nwith open(weights_path, \"wb\") as f:\n pickle.dump(model.get_weights(), f)\nprint(f\"Wrote {weights_path}\")" + }, + { + "cell_type": "markdown", + "id": "fcdd9389", + "metadata": {}, + "source": [ + "## 3. Verify the weights file\n", + "\n", + "Reload the weights into a fresh model instance to confirm shapes match and the file is readable. Run the cell that matches your framework." + ] + }, + { + "cell_type": "markdown", + "id": "62c3b96f", + "metadata": {}, + "source": [ + "### PyTorch" + ] + }, + { + "cell_type": "code", + "id": "2a98f6ae", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "import torch\n\nweights_path = os.path.join(model_dir, f\"{model_name}_weights.pth\")\n\nverify_model = module.MyModel()\nverify_model.load_state_dict(torch.load(weights_path))\nprint(f\"Loaded weights into {type(verify_model).__name__}\")\nprint(f\" parameters: {sum(p.numel() for p in verify_model.parameters()):,}\")" + }, + { + "cell_type": "markdown", + "id": "860d0ad5", + "metadata": {}, + "source": [ + "### TensorFlow / Keras" + ] + }, + { + "cell_type": "code", + "id": "bb9a78ae", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "import pickle\n\nweights_path = os.path.join(model_dir, f\"{model_name}_weights.pkl\")\n\nverify_model = module.MyModel()\nwith open(weights_path, \"rb\") as f:\n verify_model.set_weights(pickle.load(f))\nverify_model.summary()" + }, + { + "cell_type": "markdown", + "id": "8ac220ec", + "metadata": {}, + "source": [ + "## 4. Upload with your model\n", + "\n", + "Once the weights file is in place, upload both together from the main training notebook:\n", + "\n", + "```python\n", + "user.upload_model(MODEL_PATH, weights=True)\n", + "```\n", + "\n", + "The SDK will look for `_weights.pkl` (or `.pth`) next to your `.py` file." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/notebooks/templates/README.md b/notebooks/templates/README.md new file mode 100644 index 0000000..753f6ec --- /dev/null +++ b/notebooks/templates/README.md @@ -0,0 +1,294 @@ +# Family training templates + +One template per **family**, not per `(category, framework)` pair. There are +six families here and twenty legal pairs; the pairs are covered by conditional +cells inside a family's template rather than by a file each. Twenty +hand-maintained notebooks rot within a quarter β€” six families do not. + +`notebook.render` still resolves by `(category, framework)`. It is the *files* +that are per family, not the lookup. + +These are **not** the user-facing guide. They carry no Colab lines and no +install cell: the pod they render in already has the SDK and the model zoo. They +also carry no login prompt β€” though that one is a *target*, not yet true; see +"Two dependencies" below, because environment login is merged and unreleased. +For the guide, see +[`../traceblocTrainingGuide.ipynb`](../traceblocTrainingGuide.ipynb). + +## The families and their pre-fills + +| family | categories | `cycles` | `epochs` | +|---|---|---|---| +| [Vision, from scratch](vision_from_scratch.ipynb) | `image_classification`, `object_detection`, `semantic_segmentation`, `keypoint_detection` | 20 | 1 | +| [NLP fine-tune (incl. LoRA)](nlp_finetune.ipynb) | `text_classification`, `sentence_pair_classification`, `token_classification` | 8 | 1 | +| [NLP generative](nlp_generative.ipynb) | `causal_language_modeling`, `seq2seq`, `masked_language_modeling` | 5 | 1 | +| [Embeddings](embeddings.ipynb) | `embeddings` | 8 | 1 | +| [Tabular / time series](tabular_timeseries.ipynb) | `tabular_classification`, `tabular_regression`, `time_series_classification`, `time_series_forecasting` | 15 | 1 | +| [Survival](survival.ipynb) | `time_to_event_prediction` | 1 | 1 | + +> **What actually enforces this.** `scripts/check_templates.py` is run by +> `.github/workflows/template-rules.yml` on every PR. Before that workflow +> existed nothing ran it, so every "enforced" below meant "enforced if the +> author remembers" β€” the defect this whole file is otherwise about +> (start-training#89, 5/10). It is **not yet a required status check**; +> until an admin adds it to develop's contexts, a red here is visible but +> not blocking. + +The numbers live in [`families.json`](families.json), and +[`../../scripts/check_templates.py`](../../scripts/check_templates.py) asserts +that each notebook's settings cell really sets them β€” so this table, that file +and the notebooks cannot drift apart. + +**`cycles` is what federates; `epochs` is what drifts.** The SDK's own +`epochs=10, cycles=1` is one round with ten local epochs, which is the textbook +client-drift setup and the worst corner of the trade-off; it is deliberately not +any template's pre-fill. + +Two rules bind every row, both enforced by the checker: + +1. **`cycles Γ— epochs ≀ 20` for any pre-fill** β€” so a peer's first Start can + never be the reason the team's budget is spent. +2. **No template pre-fills `epochs > 1` unless its aggregation strategy carries + a drift correction** (`fedprox`, `fedadam`, `fedyogi`, `fedadagrad`). Plain + FedAvg with many local epochs *is* the drift setup, so the cell comment and + the number sitting next to it have to agree. + +The checker also holds the rule that a family template must never offer a +setter to a category that refuses it. It carries the setter-to-category map +read off the SDK's own gates, and audits every cell against the categories that +cell is offered to β€” the whole family for a shared cell, only the gated subset +for a fragment. Three real instances of that mistake were found by hand in the +Tabular / time series template before the check existed (`feature_points`, +which only `tabular_classification` takes; `encoding_strategy` and +`normalize_features`, which the forecasting path does not read; and a `scaler` +default that was right for tabular and wrong for time series). + +**A `cycles` pre-fill cannot reach a single-pass framework.** The SDK forces +`cycles` and `epochs` to 1 for `sklearn`, `lifelines` and `scikit_survival` β€” +by *framework*, for every category β€” and it only warns, so an ungated +`cycles(15)` on a sklearn tabular model prints "cycles cannot be updated" and +quietly trains one round. D9's table gives one `cycles` per family keyed on +category, which cannot be honoured for those pairs. Tabular / time series +(`sklearn`) and Survival (`lifelines`, `scikit_survival`) therefore gate +`cycles`/`epochs` on framework: pytorch pairs get the pre-fill, single-pass +pairs get a comment saying both are forced to 1. `families.json` records each +family's `frameworks` and the `single_pass_frameworks` set, and the checker +refuses an ungated pre-fill on a family with such a pair. + +The `single_pass_frameworks` list is a **mirror** of the SDK's +`tracebloc.training.plan._SURVIVAL_FRAMEWORKS`, and a mirror goes stale +silently β€” the same objection this repo's CLAUDE.md raises about restating the +SDK's Python bound. The checker therefore *derives* the set when the SDK is +importable and fails on drift. The SDK is not a dependency here (it pulls +torch), so CI cannot import it β€” and in that case the checker **prints** that +the cross-check was skipped rather than passing quietly, because an +unverifiable mirror that says nothing is exactly the can't-fail shape this +checker exists to catch. + +No family currently pre-fills `epochs > 1`, so none needs a drift correction. +The vision row buys its twenty effective passes with rounds (`cycles = 20`) +rather than local epochs, which is rule 2's whole point: from-scratch vision is +exactly where drift bites hardest. + +### What the checker renders + +Rule 12 renders every `(category, framework)` pair from the family's **own** +`categories` and `frameworks` lists. It used to hardcode `pytorch` and +`sklearn`, which both skipped real pairs β€” Survival's `lifelines` and +`scikit_survival` were never rendered, so a fragment gated only on those could +carry a syntax error while the check reported all rules holding β€” and rendered +illegal ones like vision + `sklearn`. Found by Cursor Bugbot. + +`frameworks` is per family, so this still over-approximates: it renders +`(time_series_classification, sklearn)` though the zoo ships no sklearn model +there. That is the safe direction for a validator β€” it can only demand more +validity, never less β€” and the precise legal-pair matrix belongs to the engine +registry rather than being restated here. + +## Verification status + +D9: *a template ships only once a live experiment on dev has reached COMPLETED +using its own defaults* β€” a pre-fill nobody has run is a guess with a Start +button attached. + +The records are in [`verification-dev.json`](verification-dev.json), committed +rather than left on the dev cluster: dev gets swept, and a verification that +depends on state we do not control is a claim, not a proof. The checker's +rule 13 binds that file to `families.json`, so **editing a pre-fill here fails +the check** until the dev run is redone and the evidence refreshed. A family +that cannot be verified must instead carry `unverified` in `families.json` +naming the ticket that explains why; a family that is neither is refused. + +Verified on the complete settings cell: + +| family | `cycles` | experiment | minutes | +|---|---|---|---| +| [Vision, from scratch](vision_from_scratch.ipynb) | 20 | `eaxx647p` | 40.3 | +| [NLP fine-tune](nlp_finetune.ipynb) | 8 | `enpm5bls` | 16.9 | +| [NLP generative](nlp_generative.ipynb) | 5 | `e6lymzry` | 11.4 | +| [Embeddings](embeddings.ipynb) | 8 | `erl4u2pk` | 18.0 | +| [Tabular / time series](tabular_timeseries.ipynb) | 15 | `eeofzvuj` | 28.9 | + +**[Survival](survival.ipynb) is UNVERIFIED** β€” tracked internally. Both dev +`time_to_event_prediction` datasets carry a `label` of time-like values instead +of the 0/1 event indicator, so the engine's TTE validator refuses every +experiment on them, and there is no other TTE dataset on dev. Its pre-fills are +not in question; the evidence is missing, and by D9's own rule that means the +template has not shipped. + +An earlier round applied only `cycles`/`epochs`/`training_classes`. Every other +field *coincided* with the template except `callbacks`, which came back `'[]'` +because `terminate_on_nan_callback()` was never called β€” a coincidence reads +exactly like a verification until someone reads the record. Both rounds are +kept in the evidence file, scoped for what each actually shows. + +Those durations are also, as far as this epic has measured, the only dev-edge +timing baseline for federated runs by family. The second round was *faster* +than the first on every family despite six runs sharing the edge, so they are a +usable baseline rather than a contended outlier. `estimatedflops` is **not** a +progress meter, incidentally: completed runs land at 80-95% of it. + +## The render contract + +The backend stores no template and renders nothing. It passes the context as +environment; the SDK **in the pod** renders with +`notebook.render(category, framework, context)` from the templates in the image. +One manifest pins SDK, zoo and templates together, so a template cannot be +rendered by an SDK that does not match it. + +Rendering does two things, and only these two: + +**1. Substitute `{{ key }}` placeholders** from the context. Keys used here: + +| key | what it is | +|---|---| +| `use_case` | use-case name, for the facts table | +| `dataset_id` | dataset key to link against | +| `category`, `framework` | the pair being rendered | +| `edge_count`, `records_per_edge` | facts-table figures | +| `experiment_name` | pre-filled *β€Ήmodelβ€Ί on β€Ήuse caseβ€Ί #β€Ήnβ€Ί* | +| `model_path` | path the picker chose | +| `validation_split` | dataset-derived, not a family constant β€” the SDK computes it from the smallest edge's record count and the class count | +| `training_classes` | per-class subsample map | +| `data_type` | `rgb` or `grayscale`; a 1-channel dataset fails the channel check at the rgb default | +| `feature_points` | column count; must agree with the dataset or the link is refused | +| `sequence_length`, `forecast_horizon` | sequence shape | +| `scaler` | category-derived, not a family constant β€” `MinMaxScaler` for time series, `StandardScaler` for tabular and time-to-event | +| `tokenizer_path` | contributor `tokenizer.json`, when one is not resolvable by name | + +**2. Drop cells that do not apply.** A cell carrying +`metadata.tracebloc.applies_to` is kept only when the pair being rendered +matches it: + +- `{"category": [...]}` β€” keep only for these categories. Used for the Tabular + / time series settings fragments, whose settings genuinely differ by + category: `feature_points` (only `tabular_classification` of that family + takes it), `sequence_length` (time series only), `forecast_horizon` + (forecasting only), `missingness_indicators` (time-series classification + only), and the `encoding_strategy` / `normalize_features` pair (everything + but forecasting). + + > An earlier draft cited the custom-loss cell here as an + > `object_detection`-only gate. That is no longer true and was the defect + > described under *Verification* below: the cell is ungated in the settings + > cell of every family that accepts a custom loss, because all of them do + > except embeddings. +- `{"framework": [...]}` β€” keep only for these frameworks. Used for the vision + augmentation group: those ten setters are gated on the *framework*, not the + category, and they **refuse** rather than warn, so calling one on a + non-pytorch model poisons the plan and Start then blocks. `shuffle`, the + eleventh, is not framework-gated and stays in the shared cell. +- `{"dataset_flag": "allow_feature_modification"}` β€” keep only when the + dataset sets that flag. Used for the feature-interaction cell, which is also + a settings fragment: its calls have to land *in* the settings cell to be + applied at all (see the `start()` note under *What is deliberately absent*). + +Cells without that metadata are always kept. + +The checker **executes** this contract rather than describing it: for every +family, for each category it owns, it renders the template and compiles the +resulting settings cell. That is what catches a placeholder sitting in a +position where no real value parses β€” a `{{ key }}` inside a string literal and +one standing as a bare argument are not interchangeable, and nothing else would +notice. + +**Settings fragments.** A family can span categories that do not take the same +settings β€” Tabular / time series is the case that forces this: `sequence_length` +is meaningless for `tabular_classification`, `forecast_horizon` applies only to +forecasting, and `missingness_indicators` only to time-series classification. +Those live in cells marked `metadata.tracebloc.settings_fragment: true` +alongside their `applies_to` gate. A fragment that survives the gate is +**concatenated into the single settings cell**, in document order, after the +main block; one that does not is dropped. + +That keeps both halves of the design true at once: the peer still sees *one* +settings cell with the complete applicable settings and no inapplicable group, +and one file still covers four categories. A fragment with no `applies_to` +would apply to the whole family and belongs in the main block instead β€” the +checker rejects it, because left as a fragment it quietly becomes a second +settings cell. + +## Two dependencies that are not satisfied yet + +**Environment login is merged but NOT RELEASED.** The connect cell's premise β€” +the pod is already authenticated and there is no password prompt β€” needs an SDK +*release* carrying environment login. It is on the SDK's `develop` +(pyproject 1.0.9) and **absent from the latest tag v1.0.7**, which is what +`pip install tracebloc` resolves; v1.0.7 has no `env_login` module and no +`TRACEBLOC_TOKEN` path at all (verified 2026-09-09). Until a release ships it, +`User()` prompts interactively, which is wrong for a pod. The cell says so in +the cell. Keying this on the change *merging* would have looked satisfied the +moment it did. + +**`notebook.render` does not exist yet.** These files are templates and the +render contract above is a specification for the SDK side, not a description of +shipped behaviour. + +## What is deliberately absent + +- **`start()`.** No cell calls it. Start is a button, and there is no Run All: + it re-links, executes the settings cell, then starts β€” in that order, because + `start()` is one-shot and resets the plan. + + **A corollary that cost this PR a round of review:** because Start executes + *only* the settings cell, a `training.*` call in any other cell is **inert**, + however inviting it looks. An earlier draft put the custom-loss and + feature-interaction calls in standalone cells sitting *before* the link, so + uncommenting one applied nothing β€” and for a YOLO model, which cannot start + without a custom loss, that made the run unstartable. Every setter now lives + in the settings cell or in a gated fragment concatenated into it, and the + checker refuses any that does not (found by Cursor Bugbot). +- **The five dead setters** β€” `horizontal_flip`, `vertical_flip`, + `samplewise_center`, `samplewise_std_normalization`, `layers_freeze`. They are + no-ops on every surviving framework and `start()` refuses some of them. They + are still public API, so their removal needs its own major bump; a template + pre-filling them would be pre-filling a failure. The checker rejects them. +- **A custom-loss cell in the Embeddings template.** The contrastive objective + is intrinsic (InfoNCE over in-batch negatives) and the SDK rejects a supplied + loss for that category, so offering the cell would offer a rejection. It is + the *only* such family: the SDK's NLP base hook documents itself as "Base: + every family supports one β€” no-op" and embeddings alone overrides it to + raise, so every other template carries the cell. An earlier draft gated it to + `object_detection` only, which hid a supported feature from eleven + categories; `families.json` now records the exception list and the checker + enforces both directions β€” by *coverage*, not presence: the offering must + reach every category in the family. A first version of that rule asked only + whether `training.loss_function` appeared *somewhere* in the file, and so + passed the very defect it was written to stop. +- **Augmentation outside the vision template.** The eleven live augmentation + setters apply to pytorch image models only; the sklearn branches are refused + outright. +- **`data_shape` in the vision template.** Image size is fixed per model for + pytorch and is not user-settable β€” `start()` refuses the call. + +## Ownership + +Per *family*, not per pair β€” six owners, in Data Science. Defaults recorded +2026-09-08. The conditional-cell mechanism above is what makes the smaller +number work. + +> **Reaching users.** These templates render inside the pod, so they reach a +> peer through the notebook image, not through the Colab link. They are +> unrelated to the Drive-hosted copy of the *guide* that the web app's "Start +> training" button still points at β€” nothing in this +> repo can change that copy, and nothing here needs to. diff --git a/notebooks/templates/embeddings.ipynb b/notebooks/templates/embeddings.ipynb new file mode 100644 index 0000000..2d4ca2f --- /dev/null +++ b/notebooks/templates/embeddings.ipynb @@ -0,0 +1,192 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Embeddings \u2014 training template\n", + "\n", + "Renders for every `(category, framework)` pair in this family:\n", + "`embeddings`.\n", + "\n", + "This is a **template**, not a guide. The pod renders it for one use case with\n", + "`notebook.render(category, framework, context)`: `{{ ... }}` placeholders are\n", + "substituted from the context, and cells carrying\n", + "`metadata.tracebloc.applies_to` are dropped when they do not apply to the pair\n", + "being rendered. See `README.md` in this directory for the contract.\n", + "\n", + "**Nothing here calls `start()`.** Start is a button, and there is no Run All." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## This run\n", + "\n", + "| | |\n", + "|---|---|\n", + "| Use case | {{ use_case }} |\n", + "| Dataset | `{{ dataset_id }}` |\n", + "| Category | `{{ category }}` |\n", + "| Framework | `{{ framework }}` |\n", + "| Edges | {{ edge_count }} |\n", + "| Records per edge | {{ records_per_edge }} |" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# PENDING A RELEASED SDK. Once the image carries an SDK release with\n", + "# environment login, the pod is already authenticated -- it reads its scoped,\n", + "# short-lived credential from the environment, so there is no email/password\n", + "# prompt here and no token in the notebook.\n", + "#\n", + "# Until then `User()` PROMPTS interactively, which is wrong for a pod. Note\n", + "# that merged is not enough: environment login is on the SDK's `develop`\n", + "# (pyproject 1.0.9) but ABSENT from the latest tag v1.0.7, which is what\n", + "# `pip install tracebloc` resolves -- so this cell is contingent on a RELEASE,\n", + "# not on the change landing. Verified 2026-09-09: v1.0.7 contains no\n", + "# `env_login` module and no `TRACEBLOC_TOKEN` path at all.\n", + "from tracebloc import User\n", + "\n", + "user = User()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Filled in by the model picker. Change the path to point at your own file.\n", + "MODEL_PATH = \"{{ model_path }}\"\n", + "\n", + "user.upload_model(MODEL_PATH)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# --- Tokenizer -----------------------------------------------------------\n", + "# NLP models must ship a tokenizer; there is no fallback. The SDK picks up a\n", + "# `_tokenizer.json` sitting next to the model file, so most zoo models\n", + "# need nothing here. Pass one explicitly only when yours is named differently:\n", + "#\n", + "# user.upload_model(MODEL_PATH, tokenizer=\"{{ tokenizer_path }}\")\n", + "#\n", + "# For a HuggingFace-hosted tokenizer, set the model file's `tokenizer_id`\n", + "# instead. An empty `tokenizer_id` is refused at upload, not at training." + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "training = user.link_model_dataset(\"{{ dataset_id }}\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# ======================================================================\n", + "# Settings \u2014 the complete plan for this run, as plain SDK calls.\n", + "# Edit a value, then press Start. Nothing here calls start().\n", + "# ======================================================================\n", + "\n", + "# --- Experiment ----------------------------------------------------------\n", + "training.experiment_name(\"{{ experiment_name }}\")\n", + "\n", + "\n", + "# --- Federation ----------------------------------------------------------\n", + "# cycles = federated rounds; epochs = local epochs per round\n", + "# Contrastive training benefits from more rounds of in-batch negatives rather than more local epochs over the same batches.\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#training-parameters\n", + "training.cycles(8)\n", + "training.epochs(1)\n", + "# Plain FedAvg is safe here because epochs is 1: there is no local drift to\n", + "# correct. Raising epochs above 1 means moving to a drift-correcting strategy\n", + "# (fedprox, fedadam, fedyogi, fedadagrad) in the same edit.\n", + "training.aggregation_strategy(\"fedavg\")\n", + "\n", + "# --- Optimization --------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#1-optimizer\n", + "training.optimizer(\"sgd\")\n", + "training.learning_rate({\"type\": \"constant\", \"value\": 0.001})\n", + "training.seed(0) # 0 means no fixed seed\n", + "\n", + "# --- Data ----------------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#dataset-parameters-optional\n", + "training.validation_split({{ validation_split }})\n", + "training.training_classes({{ training_classes }})\n", + "\n", + "# --- Sequence ------------------------------------------------------------\n", + "# Maximum token sequence length. Longer costs quadratically in attention.\n", + "# https://docs.tracebloc.io/join-use-case/how-training-works#per-use-case\n", + "training.sequence_length({{ sequence_length }})\n", + "\n", + "# --- LoRA ----------------------------------------------------------------\n", + "# Off by default. Target modules are derived for you; only the four knobs\n", + "# below are yours. LoRA changes what is averaged: adapters, not full weights.\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#llm-parameters-text-classification\n", + "# training.enable_lora(True)\n", + "# training.set_lora_parameters(256, 512, 0.05, False) # r, alpha, dropout, q_lora\n", + "\n", + "# --- Callbacks -----------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#callbacks\n", + "training.terminate_on_nan_callback()\n", + "# training.early_stop_callback(monitor=\"val_loss\", patience=3)\n", + "# training.model_checkpoint_callback(monitor=\"val_loss\", save_best_only=True)\n", + "# training.reduce_lr_callback(monitor=\"val_loss\", factor=0.1, patience=2, min_delta=1e-4)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Start\n", + "\n", + "Press **Start**. It re-links the model and dataset, executes the settings cell\n", + "above, and then starts the experiment \u2014 in that order, because `start()` is\n", + "one-shot and resets the plan. Your remaining team budget is shown beside the\n", + "button.\n", + "\n", + "To iterate: change a value above and press Start again.\n", + "\n", + "Prefer to leave? *Download .ipynb* and *Copy as script* both give you the same\n", + "settings as plain SDK calls." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + }, + "tracebloc": { + "template": "embeddings", + "family": "Embeddings", + "categories": [ + "embeddings" + ] + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/templates/families.json b/notebooks/templates/families.json new file mode 100644 index 0000000..83c3b3a --- /dev/null +++ b/notebooks/templates/families.json @@ -0,0 +1,164 @@ +{ + "_comment": [ + "The family-template pre-fill table, machine-readable. This file is the single", + "source for the numbers; scripts/check_templates.py asserts that each", + "template's settings cell actually sets these values, so the table and the", + "notebooks cannot drift apart.", + "", + "Two rules bind every row, both enforced by that checker:", + " 1. cycles * epochs <= 20 for any pre-fill.", + " 2. No family pre-fills epochs > 1 unless its aggregation strategy carries", + " a drift correction. Plain FedAvg with many local epochs IS the", + " client-drift setup.", + "No row currently pre-fills epochs > 1, so no row needs a drift correction.", + "cycles is what federates; epochs is what drifts.", + "", + "A family is VERIFIED when verification-dev.json records a COMPLETED", + "full-settings-cell run whose cycles/epochs match this file. A family that", + "is not must carry `unverified` naming the ticket that explains why; the", + "checker refuses a family that is neither. So changing a pre-fill here fails", + "the check until the dev run is redone and the evidence refreshed.", + "", + "`frameworks` is the set of frameworks the model zoo actually ships for this", + "family's categories, with the retired TensorFlow ones excluded. It matters", + "because the SDK forces cycles and epochs to 1 for the frameworks listed in", + "`single_pass_frameworks` -- by FRAMEWORK, for every category -- so a family", + "with such a pair cannot offer one cycles value to all of them. Measured: a", + "`cycles(15)` on a sklearn model prints \"cycles cannot be updated\" and sets 1.", + "Those families gate cycles/epochs on framework instead; the checker enforces it.", + "", + "`no_custom_loss_families` lists the families whose objective is INTRINSIC, so", + "a supplied loss.py is refused at upload rather than ignored at training. Only", + "embeddings: the SDK's NLP base hook documents itself as \"Base: every family", + "supports one -- no-op\" and embeddings alone overrides it to raise. Every other", + "template therefore carries a custom-loss cell; the checker enforces both", + "directions.", + "", + "`sdk_version_floor` is the oldest tracebloc release whose agreement with", + "`single_pass_frameworks` means anything. Measured: pip under an interpreter", + "the SDK's requires-python excludes backtracks silently to 0.8.1, which", + "predates these templates and whose _SURVIVAL_FRAMEWORKS matches by", + "coincidence -- so an ancient install 'confirmed' the mirror." + ], + "drift_correcting_strategies": [ + "fedprox", + "fedadam", + "fedyogi", + "fedadagrad" + ], + "families": [ + { + "key": "vision_from_scratch", + "title": "Vision, from scratch", + "template": "vision_from_scratch.ipynb", + "categories": [ + "image_classification", + "object_detection", + "semantic_segmentation", + "keypoint_detection" + ], + "cycles": 20, + "epochs": 1, + "aggregation_strategy": "fedavg", + "rationale": "Twenty effective passes bought with rounds, not local epochs. From-scratch vision is where client drift bites hardest, so this row is the clearest case for spending the budget on cycles.", + "frameworks": [ + "pytorch" + ] + }, + { + "key": "nlp_finetune", + "title": "NLP fine-tune (incl. LoRA)", + "template": "nlp_finetune.ipynb", + "categories": [ + "text_classification", + "sentence_pair_classification", + "token_classification" + ], + "cycles": 8, + "epochs": 1, + "aggregation_strategy": "fedavg", + "rationale": "A pretrained encoder needs few passes to fit a head; eight rounds leaves budget for a second and third experiment.", + "frameworks": [ + "pytorch" + ] + }, + { + "key": "nlp_generative", + "title": "NLP generative", + "template": "nlp_generative.ipynb", + "categories": [ + "causal_language_modeling", + "seq2seq", + "masked_language_modeling" + ], + "cycles": 5, + "epochs": 1, + "aggregation_strategy": "fedavg", + "rationale": "Generative runs are the most expensive per pass, so the pre-fill is the most conservative of the language families.", + "frameworks": [ + "pytorch" + ] + }, + { + "key": "embeddings", + "title": "Embeddings", + "template": "embeddings.ipynb", + "categories": [ + "embeddings" + ], + "cycles": 8, + "epochs": 1, + "aggregation_strategy": "fedavg", + "rationale": "Contrastive training benefits from more rounds of in-batch negatives rather than more local epochs over the same batches.", + "frameworks": [ + "pytorch" + ] + }, + { + "key": "tabular_timeseries", + "title": "Tabular / time series", + "template": "tabular_timeseries.ipynb", + "categories": [ + "tabular_classification", + "tabular_regression", + "time_series_classification", + "time_series_forecasting" + ], + "cycles": 15, + "epochs": 1, + "aggregation_strategy": "fedavg", + "rationale": "Cheap passes over small feature tables; fifteen rounds still sits inside the twenty-pass ceiling.", + "frameworks": [ + "pytorch", + "sklearn" + ] + }, + { + "key": "survival", + "title": "Survival", + "template": "survival.ipynb", + "categories": [ + "time_to_event_prediction" + ], + "cycles": 1, + "epochs": 1, + "aggregation_strategy": "fedavg", + "rationale": "The survival frameworks (sklearn, lifelines, scikit-survival) train in a single pass and the SDK forces epochs to 1 for them; the pre-fill matches that rather than fighting it.", + "unverified": "tracked internally: the dev time_to_event_prediction datasets carry time-like labels instead of a 0/1 event indicator", + "frameworks": [ + "pytorch", + "lifelines", + "scikit_survival" + ] + } + ], + "single_pass_frameworks": [ + "sklearn", + "lifelines", + "scikit_survival" + ], + "no_custom_loss_families": [ + "embeddings" + ], + "sdk_version_floor": "1.0.7" +} diff --git a/notebooks/templates/nlp_finetune.ipynb b/notebooks/templates/nlp_finetune.ipynb new file mode 100644 index 0000000..645513a --- /dev/null +++ b/notebooks/templates/nlp_finetune.ipynb @@ -0,0 +1,205 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# NLP fine-tune (incl. LoRA) \u2014 training template\n", + "\n", + "Renders for every `(category, framework)` pair in this family:\n", + "`text_classification`, `sentence_pair_classification`, `token_classification`.\n", + "\n", + "This is a **template**, not a guide. The pod renders it for one use case with\n", + "`notebook.render(category, framework, context)`: `{{ ... }}` placeholders are\n", + "substituted from the context, and cells carrying\n", + "`metadata.tracebloc.applies_to` are dropped when they do not apply to the pair\n", + "being rendered. See `README.md` in this directory for the contract.\n", + "\n", + "**Nothing here calls `start()`.** Start is a button, and there is no Run All." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## This run\n", + "\n", + "| | |\n", + "|---|---|\n", + "| Use case | {{ use_case }} |\n", + "| Dataset | `{{ dataset_id }}` |\n", + "| Category | `{{ category }}` |\n", + "| Framework | `{{ framework }}` |\n", + "| Edges | {{ edge_count }} |\n", + "| Records per edge | {{ records_per_edge }} |" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# PENDING A RELEASED SDK. Once the image carries an SDK release with\n", + "# environment login, the pod is already authenticated -- it reads its scoped,\n", + "# short-lived credential from the environment, so there is no email/password\n", + "# prompt here and no token in the notebook.\n", + "#\n", + "# Until then `User()` PROMPTS interactively, which is wrong for a pod. Note\n", + "# that merged is not enough: environment login is on the SDK's `develop`\n", + "# (pyproject 1.0.9) but ABSENT from the latest tag v1.0.7, which is what\n", + "# `pip install tracebloc` resolves -- so this cell is contingent on a RELEASE,\n", + "# not on the change landing. Verified 2026-09-09: v1.0.7 contains no\n", + "# `env_login` module and no `TRACEBLOC_TOKEN` path at all.\n", + "from tracebloc import User\n", + "\n", + "user = User()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Filled in by the model picker. Change the path to point at your own file.\n", + "MODEL_PATH = \"{{ model_path }}\"\n", + "\n", + "user.upload_model(MODEL_PATH)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# --- Tokenizer -----------------------------------------------------------\n", + "# NLP models must ship a tokenizer; there is no fallback. The SDK picks up a\n", + "# `_tokenizer.json` sitting next to the model file, so most zoo models\n", + "# need nothing here. Pass one explicitly only when yours is named differently:\n", + "#\n", + "# user.upload_model(MODEL_PATH, tokenizer=\"{{ tokenizer_path }}\")\n", + "#\n", + "# For a HuggingFace-hosted tokenizer, set the model file's `tokenizer_id`\n", + "# instead. An empty `tokenizer_id` is refused at upload, not at training." + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "training = user.link_model_dataset(\"{{ dataset_id }}\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# ======================================================================\n", + "# Settings \u2014 the complete plan for this run, as plain SDK calls.\n", + "# Edit a value, then press Start. Nothing here calls start().\n", + "# ======================================================================\n", + "\n", + "# --- Experiment ----------------------------------------------------------\n", + "training.experiment_name(\"{{ experiment_name }}\")\n", + "\n", + "\n", + "# --- Federation ----------------------------------------------------------\n", + "# cycles = federated rounds; epochs = local epochs per round\n", + "# A pretrained encoder needs few passes to fit a head; eight rounds leaves budget for a second and third experiment.\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#training-parameters\n", + "training.cycles(8)\n", + "training.epochs(1)\n", + "# Plain FedAvg is safe here because epochs is 1: there is no local drift to\n", + "# correct. Raising epochs above 1 means moving to a drift-correcting strategy\n", + "# (fedprox, fedadam, fedyogi, fedadagrad) in the same edit.\n", + "training.aggregation_strategy(\"fedavg\")\n", + "\n", + "# --- Optimization --------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#1-optimizer\n", + "training.optimizer(\"sgd\")\n", + "training.learning_rate({\"type\": \"constant\", \"value\": 0.001})\n", + "training.seed(0) # 0 means no fixed seed\n", + "\n", + "# --- Data ----------------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#dataset-parameters-optional\n", + "training.validation_split({{ validation_split }})\n", + "training.training_classes({{ training_classes }})\n", + "\n", + "# --- Sequence ------------------------------------------------------------\n", + "# Maximum token sequence length. Longer costs quadratically in attention.\n", + "# https://docs.tracebloc.io/join-use-case/how-training-works#per-use-case\n", + "training.sequence_length({{ sequence_length }})\n", + "\n", + "# --- LoRA ----------------------------------------------------------------\n", + "# Off by default. Target modules are derived for you; only the four knobs\n", + "# below are yours. LoRA changes what is averaged: adapters, not full weights.\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#llm-parameters-text-classification\n", + "# training.enable_lora(True)\n", + "# training.set_lora_parameters(256, 512, 0.05, False) # r, alpha, dropout, q_lora\n", + "\n", + "# --- Callbacks -----------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#callbacks\n", + "training.terminate_on_nan_callback()\n", + "# training.early_stop_callback(monitor=\"val_loss\", patience=3)\n", + "# training.model_checkpoint_callback(monitor=\"val_loss\", save_best_only=True)\n", + "# training.reduce_lr_callback(monitor=\"val_loss\", factor=0.1, patience=2, min_delta=1e-4)\n", + "\n", + "# --- Custom loss ---------------------------------------------------------\n", + "# Optional. Point at a `loss.py` next to your model file; it is validated at\n", + "# LINK time, so a loss that returns a non-scalar, a NaN, or a tensor detached\n", + "# from the graph is refused there rather than wasting a training run.\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#3-loss-function\n", + "#\n", + "# training.loss_function({\"type\": \"custom\", \"value\": \"loss.py\"})\n", + "#\n", + "# Or pick a standard one:\n", + "# Or a standard one. THE VALID VALUES DEPEND ON THE PAIR, and the SDK\n# refuses anything outside its allowlist by poisoning the plan, so Start\n# then blocks -- an earlier draft suggested \"crossentropy\" here and that is\n# wrong for several pairs this template renders for:\n# time_series_forecasting mse | l1\n# time_to_event_prediction coxph\n# any sklearn pair mse | binarycrossentropy (custom loss.py refused)\n# lifelines / scikit_survival a custom loss.py is IGNORED, not applied\n# everything else crossentropy | mse | l1\n# training.loss_function({\"type\": \"standard\", \"value\": \"\"})\n#\n# Per-category suggestions via gated fragments are tracked separately;\n# until then this cell names the allowlist rather than guessing for you.\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Start\n", + "\n", + "Press **Start**. It re-links the model and dataset, executes the settings cell\n", + "above, and then starts the experiment \u2014 in that order, because `start()` is\n", + "one-shot and resets the plan. Your remaining team budget is shown beside the\n", + "button.\n", + "\n", + "To iterate: change a value above and press Start again.\n", + "\n", + "Prefer to leave? *Download .ipynb* and *Copy as script* both give you the same\n", + "settings as plain SDK calls." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + }, + "tracebloc": { + "template": "nlp_finetune", + "family": "NLP fine-tune (incl. LoRA)", + "categories": [ + "text_classification", + "sentence_pair_classification", + "token_classification" + ] + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/templates/nlp_generative.ipynb b/notebooks/templates/nlp_generative.ipynb new file mode 100644 index 0000000..56988eb --- /dev/null +++ b/notebooks/templates/nlp_generative.ipynb @@ -0,0 +1,205 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# NLP generative \u2014 training template\n", + "\n", + "Renders for every `(category, framework)` pair in this family:\n", + "`causal_language_modeling`, `seq2seq`, `masked_language_modeling`.\n", + "\n", + "This is a **template**, not a guide. The pod renders it for one use case with\n", + "`notebook.render(category, framework, context)`: `{{ ... }}` placeholders are\n", + "substituted from the context, and cells carrying\n", + "`metadata.tracebloc.applies_to` are dropped when they do not apply to the pair\n", + "being rendered. See `README.md` in this directory for the contract.\n", + "\n", + "**Nothing here calls `start()`.** Start is a button, and there is no Run All." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## This run\n", + "\n", + "| | |\n", + "|---|---|\n", + "| Use case | {{ use_case }} |\n", + "| Dataset | `{{ dataset_id }}` |\n", + "| Category | `{{ category }}` |\n", + "| Framework | `{{ framework }}` |\n", + "| Edges | {{ edge_count }} |\n", + "| Records per edge | {{ records_per_edge }} |" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# PENDING A RELEASED SDK. Once the image carries an SDK release with\n", + "# environment login, the pod is already authenticated -- it reads its scoped,\n", + "# short-lived credential from the environment, so there is no email/password\n", + "# prompt here and no token in the notebook.\n", + "#\n", + "# Until then `User()` PROMPTS interactively, which is wrong for a pod. Note\n", + "# that merged is not enough: environment login is on the SDK's `develop`\n", + "# (pyproject 1.0.9) but ABSENT from the latest tag v1.0.7, which is what\n", + "# `pip install tracebloc` resolves -- so this cell is contingent on a RELEASE,\n", + "# not on the change landing. Verified 2026-09-09: v1.0.7 contains no\n", + "# `env_login` module and no `TRACEBLOC_TOKEN` path at all.\n", + "from tracebloc import User\n", + "\n", + "user = User()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Filled in by the model picker. Change the path to point at your own file.\n", + "MODEL_PATH = \"{{ model_path }}\"\n", + "\n", + "user.upload_model(MODEL_PATH)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# --- Tokenizer -----------------------------------------------------------\n", + "# NLP models must ship a tokenizer; there is no fallback. The SDK picks up a\n", + "# `_tokenizer.json` sitting next to the model file, so most zoo models\n", + "# need nothing here. Pass one explicitly only when yours is named differently:\n", + "#\n", + "# user.upload_model(MODEL_PATH, tokenizer=\"{{ tokenizer_path }}\")\n", + "#\n", + "# For a HuggingFace-hosted tokenizer, set the model file's `tokenizer_id`\n", + "# instead. An empty `tokenizer_id` is refused at upload, not at training." + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "training = user.link_model_dataset(\"{{ dataset_id }}\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# ======================================================================\n", + "# Settings \u2014 the complete plan for this run, as plain SDK calls.\n", + "# Edit a value, then press Start. Nothing here calls start().\n", + "# ======================================================================\n", + "\n", + "# --- Experiment ----------------------------------------------------------\n", + "training.experiment_name(\"{{ experiment_name }}\")\n", + "\n", + "\n", + "# --- Federation ----------------------------------------------------------\n", + "# cycles = federated rounds; epochs = local epochs per round\n", + "# Generative runs are the most expensive per pass, so the pre-fill is the most conservative of the language families.\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#training-parameters\n", + "training.cycles(5)\n", + "training.epochs(1)\n", + "# Plain FedAvg is safe here because epochs is 1: there is no local drift to\n", + "# correct. Raising epochs above 1 means moving to a drift-correcting strategy\n", + "# (fedprox, fedadam, fedyogi, fedadagrad) in the same edit.\n", + "training.aggregation_strategy(\"fedavg\")\n", + "\n", + "# --- Optimization --------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#1-optimizer\n", + "training.optimizer(\"sgd\")\n", + "training.learning_rate({\"type\": \"constant\", \"value\": 0.001})\n", + "training.seed(0) # 0 means no fixed seed\n", + "\n", + "# --- Data ----------------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#dataset-parameters-optional\n", + "training.validation_split({{ validation_split }})\n", + "training.training_classes({{ training_classes }})\n", + "\n", + "# --- Sequence ------------------------------------------------------------\n", + "# Maximum token sequence length. Longer costs quadratically in attention.\n", + "# https://docs.tracebloc.io/join-use-case/how-training-works#per-use-case\n", + "training.sequence_length({{ sequence_length }})\n", + "\n", + "# --- LoRA ----------------------------------------------------------------\n", + "# Off by default. Target modules are derived for you; only the four knobs\n", + "# below are yours. LoRA changes what is averaged: adapters, not full weights.\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#llm-parameters-text-classification\n", + "# training.enable_lora(True)\n", + "# training.set_lora_parameters(256, 512, 0.05, False) # r, alpha, dropout, q_lora\n", + "\n", + "# --- Callbacks -----------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#callbacks\n", + "training.terminate_on_nan_callback()\n", + "# training.early_stop_callback(monitor=\"val_loss\", patience=3)\n", + "# training.model_checkpoint_callback(monitor=\"val_loss\", save_best_only=True)\n", + "# training.reduce_lr_callback(monitor=\"val_loss\", factor=0.1, patience=2, min_delta=1e-4)\n", + "\n", + "# --- Custom loss ---------------------------------------------------------\n", + "# Optional. Point at a `loss.py` next to your model file; it is validated at\n", + "# LINK time, so a loss that returns a non-scalar, a NaN, or a tensor detached\n", + "# from the graph is refused there rather than wasting a training run.\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#3-loss-function\n", + "#\n", + "# training.loss_function({\"type\": \"custom\", \"value\": \"loss.py\"})\n", + "#\n", + "# Or pick a standard one:\n", + "# Or a standard one. THE VALID VALUES DEPEND ON THE PAIR, and the SDK\n# refuses anything outside its allowlist by poisoning the plan, so Start\n# then blocks -- an earlier draft suggested \"crossentropy\" here and that is\n# wrong for several pairs this template renders for:\n# time_series_forecasting mse | l1\n# time_to_event_prediction coxph\n# any sklearn pair mse | binarycrossentropy (custom loss.py refused)\n# lifelines / scikit_survival a custom loss.py is IGNORED, not applied\n# everything else crossentropy | mse | l1\n# training.loss_function({\"type\": \"standard\", \"value\": \"\"})\n#\n# Per-category suggestions via gated fragments are tracked separately;\n# until then this cell names the allowlist rather than guessing for you.\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Start\n", + "\n", + "Press **Start**. It re-links the model and dataset, executes the settings cell\n", + "above, and then starts the experiment \u2014 in that order, because `start()` is\n", + "one-shot and resets the plan. Your remaining team budget is shown beside the\n", + "button.\n", + "\n", + "To iterate: change a value above and press Start again.\n", + "\n", + "Prefer to leave? *Download .ipynb* and *Copy as script* both give you the same\n", + "settings as plain SDK calls." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + }, + "tracebloc": { + "template": "nlp_generative", + "family": "NLP generative", + "categories": [ + "causal_language_modeling", + "seq2seq", + "masked_language_modeling" + ] + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/templates/survival.ipynb b/notebooks/templates/survival.ipynb new file mode 100644 index 0000000..5d74633 --- /dev/null +++ b/notebooks/templates/survival.ipynb @@ -0,0 +1,254 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Survival \u2014 training template\n", + "\n", + "Renders for every `(category, framework)` pair in this family:\n", + "`time_to_event_prediction`.\n", + "\n", + "This is a **template**, not a guide. The pod renders it for one use case with\n", + "`notebook.render(category, framework, context)`: `{{ ... }}` placeholders are\n", + "substituted from the context, and cells carrying\n", + "`metadata.tracebloc.applies_to` are dropped when they do not apply to the pair\n", + "being rendered. See `README.md` in this directory for the contract.\n", + "\n", + "**Nothing here calls `start()`.** Start is a button, and there is no Run All." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> ## \u26a0\ufe0f This template is UNVERIFIED\n", + ">\n", + "> Every other family template ships only after a live experiment on the dev\n", + "> platform has completed on that template's own defaults. **This one has\n", + "> not**, and cannot yet: the time-to-event datasets currently available for\n", + "> testing carry a target column of time-like values rather than the 0/1 event\n", + "> indicator the training runtime requires, so it refuses every experiment on\n", + "> them. A correctly-shaped dataset is being prepared.\n", + ">\n", + "> The pre-fills below are not in question \u2014 `cycles=1`, `epochs=1` matches\n", + "> what the SDK already forces for the survival frameworks. What is missing is\n", + "> the evidence. **Treat this as a draft to sanity-check, not a signed-off\n", + "> default**, and check your first run's metrics rather than trusting them.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## This run\n", + "\n", + "| | |\n", + "|---|---|\n", + "| Use case | {{ use_case }} |\n", + "| Dataset | `{{ dataset_id }}` |\n", + "| Category | `{{ category }}` |\n", + "| Framework | `{{ framework }}` |\n", + "| Edges | {{ edge_count }} |\n", + "| Records per edge | {{ records_per_edge }} |" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# PENDING A RELEASED SDK. Once the image carries an SDK release with\n", + "# environment login, the pod is already authenticated -- it reads its scoped,\n", + "# short-lived credential from the environment, so there is no email/password\n", + "# prompt here and no token in the notebook.\n", + "#\n", + "# Until then `User()` PROMPTS interactively, which is wrong for a pod. Note\n", + "# that merged is not enough: environment login is on the SDK's `develop`\n", + "# (pyproject 1.0.9) but ABSENT from the latest tag v1.0.7, which is what\n", + "# `pip install tracebloc` resolves -- so this cell is contingent on a RELEASE,\n", + "# not on the change landing. Verified 2026-09-09: v1.0.7 contains no\n", + "# `env_login` module and no `TRACEBLOC_TOKEN` path at all.\n", + "from tracebloc import User\n", + "\n", + "user = User()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Filled in by the model picker. Change the path to point at your own file.\n", + "MODEL_PATH = \"{{ model_path }}\"\n", + "\n", + "user.upload_model(MODEL_PATH)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "training = user.link_model_dataset(\"{{ dataset_id }}\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# ======================================================================\n", + "# Settings \u2014 the complete plan for this run, as plain SDK calls.\n", + "# Edit a value, then press Start. Nothing here calls start().\n", + "# ======================================================================\n", + "\n", + "# --- Experiment ----------------------------------------------------------\n", + "training.experiment_name(\"{{ experiment_name }}\")\n", + "\n", + "\n", + "# --- Federation ----------------------------------------------------------\n", + "# cycles and epochs are set below, gated on framework: the single-pass\n", + "# frameworks (sklearn, lifelines, scikit_survival) force both to 1 and print a\n", + "# red banner if you set them, so this family cannot offer one value to every\n", + "# pair.\n", + "training.aggregation_strategy(\"fedavg\")\n", + "\n", + "# --- Optimization --------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#1-optimizer\n", + "training.optimizer(\"sgd\")\n", + "training.learning_rate({\"type\": \"constant\", \"value\": 0.001})\n", + "training.seed(0) # 0 means no fixed seed\n", + "\n", + "# --- Data ----------------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#dataset-parameters-optional\n", + "training.validation_split({{ validation_split }})\n", + "training.training_classes({{ training_classes }})\n", + "\n", + "# --- Feature shape -------------------------------------------------------\n", + "# Must agree with the dataset's column count, or the link is refused.\n", + "training.feature_points({{ feature_points }})\n", + "\n", + "# --- Preprocessing -------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#preprocessing-tabular-%26-time-series\n", + "training.handle_missing_values(True)\n", + "training.imputation_strategy(\"median\")\n", + "training.encoding_strategy(\"label\")\n", + "training.normalize_features(True)\n", + "training.scaler(\"StandardScaler\")\n", + "\n", + "# --- Callbacks -----------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#callbacks\n", + "training.terminate_on_nan_callback()\n", + "# training.early_stop_callback(monitor=\"val_loss\", patience=3)\n", + "# training.model_checkpoint_callback(monitor=\"val_loss\", save_best_only=True)\n", + "# training.reduce_lr_callback(monitor=\"val_loss\", factor=0.1, patience=2, min_delta=1e-4)\n", + "\n", + "# --- Custom loss ---------------------------------------------------------\n", + "# Optional. Point at a `loss.py` next to your model file; it is validated at\n", + "# LINK time, so a loss that returns a non-scalar, a NaN, or a tensor detached\n", + "# from the graph is refused there rather than wasting a training run.\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#3-loss-function\n", + "#\n", + "# training.loss_function({\"type\": \"custom\", \"value\": \"loss.py\"})\n", + "#\n", + "# Or pick a standard one:\n", + "# Or a standard one. THE VALID VALUES DEPEND ON THE PAIR, and the SDK\n# refuses anything outside its allowlist by poisoning the plan, so Start\n# then blocks -- an earlier draft suggested \"crossentropy\" here and that is\n# wrong for several pairs this template renders for:\n# time_series_forecasting mse | l1\n# time_to_event_prediction coxph\n# any sklearn pair mse | binarycrossentropy (custom loss.py refused)\n# lifelines / scikit_survival a custom loss.py is IGNORED, not applied\n# everything else crossentropy | mse | l1\n# training.loss_function({\"type\": \"standard\", \"value\": \"\"})\n#\n# Per-category suggestions via gated fragments are tracked separately;\n# until then this cell names the allowlist rather than guessing for you.\n", + "\n", + "# On the survival frameworks (sklearn, lifelines, scikit-survival) the SDK\n", + "# forces epochs and cycles to 1 and only warns, so a value set here is silently dropped \u2014 the calls above\n", + "# agree with that rather than fighting it. A pytorch survival model takes the\n", + "# plan normally, which is why the values are still written out." + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "tracebloc": { + "applies_to": { + "framework": [ + "pytorch" + ] + }, + "settings_fragment": true + } + }, + "source": [ + "# --- Federation ----------------------------------------------------------\n", + "# cycles = federated rounds; epochs = local epochs per round\n", + "# The survival frameworks (sklearn, lifelines, scikit-survival) train in a single pass and the SDK WARNS and sets epochs to 1 for them (it does not refuse); the pre-fill matches that rather than fighting it.\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#training-parameters\n", + "training.cycles(1)\n", + "training.epochs(1)\n", + "# Plain FedAvg is safe here because epochs is 1: there is no local drift to\n", + "# correct. Raising epochs above 1 means moving to a drift-correcting strategy\n", + "# (fedprox, fedadam, fedyogi, fedadagrad) in the same edit." + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "tracebloc": { + "applies_to": { + "framework": [ + "lifelines", + "scikit_survival" + ] + }, + "settings_fragment": true + } + }, + "source": [ + "# --- Federation ----------------------------------------------------------\n", + "# This framework trains in a single pass: the SDK WARNS and sets cycles and epochs to\n", + "# 1; it does not refuse, so a value set here would be silently dropped. One\n", + "# round over the local data, then aggregation." + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Start\n", + "\n", + "Press **Start**. It re-links the model and dataset, executes the settings cell\n", + "above, and then starts the experiment \u2014 in that order, because `start()` is\n", + "one-shot and resets the plan. Your remaining team budget is shown beside the\n", + "button.\n", + "\n", + "To iterate: change a value above and press Start again.\n", + "\n", + "Prefer to leave? *Download .ipynb* and *Copy as script* both give you the same\n", + "settings as plain SDK calls." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + }, + "tracebloc": { + "template": "survival", + "family": "Survival", + "categories": [ + "time_to_event_prediction" + ] + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/templates/tabular_timeseries.ipynb b/notebooks/templates/tabular_timeseries.ipynb new file mode 100644 index 0000000..57278f5 --- /dev/null +++ b/notebooks/templates/tabular_timeseries.ipynb @@ -0,0 +1,353 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tabular / time series \u2014 training template\n", + "\n", + "Renders for every `(category, framework)` pair in this family:\n", + "`tabular_classification`, `tabular_regression`, `time_series_classification`, `time_series_forecasting`.\n", + "\n", + "This is a **template**, not a guide. The pod renders it for one use case with\n", + "`notebook.render(category, framework, context)`: `{{ ... }}` placeholders are\n", + "substituted from the context, and cells carrying\n", + "`metadata.tracebloc.applies_to` are dropped when they do not apply to the pair\n", + "being rendered. See `README.md` in this directory for the contract.\n", + "\n", + "**Nothing here calls `start()`.** Start is a button, and there is no Run All." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## This run\n", + "\n", + "| | |\n", + "|---|---|\n", + "| Use case | {{ use_case }} |\n", + "| Dataset | `{{ dataset_id }}` |\n", + "| Category | `{{ category }}` |\n", + "| Framework | `{{ framework }}` |\n", + "| Edges | {{ edge_count }} |\n", + "| Records per edge | {{ records_per_edge }} |" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# PENDING A RELEASED SDK. Once the image carries an SDK release with\n", + "# environment login, the pod is already authenticated -- it reads its scoped,\n", + "# short-lived credential from the environment, so there is no email/password\n", + "# prompt here and no token in the notebook.\n", + "#\n", + "# Until then `User()` PROMPTS interactively, which is wrong for a pod. Note\n", + "# that merged is not enough: environment login is on the SDK's `develop`\n", + "# (pyproject 1.0.9) but ABSENT from the latest tag v1.0.7, which is what\n", + "# `pip install tracebloc` resolves -- so this cell is contingent on a RELEASE,\n", + "# not on the change landing. Verified 2026-09-09: v1.0.7 contains no\n", + "# `env_login` module and no `TRACEBLOC_TOKEN` path at all.\n", + "from tracebloc import User\n", + "\n", + "user = User()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Filled in by the model picker. Change the path to point at your own file.\n", + "MODEL_PATH = \"{{ model_path }}\"\n", + "\n", + "user.upload_model(MODEL_PATH)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "training = user.link_model_dataset(\"{{ dataset_id }}\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# ======================================================================\n", + "# Settings \u2014 the complete plan for this run, as plain SDK calls.\n", + "# Edit a value, then press Start. Nothing here calls start().\n", + "# ======================================================================\n", + "\n", + "# --- Experiment ----------------------------------------------------------\n", + "training.experiment_name(\"{{ experiment_name }}\")\n", + "\n", + "\n", + "# --- Federation ----------------------------------------------------------\n", + "# cycles and epochs are set below, gated on framework: the single-pass\n", + "# frameworks (sklearn, lifelines, scikit_survival) force both to 1 and print a\n", + "# red banner if you set them, so this family cannot offer one value to every\n", + "# pair.\n", + "training.aggregation_strategy(\"fedavg\")\n", + "\n", + "# --- Optimization --------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#1-optimizer\n", + "training.optimizer(\"sgd\")\n", + "training.learning_rate({\"type\": \"constant\", \"value\": 0.001})\n", + "training.seed(0) # 0 means no fixed seed\n", + "\n", + "# --- Data ----------------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#dataset-parameters-optional\n", + "training.validation_split({{ validation_split }})\n", + "training.training_classes({{ training_classes }})\n", + "\n", + "# --- Preprocessing -------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#preprocessing-tabular-%26-time-series\n", + "training.handle_missing_values(True)\n", + "training.imputation_strategy(\"median\")\n", + "# MinMaxScaler for time series, StandardScaler for tabular.\n", + "training.scaler(\"{{ scaler }}\")\n", + "\n", + "# --- Callbacks -----------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#callbacks\n", + "training.terminate_on_nan_callback()\n", + "# training.early_stop_callback(monitor=\"val_loss\", patience=3)\n", + "# training.model_checkpoint_callback(monitor=\"val_loss\", save_best_only=True)\n", + "# training.reduce_lr_callback(monitor=\"val_loss\", factor=0.1, patience=2, min_delta=1e-4)\n", + "\n", + "# --- Custom loss ---------------------------------------------------------\n", + "# Optional. Point at a `loss.py` next to your model file; it is validated at\n", + "# LINK time, so a loss that returns a non-scalar, a NaN, or a tensor detached\n", + "# from the graph is refused there rather than wasting a training run.\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#3-loss-function\n", + "#\n", + "# training.loss_function({\"type\": \"custom\", \"value\": \"loss.py\"})\n", + "#\n", + "# Or pick a standard one:\n", + "# Or a standard one. THE VALID VALUES DEPEND ON THE PAIR, and the SDK\n# refuses anything outside its allowlist by poisoning the plan, so Start\n# then blocks -- an earlier draft suggested \"crossentropy\" here and that is\n# wrong for several pairs this template renders for:\n# time_series_forecasting mse | l1\n# time_to_event_prediction coxph\n# any sklearn pair mse | binarycrossentropy (custom loss.py refused)\n# lifelines / scikit_survival a custom loss.py is IGNORED, not applied\n# everything else crossentropy | mse | l1\n# training.loss_function({\"type\": \"standard\", \"value\": \"\"})\n#\n# Per-category suggestions via gated fragments are tracked separately;\n# until then this cell names the allowlist rather than guessing for you.\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "tracebloc": { + "applies_to": { + "dataset_flag": "allow_feature_modification" + }, + "settings_fragment": true + } + }, + "source": [ + "# --- Feature interaction -------------------------------------------------\n", + "# Only rendered when the dataset sets `allow_feature_modification`. Derive a\n", + "# new column, or restrict training to a subset of columns:\n", + "#\n", + "# training.feature_interaction({\"feature1\": \"age\", \"feature2\": \"bmi\", \"method\": \"product\"})\n", + "# training.feature_interaction({\"feature_list\": [\"age\", \"bmi\"], \"method\": \"include\"})\n", + "#\n", + "# training.get_features() # list the columns this dataset exposes" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "tracebloc": { + "applies_to": { + "framework": [ + "pytorch" + ] + }, + "settings_fragment": true + } + }, + "source": [ + "# --- Federation ----------------------------------------------------------\n", + "# cycles = federated rounds; epochs = local epochs per round\n", + "# Cheap passes over small feature tables; fifteen rounds still sits inside the twenty-pass ceiling.\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#training-parameters\n", + "training.cycles(15)\n", + "training.epochs(1)\n", + "# Plain FedAvg is safe here because epochs is 1: there is no local drift to\n", + "# correct. Raising epochs above 1 means moving to a drift-correcting strategy\n", + "# (fedprox, fedadam, fedyogi, fedadagrad) in the same edit." + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "tracebloc": { + "applies_to": { + "framework": [ + "sklearn" + ] + }, + "settings_fragment": true + } + }, + "source": [ + "# --- Federation ----------------------------------------------------------\n", + "# This framework trains in a single pass: the SDK WARNS and sets cycles and epochs to\n", + "# 1; it does not refuse, so a value set here would be silently dropped. One\n", + "# round over the local data, then aggregation." + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "tracebloc": { + "applies_to": { + "category": [ + "tabular_classification" + ] + }, + "settings_fragment": true + } + }, + "source": [ + "# --- Feature shape -------------------------------------------------------\n", + "# Must agree with the dataset's column count, or the link is refused.\n", + "training.feature_points({{ feature_points }})" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "tracebloc": { + "applies_to": { + "category": [ + "tabular_classification", + "tabular_regression", + "time_series_classification" + ] + }, + "settings_fragment": true + } + }, + "source": [ + "# Column encoding and feature normalization. Not part of the forecasting\n", + "# path, which reads only the imputation pair above.\n", + "training.encoding_strategy(\"label\")\n", + "training.normalize_features(True)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "tracebloc": { + "applies_to": { + "category": [ + "time_series_classification", + "time_series_forecasting" + ] + }, + "settings_fragment": true + } + }, + "source": [ + "# --- Time series ---------------------------------------------------------\n", + "# Lookback window: how many past steps are fed in as input.\n", + "training.sequence_length({{ sequence_length }})" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "tracebloc": { + "applies_to": { + "category": [ + "time_series_forecasting" + ] + }, + "settings_fragment": true + } + }, + "source": [ + "# How many future steps to predict. Forecasting only.\n", + "training.forecast_horizon({{ forecast_horizon }})" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "tracebloc": { + "applies_to": { + "category": [ + "time_series_classification" + ] + }, + "settings_fragment": true + } + }, + "source": [ + "# Emit a channel flagging where values were missing. Only the\n", + "# time-series-classification engine path produces these.\n", + "training.missingness_indicators(False)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Start\n", + "\n", + "Press **Start**. It re-links the model and dataset, executes the settings cell\n", + "above, and then starts the experiment \u2014 in that order, because `start()` is\n", + "one-shot and resets the plan. Your remaining team budget is shown beside the\n", + "button.\n", + "\n", + "To iterate: change a value above and press Start again.\n", + "\n", + "Prefer to leave? *Download .ipynb* and *Copy as script* both give you the same\n", + "settings as plain SDK calls." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + }, + "tracebloc": { + "template": "tabular_timeseries", + "family": "Tabular / time series", + "categories": [ + "tabular_classification", + "tabular_regression", + "time_series_classification", + "time_series_forecasting" + ] + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/templates/verification-dev.json b/notebooks/templates/verification-dev.json new file mode 100644 index 0000000..9ab18aa --- /dev/null +++ b/notebooks/templates/verification-dev.json @@ -0,0 +1,282 @@ +{ + "_comment": [ + "Live dev verification records for the family templates. A template", + "ships only once a live", + "experiment on dev has reached COMPLETED using the template's OWN", + "defaults, so these records are what licenses each pre-fill.", + "", + "Committed rather than left on dev on purpose: the dev cluster gets", + "swept, and a verification that depends on state we do not control is", + "a claim, not a proof. scripts/check_templates.py rule 13 binds these", + "to families.json, so changing a pre-fill fails the check until the", + "run is redone and this file refreshed.", + "", + "Curated, not a raw dump: this is a public repo, so it carries the", + "settings under verification and the terminal status only -- no", + "infrastructure detail, FLOPs or carbon figures.", + "", + "round 1 applied cycles/epochs/training_classes only; every other", + "field coincided with the template EXCEPT callbacks, which came back", + "'[]' because terminate_on_nan_callback() was never called. round 2", + "applied every uncommented line of each settings cell. Both are kept:", + "round 1 is real evidence for the pre-fills, round 2 for the whole", + "cell." + ], + "captured": "2026-09-09", + "environment": "dev", + "runs": { + "vision": { + "family": "vision_from_scratch", + "dataset": "d0gfu0c1", + "model": "resnet_18", + "rounds": { + "2_full_settings_cell": { + "experiment": "eaxx647p", + "status": "COMPLETED", + "minutes": 40.3, + "cycles": 20, + "epochs": 1, + "aggregation_strategy": "fedavg", + "optimizer": "sgd", + "seed": 0, + "learningRate": { + "type": "constant", + "value": 0.001 + }, + "validation_split": 0.07, + "callbacks": "[{'terminateOnNaN': ['']}]", + "error_reason": null, + "data_type": "rgb", + "shuffle": true + }, + "1_prefills": { + "experiment": "elcm69bi", + "status": "COMPLETED", + "minutes": 48.2, + "cycles": 20, + "epochs": 1, + "aggregation_strategy": "fedavg", + "optimizer": "sgd", + "seed": 0, + "learningRate": { + "type": "constant", + "value": 0.001 + }, + "validation_split": 0.07, + "callbacks": "[]", + "error_reason": null, + "data_type": "rgb", + "shuffle": true + } + } + }, + "nlp_finetune": { + "family": "nlp_finetune", + "dataset": "d66zp15z", + "model": "simple_text", + "rounds": { + "2_full_settings_cell": { + "experiment": "enpm5bls", + "status": "COMPLETED", + "minutes": 16.9, + "cycles": 8, + "epochs": 1, + "aggregation_strategy": "fedavg", + "optimizer": "sgd", + "seed": 0, + "learningRate": { + "type": "constant", + "value": 0.001 + }, + "validation_split": 0.07, + "callbacks": "[{'terminateOnNaN': ['']}]", + "error_reason": null, + "data_shape": 128 + }, + "1_prefills": { + "experiment": "ej5olgjp", + "status": "COMPLETED", + "minutes": 19.6, + "cycles": 8, + "epochs": 1, + "aggregation_strategy": "fedavg", + "optimizer": "sgd", + "seed": 0, + "learningRate": { + "type": "constant", + "value": 0.001 + }, + "validation_split": 0.07, + "callbacks": "[]", + "error_reason": null, + "data_shape": 64 + } + } + }, + "nlp_generative": { + "family": "nlp_generative", + "dataset": "dy19kuwc", + "model": "simple_causal_lm", + "rounds": { + "2_full_settings_cell": { + "experiment": "e6lymzry", + "status": "COMPLETED", + "minutes": 11.4, + "cycles": 5, + "epochs": 1, + "aggregation_strategy": "fedavg", + "optimizer": "sgd", + "seed": 0, + "learningRate": { + "type": "constant", + "value": 0.001 + }, + "validation_split": 0.05, + "callbacks": "[{'terminateOnNaN': ['']}]", + "error_reason": null, + "data_shape": 128 + }, + "1_prefills": { + "experiment": "ehro8f4y", + "status": "COMPLETED", + "minutes": 15.3, + "cycles": 5, + "epochs": 1, + "aggregation_strategy": "fedavg", + "optimizer": "sgd", + "seed": 0, + "learningRate": { + "type": "constant", + "value": 0.001 + }, + "validation_split": 0.05, + "callbacks": "[]", + "error_reason": null, + "data_shape": 128 + } + } + }, + "embeddings": { + "family": "embeddings", + "dataset": "diyeknwn", + "model": "simple_text_encoder", + "rounds": { + "2_full_settings_cell": { + "experiment": "erl4u2pk", + "status": "COMPLETED", + "minutes": 18.0, + "cycles": 8, + "epochs": 1, + "aggregation_strategy": "fedavg", + "optimizer": "sgd", + "seed": 0, + "learningRate": { + "type": "constant", + "value": 0.001 + }, + "validation_split": 0.05, + "callbacks": "[{'terminateOnNaN': ['']}]", + "error_reason": null, + "data_shape": 128 + }, + "1_prefills": { + "experiment": "e31yuuuq", + "status": "COMPLETED", + "minutes": 20.4, + "cycles": 8, + "epochs": 1, + "aggregation_strategy": "fedavg", + "optimizer": "sgd", + "seed": 0, + "learningRate": { + "type": "constant", + "value": 0.001 + }, + "validation_split": 0.05, + "callbacks": "[]", + "error_reason": null, + "data_shape": 128 + } + } + }, + "tabular_ts": { + "family": "tabular_timeseries", + "dataset": "d7uuiv8h", + "model": "fcn", + "rounds": { + "2_full_settings_cell": { + "experiment": "eeofzvuj", + "status": "COMPLETED", + "minutes": 28.9, + "cycles": 15, + "epochs": 1, + "aggregation_strategy": "fedavg", + "optimizer": "sgd", + "seed": 0, + "learningRate": { + "type": "constant", + "value": 0.001 + }, + "validation_split": 0.05, + "callbacks": "[{'terminateOnNaN': ['']}]", + "error_reason": null, + "data_shape": 3, + "tabular_scaler": "StandardScaler", + "encoding_strategy": "label", + "normalize_features": true, + "handle_missing_values": true, + "imputation_strategy": "median" + }, + "1_prefills": { + "experiment": "ev42i50f", + "status": "COMPLETED", + "minutes": 34.0, + "cycles": 15, + "epochs": 1, + "aggregation_strategy": "fedavg", + "optimizer": "sgd", + "seed": 0, + "learningRate": { + "type": "constant", + "value": 0.001 + }, + "validation_split": 0.05, + "callbacks": "[]", + "error_reason": null, + "data_shape": 3, + "tabular_scaler": "StandardScaler", + "encoding_strategy": "label", + "normalize_features": true, + "handle_missing_values": true, + "imputation_strategy": "median" + } + } + }, + "survival": { + "family": "survival", + "dataset": "dz5irpa2", + "model": "deepsurv", + "rounds": { + "1_prefills": { + "experiment": "enpo3jbx", + "status": "PAUSED", + "minutes": null, + "cycles": 1, + "epochs": 1, + "aggregation_strategy": "fedavg", + "optimizer": "sgd", + "seed": 0, + "learningRate": { + "type": "constant", + "value": 0.001 + }, + "validation_split": 0.05, + "callbacks": "[]", + "error_reason": "ValueError: TTE data validation failed: 'label' column must contain only 0/1 binary event indicators; found 2 distinct out-of-range value(s) at rows [0, 1, 2, 3, 4] (+35 more)", + "data_shape": 12, + "tabular_scaler": "StandardScaler" + } + } + } + } +} diff --git a/notebooks/templates/vision_from_scratch.ipynb b/notebooks/templates/vision_from_scratch.ipynb new file mode 100644 index 0000000..8c8cb34 --- /dev/null +++ b/notebooks/templates/vision_from_scratch.ipynb @@ -0,0 +1,200 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Vision, from scratch \u2014 training template\n", + "\n", + "Renders for every `(category, framework)` pair in this family:\n", + "`image_classification`, `object_detection`, `semantic_segmentation`, `keypoint_detection`.\n", + "\n", + "This is a **template**, not a guide. The pod renders it for one use case with\n", + "`notebook.render(category, framework, context)`: `{{ ... }}` placeholders are\n", + "substituted from the context, and cells carrying\n", + "`metadata.tracebloc.applies_to` are dropped when they do not apply to the pair\n", + "being rendered. See `README.md` in this directory for the contract.\n", + "\n", + "**Nothing here calls `start()`.** Start is a button, and there is no Run All." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## This run\n", + "\n", + "| | |\n", + "|---|---|\n", + "| Use case | {{ use_case }} |\n", + "| Dataset | `{{ dataset_id }}` |\n", + "| Category | `{{ category }}` |\n", + "| Framework | `{{ framework }}` |\n", + "| Edges | {{ edge_count }} |\n", + "| Records per edge | {{ records_per_edge }} |" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# PENDING A RELEASED SDK. Once the image carries an SDK release with\n", + "# environment login, the pod is already authenticated -- it reads its scoped,\n", + "# short-lived credential from the environment, so there is no email/password\n", + "# prompt here and no token in the notebook.\n", + "#\n", + "# Until then `User()` PROMPTS interactively, which is wrong for a pod. Note\n", + "# that merged is not enough: environment login is on the SDK's `develop`\n", + "# (pyproject 1.0.9) but ABSENT from the latest tag v1.0.7, which is what\n", + "# `pip install tracebloc` resolves -- so this cell is contingent on a RELEASE,\n", + "# not on the change landing. Verified 2026-09-09: v1.0.7 contains no\n", + "# `env_login` module and no `TRACEBLOC_TOKEN` path at all.\n", + "from tracebloc import User\n", + "\n", + "user = User()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Filled in by the model picker. Change the path to point at your own file.\n", + "MODEL_PATH = \"{{ model_path }}\"\n", + "\n", + "user.upload_model(MODEL_PATH)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "training = user.link_model_dataset(\"{{ dataset_id }}\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# ======================================================================\n", + "# Settings \u2014 the complete plan for this run, as plain SDK calls.\n", + "# Edit a value, then press Start. Nothing here calls start().\n", + "# ======================================================================\n", + "\n", + "# --- Experiment ----------------------------------------------------------\n", + "training.experiment_name(\"{{ experiment_name }}\")\n", + "\n", + "\n", + "# --- Federation ----------------------------------------------------------\n", + "# cycles = federated rounds; epochs = local epochs per round\n", + "# Twenty effective passes bought with rounds, not local epochs. From-scratch vision is where client drift bites hardest, so this row is the clearest case for spending the budget on cycles.\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#training-parameters\n", + "training.cycles(20)\n", + "training.epochs(1)\n", + "# Plain FedAvg is safe here because epochs is 1: there is no local drift to\n", + "# correct. Raising epochs above 1 means moving to a drift-correcting strategy\n", + "# (fedprox, fedadam, fedyogi, fedadagrad) in the same edit.\n", + "training.aggregation_strategy(\"fedavg\")\n", + "\n", + "# --- Optimization --------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#1-optimizer\n", + "training.optimizer(\"sgd\")\n", + "training.learning_rate({\"type\": \"constant\", \"value\": 0.001})\n", + "training.seed(0) # 0 means no fixed seed\n", + "\n", + "# --- Data ----------------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#dataset-parameters-optional\n", + "training.validation_split({{ validation_split }})\n", + "training.training_classes({{ training_classes }})\n", + "\n", + "# --- Image handling ------------------------------------------------------\n", + "# Grayscale datasets need this: a 1-channel dataset fails the channel check at\n", + "# the rgb default.\n", + "training.data_type(\"{{ data_type }}\")\n", + "\n", + "# --- Shuffle -------------------------------------------------------------\n", + "training.shuffle(True)\n", + "\n", + "# --- Callbacks -----------------------------------------------------------\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#callbacks\n", + "training.terminate_on_nan_callback()\n", + "# training.early_stop_callback(monitor=\"val_loss\", patience=3)\n", + "# training.model_checkpoint_callback(monitor=\"val_loss\", save_best_only=True)\n", + "# training.reduce_lr_callback(monitor=\"val_loss\", factor=0.1, patience=2, min_delta=1e-4)\n", + "\n", + "# --- Custom loss ---------------------------------------------------------\n", + "# Optional. Point at a `loss.py` next to your model file; it is validated at\n", + "# LINK time, so a loss that returns a non-scalar, a NaN, or a tensor detached\n", + "# from the graph is refused there rather than wasting a training run.\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#3-loss-function\n", + "# YOLO model types REQUIRE a custom loss and cannot start without one.\n", + "# Keypoint RCNN model types are the one exception here: they ignore a\n", + "# supplied loss.\n", + "#\n", + "# training.loss_function({\"type\": \"custom\", \"value\": \"loss.py\"})\n", + "#\n", + "# Or pick a standard one:\n", + "# Or a standard one. THE VALID VALUES DEPEND ON THE PAIR, and the SDK\n# refuses anything outside its allowlist by poisoning the plan, so Start\n# then blocks -- an earlier draft suggested \"crossentropy\" here and that is\n# wrong for several pairs this template renders for:\n# time_series_forecasting mse | l1\n# time_to_event_prediction coxph\n# any sklearn pair mse | binarycrossentropy (custom loss.py refused)\n# lifelines / scikit_survival a custom loss.py is IGNORED, not applied\n# everything else crossentropy | mse | l1\n# training.loss_function({\"type\": \"standard\", \"value\": \"\"})\n#\n# Per-category suggestions via gated fragments are tracked separately;\n# until then this cell names the allowlist rather than guessing for you.\n", + "\n", + "# --- Augmentation --------------------------------------------------------\n", + "# --- Augmentation --------------------------------------------------------\n# pytorch only, and all off by default. Each line below passes a value the\n# SDK ACCEPTS, so uncommenting one is safe; three earlier suggestions were\n# not (`brightness_range(None)`, `rescale(None)`, `fill_mode(\"nearest\")` are\n# each refused and poison the plan). Note cval() resets fill_mode to\n# \"constant\" on pytorch, so order matters.\n", + "# actually needs. (`shuffle`, the eleventh, is above: it is not framework-gated.)\n", + "# https://docs.tracebloc.io/join-use-case/hyperparameters#data-augmentation-for-image-data\n", + "# training.rotation_range(0)\n", + "# training.width_shift_range(0.0)\n", + "# training.height_shift_range(0.0)\n", + "# training.shear_range(0.0)\n", + "# training.zoom_range(0.0)\n# training.brightness_range(0.0)\n# training.rescale(1.0)\n# training.fill_mode(\"constant\")\n", + "# training.channel_shift_range(0.0)\n", + "# training.cval(0.0)\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Start\n", + "\n", + "Press **Start**. It re-links the model and dataset, executes the settings cell\n", + "above, and then starts the experiment \u2014 in that order, because `start()` is\n", + "one-shot and resets the plan. Your remaining team budget is shown beside the\n", + "button.\n", + "\n", + "To iterate: change a value above and press Start again.\n", + "\n", + "Prefer to leave? *Download .ipynb* and *Copy as script* both give you the same\n", + "settings as plain SDK calls." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + }, + "tracebloc": { + "template": "vision_from_scratch", + "family": "Vision, from scratch", + "categories": [ + "image_classification", + "object_detection", + "semantic_segmentation", + "keypoint_detection" + ] + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/traceblocTrainingGuide.ipynb b/notebooks/traceblocTrainingGuide.ipynb new file mode 100644 index 0000000..aff484e --- /dev/null +++ b/notebooks/traceblocTrainingGuide.ipynb @@ -0,0 +1,417 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Guide to Train Machine Learning Models on tracebloc \ud83d\ude80\n", + "\n", + "This notebook walks you through training an ML model on the tracebloc platform \u2014 from connecting your account to launching a training run.\n", + "\n", + "**What you'll do:**\n", + "1. Connect to your tracebloc account\n", + "2. Upload a model & weights\n", + "3. Link the model with a dataset\n", + "4. Configure a training plan\n", + "5. Start training\n", + "\n", + "This guide takes about **10\u201315 minutes** to complete." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Prerequisites\n\nBefore you begin, make sure you have:\n\n- \u2705 A **tracebloc account** \u2014 [Sign up here](https://ai.tracebloc.io/signup) if you don't have one\n- \u2705 **Joined a use case** \u2014 you need an active use case with a dataset. [How to join a use case \u2192](https://docs.tracebloc.io/join-use-case/explore-use-case)\n- \u2705 A **model file** (`.py`) compatible with the dataset \u2014 you can use one from the [tracebloc model zoo](https://github.com/tracebloc/model-zoo) or bring your own. [Model structure requirements \u2192](https://docs.tracebloc.io/join-use-case/model-optimization)\n\n\ud83d\udcd6 **Full documentation:** [docs.tracebloc.io](https://docs.tracebloc.io)\n\n\ud83d\udca1 **Prefer Google Colab?** [Open this guide in Colab](https://colab.research.google.com/github/tracebloc/quickstart/blob/main/notebooks/traceblocTrainingGuide.ipynb) \u2014 runs entirely in your browser, no local setup needed. Once it opens, do **File \u2192 Save a copy in Drive** so your edits persist.\n\n\ud83d\udc0d **Python** \u2014 the SDK supports the Python versions declared in its package metadata. If the install cell below fails, it prints the supported range next to the version you are running. A current default `python3` on macOS is often newer than that range, so if you run locally, check this first." + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 1. Connect to tracebloc\n", + "\n", + "First, install the tracebloc package and log in with your tracebloc account email and password." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Install the tracebloc SDK. If the install fails, explain why in plain terms.\n", + "import re\n", + "import subprocess\n", + "import sys\n", + "\n", + "# Other extras: [sklearn] [catboost] [lightgbm] [xgboost] [lifelines]\n", + "# [scikit-survival] [all]. There is no [tensorflow] extra.\n", + "SPEC = \"tracebloc[pytorch]>=0.14.0\"\n", + "PACKAGE = \"tracebloc\"\n", + "\n", + "running = f\"{sys.version_info[0]}.{sys.version_info[1]}\"\n", + "print(f\"Python {sys.version.split()[0]}\")\n", + "# pip output is captured, so nothing prints while the install runs.\n", + "print(f\"Installing {SPEC} - ~90 packages including torch, this takes a few minutes...\")\n", + "\n", + "proc = subprocess.run(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", SPEC],\n", + " capture_output=True,\n", + " text=True,\n", + " check=False,\n", + ")\n", + "output = proc.stdout + proc.stderr\n", + "\n", + "if proc.returncode == 0:\n", + " # Confirm the package actually imports before moving on.\n", + " try:\n", + " from importlib.metadata import version\n", + "\n", + " import tracebloc # noqa: F401\n", + "\n", + " print(f\"OK - tracebloc {version('tracebloc')} installed and imports.\")\n", + " except Exception as exc: # a half-installed dependency can raise almost anything\n", + " print(f\"\\npip reported success but importing tracebloc did not work: {exc!r}\")\n", + " print(\"Restart the runtime (Runtime -> Restart session) and re-run this cell.\")\n", + " raise RuntimeError(\"tracebloc installed but does not import - see above\") from exc\n", + "else:\n", + " print(output[-1500:])\n", + "\n", + " # Which requirement failed? (pip also prints Requires-Python for dependencies.)\n", + " failed = re.search(r\"No matching distribution found for (\\S+)\", output)\n", + " subject = failed.group(1) if failed else \"\"\n", + " is_sdk = re.match(rf\"{PACKAGE}\\b\", subject, re.IGNORECASE) is not None\n", + "\n", + " # Which Python range did pip read? Entries are \"; \"-separated and may contain commas.\n", + " bounds = list(dict.fromkeys(b.strip() for b in re.findall(r\"Requires-Python\\s+([^\\n;]+)\", output)))\n", + "\n", + " # Did pip find any version that installs on this interpreter?\n", + " offered = re.search(r\"\\(from versions: ([^)]*)\\)\", output)\n", + " usable = [v.strip() for v in offered.group(1).split(\",\")] if offered else []\n", + " usable = [v for v in usable if v and v.lower() != \"none\"]\n", + "\n", + " print()\n", + " if not bounds:\n", + " # Older pip versions do not print the supported-Python range at all.\n", + " print(\n", + " f\"pip printed no supported-Python range, so this cell cannot tell you \"\n", + " f\"whether Python {running} is the cause. In order:\\n\"\n", + " f\" - An older pip does not print that range. Run\\n\"\n", + " f\" `{sys.executable} -m pip install --upgrade pip` and re-run this \"\n", + " \"cell; a newer pip will say so.\\n\"\n", + " \" - Otherwise suspect the index rather than the package: unreachable \"\n", + " \"index, a custom\\n --index-url, or private-index authentication.\"\n", + " )\n", + " elif not subject:\n", + " print(\n", + " \"pip printed a Requires-Python bound \"\n", + " f\"({', '.join(bounds)}) but no 'No matching distribution found' line, \"\n", + " \"so this cell\\ncannot tell which requirement it belongs to. The pip \"\n", + " \"output above is the authority.\"\n", + " )\n", + " elif not is_sdk:\n", + " print(\n", + " f\"pip failed on `{subject}`, which is a dependency rather than \"\n", + " f\"{PACKAGE} itself.\\n\"\n", + " f\"The Requires-Python bound(s) shown above ({', '.join(bounds)}) belong \"\n", + " \"to that package, so\\nthis is not necessarily about your interpreter. \"\n", + " \"The pip output above is the authority.\"\n", + " )\n", + " else:\n", + " print(\n", + " f\"You are on Python {running}, and the {PACKAGE} releases pip wanted \"\n", + " f\"declare {', '.join(bounds)},\\nso every one of them was skipped. The \"\n", + " \"package is not missing; your interpreter is\\noutside the range it \"\n", + " \"supports.\"\n", + " )\n", + " if usable:\n", + " print(\n", + " f\"\\npip does list older releases that install here \"\n", + " f\"({', '.join(usable[-3:])}). Do not reach for those:\\n\"\n", + " \"they predate the API this notebook uses. Use a Python inside the \"\n", + " \"range instead.\"\n", + " )\n", + " else:\n", + " print(\"\\nNo release on the index installs on this Python, so no pin helps.\")\n", + " print(\n", + " \" - On Colab the runtime's Python is not selectable. Please report the \"\n", + " \"version above at\\n\"\n", + " \" https://github.com/tracebloc/quickstart/issues\\n\"\n", + " \" - Locally, make a virtualenv on an interpreter inside that range and \"\n", + " \"register it as\\n this notebook's kernel.\"\n", + " )\n", + " # Stop here so \"Run All\" halts on the cell that explains the failure.\n", + " raise RuntimeError(f\"could not install {SPEC} - see the explanation above\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tracebloc import User\n", + "\n", + "# This will prompt you for your tracebloc email and password\n", + "user = User()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Expected output:** You'll see a prompt asking for your email and password. After entering them, you should see a confirmation that you're logged in.\n", + "\n", + "\u26a0\ufe0f **If login fails:**\n", + "- Double-check your email and password at [ai.tracebloc.io](https://ai.tracebloc.io)\n", + "- Make sure you've verified your email address\n", + "- If you don't have an account yet, [sign up here](https://ai.tracebloc.io/signup)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 2. Upload model & weights file\n", + "\n", + "Next, upload your model file to the platform. You have two options:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Option A: Use a model from the tracebloc model zoo\n", + "\n", + "The [tracebloc model zoo](https://github.com/tracebloc/model-zoo) has ready-to-use models for common tasks:\n", + "\n", + "| Task | Framework | Path |\n", + "|------|-----------|------|\n", + "| Image classification | PyTorch / TensorFlow | `model_zoo/image_classification/` |\n", + "| Object detection | PyTorch | `model_zoo/object_detection/pytorch/` |\n", + "| Text classification | PyTorch | `model_zoo/text_classification/pytorch/` |\n", + "| Tabular classification | PyTorch / Sklearn | `model_zoo/tabular_classification/` |\n", + "| Tabular regression | PyTorch / Sklearn | `model_zoo/tabular_regression/` |\n", + "| Time series forecasting | PyTorch | `model_zoo/time_series_forecasting/pytorch/` |\n", + "| Semantic segmentation | PyTorch | `model_zoo/semantic_segmentation/pytorch/` |\n", + "| Keypoint detection | PyTorch | `model_zoo/keypoint_detection/pytorch/` |\n", + "| Time-to-event prediction | PyTorch / Lifelines / Scikit-survival | `model_zoo/time_to_event_prediction/` |\n", + "\n", + "Clone the model zoo and pick a model that fits your use case:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Clone the tracebloc model zoo (skipped if it's already present)\n", + "![ -d ../model-zoo ] && echo \"model-zoo already present - skipping clone\" || git clone https://github.com/tracebloc/model-zoo.git ../model-zoo" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# List available models\n", + "!ls ../model-zoo/model_zoo/" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Option B: Use your own model\n", + "\n", + "You can upload your own model file. Place your `.py` file in the working directory or provide the full path.\n", + "\n", + "Make sure your model follows the [model structure requirements](https://docs.tracebloc.io/join-use-case/model-optimization)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Upload your model file to tracebloc\n", + "# Replace the path with your actual model file location\n", + "MODEL_PATH = \"../model-zoo/model_zoo/image_classification/pytorch/densenet.py\" # <-- change this\n", + "\n", + "user.upload_model(MODEL_PATH)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Expected output:** A confirmation message showing the model was uploaded successfully.\n", + "\n", + "\ud83d\udca1 **Loading weights?** Follow this naming convention:\n", + "- Model file: `mymodel.py`\n", + "- Weights file: `mymodel_weights.pkl`\n", + "\n", + "The weights file must be in the same directory as the model file.\n", + "\n", + "```python\n", + "# To upload with pretrained weights:\n", + "user.upload_model(MODEL_PATH, weights=True)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n## 3. Link uploaded model with dataset\n\nNow connect your uploaded model to a dataset from your use case.\n\n**Where to find the Dataset ID:**\n1. Go to [ai.tracebloc.io](https://ai.tracebloc.io) and open your use case\n2. Copy the ID shown next to **Dataset** in the use case panel\n3. Paste it below\n\nThe dataset ID is a short alphanumeric string (e.g., `DKbtefZy`)." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Paste your Dataset ID here\nDATASET_ID = \"YOUR_DATASET_ID_HERE\" # <-- replace with your actual dataset ID (e.g., \"DKbtefZy\")\n\ntraining = user.link_model_dataset(DATASET_ID)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "**Expected output:** A confirmation that the model and dataset are linked.\n\n\u26a0\ufe0f **If this fails:**\n- Make sure the dataset ID is correct (check your use case panel)\n- Your model must be compatible with the dataset (e.g., an image classification model for an image dataset)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 4. Set training plan\n", + "\n", + "Configure your training parameters. Start by naming your experiment, then adjust any parameters you need.\n", + "\n", + "| Command | Description | Example |\n", + "|---------|-------------|--------|\n", + "| `training.experiment_name(\"...\")` | Name your experiment | `training.experiment_name(\"My first run\")` |\n", + "| `training.epochs(n)` | Number of training epochs | `training.epochs(10)` |\n", + "| `training.optimizer(\"...\")` | Set optimizer | `training.optimizer(\"adam\")` |\n", + "| `training.learning_rate({...})` | Set learning rate | `training.learning_rate({\"type\": \"constant\", \"value\": 0.001})` |\n", + "| `training.validation_split(n)` | Validation split | `training.validation_split(0.2)` |\n", + "| `training.get_training_plan()` | View the full training plan | |\n", + "| `training.reset_training_plan()` | Reset to defaults | |\n", + "\n", + "For all available parameters, see the [Hyperparameters reference](https://docs.tracebloc.io/join-use-case/hyperparameters)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Set experiment name\n", + "training.experiment_name(\"My Experiment\")\n", + "\n", + "# Set training parameters\n", + "training.epochs(10)\n", + "\n", + "# Review your training plan\n", + "training.get_training_plan()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Expected output:** A summary showing all training parameters including:\n", + "- **Training Description** \u2014 experiment name, model name, objective\n", + "- **Dataset Parameters** \u2014 dataset ID, size, classes\n", + "- **Training Parameters** \u2014 epochs, cycles, batch size, validation split\n", + "- **Hyperparameters** \u2014 optimizer, loss function, learning rate, callbacks\n", + "- **Augmentation Parameters** \u2014 data augmentation settings\n", + "\n", + "Review these carefully before starting. Adjust any values using the commands in the table above." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 5. Start training\n", + "\n", + "Everything configured? Launch the training run:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "training.start() # start the experiment as configured above" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## What happens next?\n", + "\n", + "Your model is now being trained on the tracebloc infrastructure. Here's what to expect:\n", + "\n", + "1. **Training starts** \u2014 the model will begin training on the linked dataset inside a secure environment\n", + "2. **Monitor progress** \u2014 go to your use case on [ai.tracebloc.io](https://ai.tracebloc.io) to see training status and logs\n", + "3. **View results** \u2014 once training completes, check the leaderboard in your use case to see how your model performed\n", + "4. **Compare models** \u2014 if other team members or vendors have submitted models, you can compare performance metrics side by side\n", + "\n", + "Training time depends on your dataset size, model complexity, and the number of epochs. A typical training run takes a few minutes to a few hours.\n", + "\n", + "\ud83d\udcd6 **Learn more:** [How to evaluate models \u2192](https://docs.tracebloc.io/join-use-case/model-evaluation)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Logout\n", + "\n", + "When you're done, log out to end your session:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "user.logout()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Need help?\n", + "\n", + "- \ud83d\udcd6 [Documentation](https://docs.tracebloc.io)\n", + "- \ud83d\udce7 [support@tracebloc.io](mailto:support@tracebloc.io)\n", + "- \ud83d\udc1b [Open an issue](https://github.com/tracebloc/quickstart/issues)\n", + "- \ud83d\udcac [Discord](https://discord.gg/tracebloc)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/scripts/check_templates.py b/scripts/check_templates.py new file mode 100755 index 0000000..bc0a475 --- /dev/null +++ b/scripts/check_templates.py @@ -0,0 +1,1363 @@ +#!/usr/bin/env python3 +"""Enforce the D9 template rules on the family templates. + +Run from the repo root: + + python3 scripts/check_templates.py + +Exits non-zero and prints every violation. The rules this encodes are the ones +prose cannot hold: they are about numbers inside notebook cells, and a reviewer +reading a diff of `.ipynb` JSON will not catch a `cycles` that stopped matching +the table. + +Checked: + +1. `cycles * epochs <= 20` for every family pre-fill. +2. No family pre-fills `epochs > 1` unless its aggregation strategy carries a + drift correction. +3. Each notebook's settings cell really calls `cycles(...)` / `epochs(...)` / + `aggregation_strategy(...)` with the values `families.json` declares, so the + table and the notebooks cannot drift apart. +4. Every category appears in exactly one family, and every family names a + notebook that exists and is valid JSON. +5. No template calls `start()` β€” Start is a button, and a template that starts + a run has stopped being a template. +6. No template mentions one of the five dead setters, which are no-ops on every + surviving framework. +7. No template carries a Colab-specific line; these render in the pod. +8. Every `metadata.tracebloc` cell uses keys the render contract defines, and + no cell is gated on a category OR framework outside its own family β€” a typo must not + silently make a conditional cell unconditional, or a gated cell unrenderable. +9. No `settings_fragment` is ungated: one that applies to the whole family + belongs in the settings cell, not beside it. +10. Every `{{ key }}` placeholder names a context key the render contract + defines, and no template carries a half-written `{ key }` -- which is what + an f-string silently turns `{{ key }}` into. +17. No cell carries an internal reference β€” a private tracker id, an RFC id, + an internal hostname. The templates render into a peer's notebook, so + unlike this repo's prose docs, anything in a cell is shown outside the + org. +16. No `training.*` setter call sits outside the settings cell or a settings + fragment, and the settings cell comes after the cell that assigns + `training`. Start re-links and then executes ONLY the settings cell, so a + setter in a standalone cell is inert however inviting it looks β€” and one + placed before the link would `NameError` if a peer uncommented it. +15. The custom-loss offering REACHES EVERY category that accepts one β€” not + merely appears somewhere in the file β€” and reaches none in the families + listed in `no_custom_loss_families`, whose objective is intrinsic. Checking + presence rather than coverage let the exact defect this rule was written to + stop pass it. +14b. `single_pass_frameworks` in `families.json` mirrors the SDK's + `_SURVIVAL_FRAMEWORKS`; where the SDK is importable the two are compared + and drift fails. Where it is not, the skip is PRINTED rather than silent β€” + an unverifiable mirror that says nothing is the same can't-fail shape this + checker exists to catch. +14. `cycles` / `epochs` are never offered to a single-pass framework. The SDK + forces both to 1 for sklearn, lifelines and scikit_survival -- by FRAMEWORK, + for every category -- and only WARNS, so an ungated pre-fill on a family + with such a pair is both silently unhonoured and noisy on exactly the pairs + it cannot serve. +13. Every family is either backed by a COMPLETED full-settings-cell run in + `verification-dev.json` whose cycles/epochs match `families.json`, or + explicitly marked `unverified` with the ticket that explains why. D9 ships + a template only once a live run has COMPLETED on its own defaults, so a + pre-fill edited without a fresh run must fail rather than inherit the old + run's credibility. +12. Every family renders, for every (category, framework) pair it owns β€” read + from the family's own lists, never hardcoded β€” to a settings cell that + is valid Python and carries no unsubstituted placeholder -- the one check + that actually executes the render contract rather than describing it. +11. No setter is offered to a category that refuses it: one in the shared + settings cell must be accepted by EVERY category in the family, and one in + a gated fragment by every category its gate admits. This is the check that + closes the class -- three separate instances of it were found by hand + before it existed. +""" + +from __future__ import annotations + +import json +import os +import re +import sys + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +TEMPLATES = os.path.join(ROOT, "notebooks", "templates") +TABLE = os.path.join(TEMPLATES, "families.json") +EVIDENCE = os.path.join(TEMPLATES, "verification-dev.json") + +MAX_EFFECTIVE_PASSES = 20 + +DEAD_SETTERS = ( + "horizontal_flip", + "vertical_flip", + "samplewise_center", + "samplewise_std_normalization", + "layers_freeze", +) + +COLAB_MARKERS = ("colab.research.google.com", "google.colab", "drive.mount") + +# Internal references must not appear in a TEMPLATE CELL. The templates render +# into a peer's notebook, so anything here is shown to people outside the org +# β€” unlike this repo's prose docs. The survival banner shipped with a private +# tracker id AND an RFC id rendered to strangers; grep-expressible, so it is a +# rule rather than something to stay vigilant about. +# Match the FORM, not a list of spellings. The first version enumerated the +# two leaks that had already happened (one `#N` reference, an RFC id) and so would +# have missed `client#12`, `model-zoo#7`, or a different internal host β€” the +# same "instrument shaped by what I already held" trap this checker keeps +# finding elsewhere. +INTERNAL_REF = re.compile( + r"""( + \b[a-z0-9][a-z0-9._-]*\#\d+ # any #N cross-reference + | \bRFC[- ]?[A-Z]*-?\d+ # RFC-NNNN, RFC-AREA-NNNN + # Any tracebloc host EXCEPT the public ones the templates legitimately + # link (docs. and ai.). Written as a negative lookahead rather than a + # list of internal hosts, so a new internal host is caught by default + # instead of needing to be enumerated β€” which is the whole point of + # matching the form. + | \b(?!docs\.|ai\.)[a-z0-9-]+\.tracebloc\.io\b + | \b(?:dev|staging)-api\b + )""", + re.IGNORECASE | re.VERBOSE, +) + +# The public docs/ai links the templates legitimately carry are URLs, and a +# heading anchor on those pages can be a bare number β€” `hyperparameters#1-optimizer`, +# `hyperparameters#3-loss-function` (the rendered GitBook/Mintlify ids for +# "1. Optimizer" / "3. Loss Function"). INTERNAL_REF's `#N` branch reads +# the `#1` / `#3` fragment of such a URL as a cross-reference, so strip the +# allowed `docs.`/`ai.` URLs (the SAME two hosts the host branch above exempts) +# BEFORE scanning. A real leak like `example-repo#4242` is not inside one +# of these URLs, so it still survives the strip and is still caught. +ALLOWED_DOC_URL = re.compile(r"https?://(?:docs|ai)\.tracebloc\.io/\S*", re.IGNORECASE) + +APPLIES_TO_KEYS = {"category", "framework", "dataset_flag"} +#: What each axis's value must BE. `category`/`framework` are lists the renderer +#: membership-tests; `dataset_flag` is a single string (rule 9b enforces that +#: separately, and `render()` only tests the key's presence). Keyed off +#: APPLIES_TO_KEYS so a new axis cannot be added without declaring its shape. +AXIS_VALUE_TYPES = {"category": list, "framework": list, "dataset_flag": str} +assert set(AXIS_VALUE_TYPES) == APPLIES_TO_KEYS, "declare a shape for every axis" + +TRACEBLOC_CELL_KEYS = {"applies_to", "settings_fragment"} + +# The context keys the render contract defines (notebooks/templates/README.md). +# A `{{ key }}` outside this set would render as literal text in the peer's +# notebook, so it is a typo, not an extension point. +CONTEXT_KEYS = { + "use_case", + "dataset_id", + "category", + "framework", + "edge_count", + "records_per_edge", + "experiment_name", + "model_path", + "validation_split", + "training_classes", + "data_type", + "feature_points", + "sequence_length", + "forecast_horizon", + "scaler", + "tokenizer_path", +} + +PLACEHOLDER = re.compile(r"\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}") +# A lone `{ name }` is a placeholder someone lost a brace on -- exactly what an +# f-string does to `{{ name }}`. Deliberately requires a bare identifier so it +# cannot match a real dict literal like {"type": "constant"}. +HALF_PLACEHOLDER = re.compile( + r"(?(` call", shared by the discovery scan and +#: by `called_with_all` so the two cannot disagree about what a call is. +#: +#: They did disagree, and it cost two holes (@LukasWodka in review). `SETTER_CALL` +#: was `training\.([a-z_]+)\s*\(` -- no `\s*` around the DOT -- and ran on the RAW +#: source, while `called_with_all` had `training\s*\.\s*` and ran on `code_only(src)`: +#: +#: `training . optimizer("adamw")` Start APPLIES it; discovery missed it, so rule 13 +#: never compared it to the record and the family +#: stayed verified. +#: a commented mapped setter discovery FOUND it (comments survive a raw scan) +#: but `called_with_all` returned nothing, so the +#: comparison loop never ran and a stale COMPLETED +#: run kept licensing the cell. +#: +#: One fragment, used by both. That is the derive-not-restate rule this checker exists to +#: enforce, applied to the checker itself. +SETTER_DOT = r"training\s*\.\s*" +SETTER_CALL = re.compile(SETTER_DOT + r"([a-z_]+)\s*\(") + +# A syntactically representative value per context key, used only to render a +# template and compile the result. The point is not the values but that every +# placeholder sits in a position where a real value parses -- a `{{ key }}` +# inside a string literal and one standing as a bare argument are not +# interchangeable, and nothing else would catch the difference. +SAMPLE_CONTEXT = { + "use_case": "Chest X-ray triage", + "dataset_id": "d0gfu0c1", + "category": "image_classification", + "framework": "pytorch", + "edge_count": "2", + "records_per_edge": "30", + "experiment_name": "resnet_18 on Chest X-ray triage #1", + "model_path": "/models/resnet_18.py", + "validation_split": "0.2", + "training_classes": '{"cat": 15, "dog": 15}', + "data_type": "rgb", + "feature_points": "3", + "sequence_length": "128", + "forecast_horizon": "1", + "scaler": "StandardScaler", + "tokenizer_path": "tokenizer.json", +} + + + +# --- one definition of each artefact question ----------------------------- +# +# These existed seven times over, hand-rolled with different fallbacks, and the +# divergence was a finding in its own right: rule 16 skipped a cell as "the +# settings cell" that `render()` simultaneously treated as main, and a +# `"applies_to": null` crashed rule 14 with AttributeError instead of +# reporting. One definition each, used everywhere. + +SETTINGS_MARKER = "# Settings β€” the complete plan for this run, as plain SDK calls." + +# The only dataset flag the platform actually has (`_linking.py` is the source). +# RESTATED MIRRORS, acknowledged rather than implied. These three are copies +# of facts owned elsewhere -- ALL_CATEGORIES and SETTER_CATEGORIES of the +# engine registry and the SDK's per-setter gates, KNOWN_DATASET_FLAGS of the +# backend's dataset flags -- and only `single_pass_frameworks` is cross-checked +# against its source (rule 14b). The others go stale silently, which is the +# same objection this repo's CLAUDE.md raises about restating the SDK's Python +# bound. Deriving them needs an importable engine registry; until then the +# honest position is that they are unverified copies and this comment says so. +KNOWN_DATASET_FLAGS = {"allow_feature_modification"} + + + +# --- rule 13's field map -------------------------------------------------- +# +# Which record field each settings-cell literal must agree with. Binding only +# status/cycles/epochs was a finding: aggregation_strategy, optimizer, seed and +# the rest could all change and inherit the old run's credibility, while the +# README claimed "a new number cannot inherit the old run's credibility". +# +# `None` means "the record does not carry this, so do not claim it is +# verified" -- listed explicitly rather than omitted, so adding a setter to a +# template forces a decision here instead of silently going unchecked. +# +# Several settings land under a DIFFERENT record name than the setter: a text +# `sequence_length(N)` and a tabular `feature_points(N)` both arrive as +# `data_shape`, which is why this is a map and not a name match. +EVIDENCE_FIELD_MAP = { + "cycles": "cycles", + "epochs": "epochs", + "aggregation_strategy": "aggregation_strategy", + "optimizer": "optimizer", + "seed": "seed", + "data_type": "data_type", + "shuffle": "shuffle", + "sequence_length": "data_shape", + "feature_points": "data_shape", + "validation_split": "validation_split", + "learning_rate": "learningRate", + # These ARE recorded -- verification-dev.json exports them for the tabular + # and survival families -- and marking them None meant a change to any of + # them inherited the old COMPLETED run. `scaler` lands under + # `tabular_scaler`, which is the same setter-name-vs-record-name skew that + # makes this a map rather than a name match. + "scaler": "tabular_scaler", + "handle_missing_values": "handle_missing_values", + "imputation_strategy": "imputation_strategy", + "encoding_strategy": "encoding_strategy", + "normalize_features": "normalize_features", + # Deliberately unverifiable against the record, and named so: + "training_classes": None, # the record stores it as `subdataset` + "terminate_on_nan_callback": "callbacks", + "missingness_indicators": None, + "forecast_horizon": None, + "loss_function": None, # every occurrence is commented out + "enable_lora": None, + "set_lora_parameters": None, + "early_stop_callback": None, + "model_checkpoint_callback": None, + "reduce_lr_callback": None, + "feature_interaction": None, + "get_features": None, + "experiment_name": None, # pre-filled per run, not a family constant + # The augmentation group: every line is COMMENTED OUT in the template, so + # the run could not have exercised any of them. Listed rather than omitted + # so the map stays a complete statement about what the evidence covers. + "rotation_range": None, + "width_shift_range": None, + "height_shift_range": None, + "brightness_range": None, + "shear_range": None, + "zoom_range": None, + "channel_shift_range": None, + "fill_mode": None, + "cval": None, + "rescale": None, +} + + +def _literal_matches(setter, literal, recorded): + """Does a settings-cell literal agree with the recorded value?""" + if setter == "terminate_on_nan_callback": + # No argument; presence in the cell must mean presence on the record. + return "terminateOnNaN" in str(recorded) + text = str(recorded) + lit = literal.strip() + if lit.startswith(("'", '"')) and lit[-1:] in "'\"": + return lit[1:-1] == text + try: + return json.loads(lit.replace("'", '"')) == json.loads( + text.replace("'", '"') + ) + except Exception: # noqa: BLE001 - fall back to a text compare + return lit == text + + +def cell_meta(cell): + """The cell's `metadata.tracebloc` dict, never None and never non-dict. + + `tracebloc: true` used to reach `.get` and raise AttributeError from + `settings_source`, and a list-valued `applies_to` did the same in rule 8 β€” + a fail-closed traceback, but the header above promises these helpers end + that class, so they have to actually end it. + """ + meta = cell.get("metadata") + tb = (meta if isinstance(meta, dict) else {}).get("tracebloc") + return tb if isinstance(tb, dict) else {} + + +def cell_gate(cell): + """The cell's applies_to, or {} when it does not gate anything. + + An `applies_to` carrying no key the renderer understands -- `{}`, or only + unknown keys -- is UNGATED, because that is what the renderer does with it. + Reading it as "gated" is how a fragment with `applies_to: {}` slipped past + rule 9 and then appended itself to every render, silently overriding the + family's own values: the exact second-settings-cell that rule 9's message + claims to prevent. + """ + applies = cell_meta(cell).get("applies_to") + if not isinstance(applies, dict): + return {} + # Keys the renderer understands, INCLUDING ones whose list is empty. + # + # An earlier fix dropped empty lists here so the file would "agree", and + # that INVERTED the bug: the pod treats `category: []` as admitting NOBODY, + # while dropping the key made the checker treat it as admitting EVERYBODY. + # The checker then kept such a cell for every pair and counted it as + # offered to the whole family β€” the opposite of what ships. The gate is + # reported as written; `gate_audience()` below applies the pod's meaning. + # SHAPE, not just the key. An axis value of the wrong type used to reach every + # reader: `category: null` raised TypeError in `gate_audience` and in rule 9d's + # membership test, and `category: "vision"` -- a bare string -- did something worse + # than raise. It ITERATED, so `gate_audience` returned + # ['v','i','s','i','o','n'] and the cell was read as offered to categories named + # "v", "i", "s": a silent wrong answer rather than a crash (Bugbot reported the + # TypeError; the string case was found while reproducing it). + # + # Dropped here and REPORTED by rule 8 off the raw dict -- the same division of + # labour this function already uses for unknown keys. `render()` is no kinder to a + # null gate than these rules were (`category not in None` raises there too), so the + # checker's job is to name it, not to emulate it. + return { + k: v + for k, v in applies.items() + if k in APPLIES_TO_KEYS and isinstance(v, AXIS_VALUE_TYPES[k]) + } + + +def gates_nothing(cell): + """True when applies_to carries no key the renderer understands. + + This is rule 9's question β€” `{}`, `null`, absent, or unknown-keys-only β€” + and it is NOT the same as a key present with an empty list, which gates + everything OUT rather than nothing. + """ + return not cell_gate(cell) + + +def gate_audience(cell, axis, whole): + """Who this cell is actually offered to on `axis`, the pod's way. + + * key absent -> the whole family + * key present -> exactly its values, EVEN IF EMPTY (nobody) + + Rules 11/14/15 previously used `gate or whole`, so an empty list fell + through to `whole` and a `category: []` loss fragment satisfied rule 15 + while no pair could ever see it. + """ + gate = cell_gate(cell) + return list(gate[axis]) if axis in gate else list(whole) + + +def is_fragment(cell): + return bool(cell_meta(cell).get("settings_fragment")) + + +def is_settings_cell(cell): + """The single settings cell. One test, used by every rule and by render(). + + Keyed on the generated marker rather than on prose, so a comment merely + mentioning `training.experiment_name` cannot impersonate it. + """ + if cell.get("cell_type") != "code": + return False + src = "".join(cell["source"]) + return SETTINGS_MARKER in src and not is_fragment(cell) + + + +def render(nb, category, framework, with_flags=False): + """Apply the render contract: drop what does not apply, substitute + placeholders, concatenate surviving settings fragments into the settings + cell. Returns the settings cell source.""" + main = "" + fragments = [] + for cell in nb["cells"]: + if cell["cell_type"] != "code": + continue + applies = cell_gate(cell) + # An empty list admits nobody, so `category not in []` drops the cell β€” + # which is exactly what the pod does. Reported here rather than + # normalised away. + if "category" in applies and category not in applies["category"]: + continue + if "framework" in applies and framework not in applies["framework"]: + continue + if "dataset_flag" in applies and not with_flags: + continue + src = "".join(cell["source"]) + if is_fragment(cell): + fragments.append(src) + elif is_settings_cell(cell): + main = src + text = main + "\n" + "\n".join(fragments) + for k, v in SAMPLE_CONTEXT.items(): + text = re.sub(r"\{\{\s*" + k + r"\s*\}\}", v.replace("\\", "\\\\"), text) + return text + + +def derive_single_pass_frameworks(floor): + """Read the single-pass framework set from the SDK, with its version. + + Returns (frameworks, version, problem). Exactly one of `frameworks` / + `problem` is meaningful. + + Three things this used to get wrong, all found on start-training#89 (9/10): + + * It imported whatever `tracebloc` happened to be on `sys.path` and + reported "matches the installed SDK" with no version. Measured: `pip + install tracebloc` under Python 3.9 silently backtracks to **0.8.1** β€” a + release predating these templates β€” whose `_SURVIVAL_FRAMEWORKS` happens + to match, so an ancient SDK "confirmed" the mirror. Hence the floor. + * A bare `except Exception` reported a present-but-BROKEN SDK as "not + importable", which is a different fact and hides a real problem. + * "Cannot tell" printed a note and exited 0 β€” in CI, always. A + cross-check that never runs where it gates is the defect this whole file + exists to catch, so `TRACEBLOC_CHECK_STRICT` turns every + cannot-tell into a failure. + """ + try: + import importlib.metadata as md + + version = md.version("tracebloc") + except ModuleNotFoundError: + return None, None, "the tracebloc distribution is not installed" + except Exception as exc: # noqa: BLE001 - metadata present but unreadable + return None, None, f"tracebloc metadata unreadable: {exc!r}" + + if _version_tuple(version) < _version_tuple(floor): + return ( + None, + version, + f"installed tracebloc {version} is below the floor {floor} recorded " + f"in families.json, so it cannot confirm this mirror β€” an older " + f"release can agree by coincidence", + ) + try: + from tracebloc.training.plan import _SURVIVAL_FRAMEWORKS + except ImportError as exc: + return None, version, f"tracebloc {version} is installed but not importable: {exc}" + except Exception as exc: # noqa: BLE001 - importable but raising + return None, version, f"tracebloc {version} raised on import: {exc!r}" + return ( + {getattr(f, "value", f) for f in _SURVIVAL_FRAMEWORKS}, + version, + None, + ) + + +def _version_tuple(v): + """The numeric release segment, for a floor comparison. + + Concatenating the digits of a chunk put a PRE-RELEASE ABOVE the floor: + "1.0.7rc1" became (1, 0, 71), so a release candidate of the floor version + read as newer than the floor. Split on the first non-digit instead, which + makes 1.0.7rc1 -> (1, 0, 7) β€” equal to the floor, and accepted, which is + the conservative reading for a floor rather than the flattering one. + """ + parts = [] + for chunk in str(v).split(".")[:3]: + digits = "" + for ch in chunk: + if not ch.isdigit(): + break + digits += ch + parts.append(int(digits) if digits else 0) + return tuple(parts) + + +STRICT = os.environ.get("TRACEBLOC_CHECK_STRICT") == "1" + + +def settings_source(nb): + """Return the settings cell plus its fragments, or None if there is no cell. + + Fragments are concatenated into the settings cell at render time, so a + value that lives in a gated fragment is still part of the settings the peer + sees. Reading only the main cell would false-fail a template that gates a + setting per framework β€” which is exactly what `cycles`/`epochs` need in the + families with single-pass (sklearn / lifelines / scikit_survival) pairs. + """ + main = None + fragments = [] + for cell in nb["cells"]: + if cell["cell_type"] != "code": + continue + src = "".join(cell["source"]) + if is_fragment(cell): + fragments.append(src) + elif is_settings_cell(cell): + main = src + if main is None: + return None + return "\n".join([main] + fragments) + + +def code_only(src): + """`src` with comment text removed, so only LIVE calls remain. + + Two different questions were sharing one matcher: + + * "is this pre-fill APPLIED?" β€” rules 3 and 13. Only live code counts. + * "does this template OFFER this?" β€” rules 5, 6, 16. A commented + `# training.start()` or a commented dead setter still invites a peer to + uncomment it, so comments MUST count there. + + Anchoring `called_with_all` at line start used to exclude comments as a + side effect. Widening it to catch `training . cycles (99)` lost that, and + the regression was worse than the bug it fixed: commenting out the real + `training.cycles(20)` and leaving the comment made rule 3 agree with + families.json while Start applied the SDK default. Now the two questions + use two matchers, on purpose. + + Deliberately naive about `#` inside string literals: the settings cells + contain none, and a false "this is a comment" makes rule 3 report a + MISSING pre-fill, which fails loudly rather than passing quietly. + """ + out = [] + for line in src.split("\n"): + out.append(line.split("#", 1)[0]) + return "\n".join(out) + + +def called_with_all(src, method): + """Every literal argument of `training.(...)`, in order. + + Returning only the FIRST match was a finding: a second + `training.cycles(99)` after the pre-fill left rule 3 reading 20 and passing + while Python applied 99, bypassing the 20-pass ceiling the rule exists to + hold. Callers must decide what more than one means; for a pre-fill it is + always an error. + """ + # Match every form `SETTER_CALL` does β€” `training . cycles ( 99 )`, an + # indented call, an assignment β€” not just a line-start `training.m(`. + # Widening to "all matches" while keeping the NARROW pattern left the hole + # open: a second call Python would apply was still invisible to rule 3, + # so the checker could print OK while Start ran a different value. + # SETTER_DOT, not a second copy of it. The two spellings drifting is exactly the + # defect this function's own comment describes one paragraph up. + return re.findall( + rf"{SETTER_DOT}{method}\s*\(\s*([^)]*?)\s*\)", code_only(src) + ) + + +def main() -> int: + errors = [] + notes = [] + + with open(TABLE) as fh: + table = json.load(fh) + families = table["families"] + drift_ok = set(table["drift_correcting_strategies"]) + + seen_categories = {} + # Each notebook is parsed once here and reused by the later rules; the + # families whose file is missing or unparseable are simply absent, which is + # what lets those rules drop their own existence guards. + loaded = {} + + for fam in families: + key = fam["key"] + cycles, epochs = fam["cycles"], fam["epochs"] + strategy = fam["aggregation_strategy"] + + # Rule 1 β€” the budget ceiling. + if cycles * epochs > MAX_EFFECTIVE_PASSES: + errors.append( + f"{key}: cycles*epochs = {cycles}*{epochs} = {cycles * epochs} " + f"> {MAX_EFFECTIVE_PASSES}" + ) + + # Rule 2 β€” epochs > 1 needs a drift correction. + if epochs > 1 and strategy not in drift_ok: + errors.append( + f"{key}: pre-fills epochs={epochs} with aggregation_strategy " + f"'{strategy}', which carries no drift correction. Use one of " + f"{sorted(drift_ok)}, or keep epochs at 1." + ) + + # Rule 4 β€” categories partition, notebook exists. + for cat in fam["categories"]: + if cat in seen_categories: + errors.append( + f"category '{cat}' claimed by both '{seen_categories[cat]}' " + f"and '{key}'" + ) + seen_categories[cat] = key + if cat not in ALL_CATEGORIES: + errors.append(f"{key}: unknown category '{cat}'") + + path = os.path.join(TEMPLATES, fam["template"]) + if not os.path.isfile(path): + errors.append(f"{key}: template not found: {fam['template']}") + continue + try: + with open(path) as fh: + nb = json.load(fh) + except json.JSONDecodeError as exc: + errors.append(f"{key}: {fam['template']} is not valid JSON: {exc}") + continue + loaded[key] = nb + + # Join with a newline, not "": a cell's last source line carries no + # trailing newline, so concatenating directly welds the last line of + # one cell onto the first line of the next. That both hides + # line-anchored matches and invents text that is in no cell. + whole = "\n".join("".join(c["source"]) for c in nb["cells"]) + + # Rule 3 moved INTO rule 12's per-pair loop. It used to read a + # gate-blind CONCATENATION of the settings cell plus every fragment, + # which made it count one `cycles` call per framework-gated fragment: + # adding a second framework to a family failed the check, so the rule + # actively BLOCKED a family from ever gaining a framework. Per pair is + # the only frame where "single-pass means zero calls, everything else + # exactly one equal to the table" is expressible at all. + # Rule 5 β€” no template starts a run. Deliberately not anchored to the + # start of a line: a commented-out `# training.start()` is still a + # template telling a peer to bypass the button. + if re.search(r"training\s*\.\s*start\s*\(", whole): + errors.append( + f"{key}: {fam['template']} calls training.start(). Start is a " + f"button; no cell may start a run." + ) + + # Rule 6 β€” no dead setters. + for dead in DEAD_SETTERS: + if f"training.{dead}" in whole: + errors.append( + f"{key}: {fam['template']} mentions dead setter " + f"'{dead}' (a no-op on every surviving framework)" + ) + + # Rule 7 β€” no Colab lines. + for marker in COLAB_MARKERS: + if marker in whole: + errors.append( + f"{key}: {fam['template']} carries the Colab-specific " + f"marker '{marker}'; templates render in the pod" + ) + + # Rule 10 β€” placeholders are real context keys, and none is + # half-written. `{ scaler }` instead of `{{ scaler }}` renders as + # literal text in the peer's notebook and no other check sees it. + for name in sorted(set(PLACEHOLDER.findall(whole))): + if name not in CONTEXT_KEYS: + errors.append( + f"{key}: {fam['template']} uses placeholder " + f"{{{{ {name} }}}}, which is not a context key " + f"({sorted(CONTEXT_KEYS)})" + ) + for name in sorted(set(HALF_PLACEHOLDER.findall(whole))): + if name in CONTEXT_KEYS: + errors.append( + f"{key}: {fam['template']} has a half-written placeholder " + f"{{ {name} }} β€” it needs double braces to be substituted" + ) + + # Rules 8 and 9 β€” the render metadata is well formed. + for i, cell in enumerate(nb["cells"]): + # BEFORE the falsy-guard: cell_meta() normalises a non-dict to {}, + # so a `tracebloc: true` would `continue` here and the cell would + # be silently treated as ungated. Name it first. + raw_tb = (cell.get("metadata") or {}).get("tracebloc") + if raw_tb is not None and not isinstance(raw_tb, dict): + errors.append( + f"{key}: {fam['template']} cell {i} has a " + f"metadata.tracebloc that is {type(raw_tb).__name__}, not " + f"an object. Nothing crashes, because cell_meta() " + f"normalises it β€” but silence would turn a gated cell into " + f"an UNGATED one, trading a loud failure for a quiet one." + ) + tb = cell_meta(cell) + if not tb: + continue + + unknown_meta = set(tb) - TRACEBLOC_CELL_KEYS + if unknown_meta: + errors.append( + f"{key}: {fam['template']} cell {i} has unknown " + f"metadata.tracebloc keys {sorted(unknown_meta)}; known " + f"keys are {sorted(TRACEBLOC_CELL_KEYS)}" + ) + + raw_applies = tb.get("applies_to") + if raw_applies is not None and not isinstance(raw_applies, dict): + errors.append( + f"{key}: {fam['template']} cell {i} has an applies_to that " + f"is {type(raw_applies).__name__}, not an object β€” the " + f"renderer reads keys off it. Reported rather than raising, " + f"which this file's shared helpers exist to guarantee." + ) + applies = cell_gate(cell) + if raw_applies is not None: + # RAW dict: cell_gate() has already dropped unknown keys, so + # checking its output could only ever catch the + # typo-is-the-only-key case. A typo ALONGSIDE a valid key is + # the case this rule was written for β€” it turns a conditional + # cell unconditional on the axis that was misspelt. + # `isinstance` FIRST: a non-dict `applies_to` is already reported + # above, and `.items()` on the LIST form raises AttributeError -- + # which broke mutation 8c the moment this loop was added. The + # sibling `set(raw_applies or {})` below tolerates a list by + # accident; this one has to say so. + axis_items = raw_applies.items() if isinstance(raw_applies, dict) else () + for axis, value in axis_items: + if axis not in APPLIES_TO_KEYS: + continue # named by the unknown-keys check below + if not isinstance(value, AXIS_VALUE_TYPES[axis]): + errors.append( + f"{key}: {fam['template']} cell {i} has an " + f"applies_to.{axis} that is " + f"{type(value).__name__}, not " + f"{AXIS_VALUE_TYPES[axis].__name__} β€” the renderer " + f"membership-tests it. Reported rather than raising: " + f"a null crashed the checker and a bare string was " + f"read as its own characters." + ) + unknown = set(raw_applies or {}) - APPLIES_TO_KEYS + if unknown: + errors.append( + f"{key}: {fam['template']} cell {i} has applies_to keys " + f"{sorted(unknown)}; known keys are " + f"{sorted(APPLIES_TO_KEYS)}" + ) + for cat in applies.get("category", []): + if cat not in fam["categories"]: + errors.append( + f"{key}: {fam['template']} cell {i} is gated on " + f"category '{cat}', which is not in this family " + f"({sorted(fam['categories'])}) β€” it would never " + f"render" + ) + # The same check on the framework axis, which had none. Rule 12 + # renders per (category, framework) from the family's own + # lists, so a gate naming a framework the family does not ship + # is a cell nothing can ever render β€” and nothing would say so. + for fw in applies.get("framework", []): + if fw not in fam.get("frameworks", ()): + errors.append( + f"{key}: {fam['template']} cell {i} is gated on " + f"framework '{fw}', which is not in this family " + f"({sorted(fam.get('frameworks', ()))}) β€” it would " + f"never render" + ) + + # Rule 9 β€” a settings fragment with no EFFECTIVE gate is not a + # fragment. It applies to the whole family, so it belongs in the + # settings cell; left as a fragment it appends itself after the + # main block on every render and silently overrides the family's + # own values β€” the second settings cell this rule claims to stop. + # + # "No effective gate" now means what the RENDERER means, via + # cell_gate(): absent, null, `{}`, or only unknown keys. Testing + # `applies_to is None` let `{}` through, and a `{}`-gated fragment + # carrying optimizer/seed overrode sgd/0 on every pair. + if is_fragment(cell) and gates_nothing(cell): + raw = cell_meta(cell).get("applies_to", "") + errors.append( + f"{key}: {fam['template']} cell {i} is a settings_fragment " + f"whose applies_to ({raw!r}) gates nothing, so the " + f"renderer keeps it for every pair. An ungated fragment " + f"belongs in the settings cell." + ) + + # Rule 9c β€” a gate key present with an EMPTY list renders for no + # pair at all. cell_gate() drops it so nothing mistakes it for a + # gate, but silence would leave a cell that can never appear. + raw_gate = cell_meta(cell).get("applies_to") + if isinstance(raw_gate, dict): + for gk, gv in raw_gate.items(): + if gk in APPLIES_TO_KEYS and gv == []: + errors.append( + f"{key}: {fam['template']} cell {i} is gated on " + f"an EMPTY {gk} list, which admits no pair β€” the " + f"cell can never render. Remove the key, or name " + f"the {gk} values it applies to." + ) + + # Rule 9b β€” a dataset_flag must name a flag that exists, and a + # gate that admits the whole family is not gating. + flag = cell_gate(cell).get("dataset_flag") + if flag is not None and not isinstance(flag, str): + errors.append( + f"{key}: {fam['template']} cell {i} has a dataset_flag " + f"that is {type(flag).__name__}, not a string" + ) + elif flag is not None and flag not in KNOWN_DATASET_FLAGS: + errors.append( + f"{key}: {fam['template']} cell {i} is gated on " + f"dataset_flag '{flag}', which is not a flag the platform " + f"has ({sorted(KNOWN_DATASET_FLAGS)}). render() never " + f"reads the value, so a typo here renders the cell always " + f"or never with nothing to say so." + ) + # Does the WHOLE gate drop any pair the family renders? Asking + # only about the `category` key both missed a framework gate that + # admits every framework the family ships, and FALSELY rejected a + # framework-gated fragment that also listed every category β€” + # render() does drop that one, on the framework axis. + gate = cell_gate(cell) + if gate and set(gate) & APPLIES_TO_KEYS: + # EVERY axis render() drops on, not two of the three. Asking only + # about `category` missed a framework gate admitting every framework + # the family ships, and falsely rejected a framework-gated fragment + # that also listed every category. Adding `framework` fixed those and + # left the SAME defect one axis along: a gate carrying `dataset_flag` + # was judged on category/framework alone and called a no-op, when + # render() drops it on the flags-off pass (`render()`: `if + # "dataset_flag" in applies and not with_flags: continue`). + # + # So the space is the render signature's own product -- + # (category, framework, with_flags) -- and the three drop conditions + # below mirror render's three, in its order. APPLIES_TO_KEYS is the + # authority for how many axes there are, so a fourth gate key added + # there cannot silently leave this rule judging three. + renders = [ + (c, f, w) + for c in fam["categories"] + for f in fam.get("frameworks", ("pytorch",)) + for w in (False, True) + ] + drops_some = any( + ("category" in gate and c not in gate["category"]) + or ("framework" in gate and f not in gate["framework"]) + or ("dataset_flag" in gate and not w) + for c, f, w in renders + ) + if not drops_some: + errors.append( + f"{key}: {fam['template']} cell {i} is gated on " + f"{ {k: gate[k] for k in gate if k in APPLIES_TO_KEYS} } " + f"but that admits EVERY render the family produces, so " + f"the gate never drops it. Remove it, or narrow it to " + f"what actually differs." + ) + + # Rule 11 β€” no setter offered to a category that refuses it. + for fam in families: + nb = loaded.get(fam["key"]) + if nb is None: + continue + for i, cell in enumerate(nb["cells"]): + if cell["cell_type"] != "code": + continue + tb = cell_meta(cell) + audience = tuple(gate_audience(cell, "category", fam["categories"])) + src = "".join(cell["source"]) + for setter in sorted(set(SETTER_CALL.findall(src))): + allowed = SETTER_CATEGORIES.get(setter) + if allowed is None: + continue + refused = [c for c in audience if c not in allowed] + if refused: + errors.append( + f"{fam['key']}: {fam['template']} cell {i} offers " + f"training.{setter}() to {sorted(refused)}, which " + f"refuse it. Gate the cell, or move the call into a " + f"fragment for {sorted(set(audience) & set(allowed))}." + ) + + # Rule 12 β€” the render contract, executed. + for fam in families: + nb = loaded.get(fam["key"]) + if nb is None: + continue + # The family's OWN frameworks. Hardcoding ("pytorch", "sklearn") both + # skipped real pairs (survival's lifelines / scikit_survival were never + # rendered, so a fragment gated only on those could carry a syntax + # error and stay green) and rendered illegal ones (vision + sklearn). + # `frameworks` is per FAMILY, so this over-approximates: it renders + # e.g. (time_series_classification, sklearn) though the zoo ships no + # sklearn model there. Over-approximating is the safe direction for a + # validator β€” it can only demand more validity, never less β€” and the + # precise per-category set is the engine registry's legal-pair matrix + # in the engine registry, not something to restate here. + for cat in fam["categories"]: + for framework in fam.get("frameworks", ("pytorch",)): + # Render twice and validate BOTH: flags off is what most + # datasets get, flags on is the only way a flag-gated fragment + # is ever compiled. An earlier version assigned `text` in the + # loop but validated after it, so only the flags-on pass was + # ever checked β€” it swapped coverage of the common path for the + # rare one and no mutation caught it, because the one template + # with a flag-gated fragment was not the one being mutated. + for with_flags in (False, True): + mode = "flags on" if with_flags else "flags off" + where = f"({cat}, {framework}, {mode})" + text = render(nb, cat, framework, with_flags) + if not text.strip(): + errors.append(f"{fam['key']}: renders empty for {where}") + continue + leftover = PLACEHOLDER.findall(text) + HALF_PLACEHOLDER.findall( + text + ) + leftover = [n for n in leftover if n in CONTEXT_KEYS] + if leftover: + errors.append( + f"{fam['key']}: {where} renders with unsubstituted " + f"placeholder(s) {sorted(set(leftover))}" + ) + try: + compile( + text, + f"{fam['template']}::{cat}/{framework}/{mode}", + "exec", + ) + except SyntaxError as exc: + errors.append( + f"{fam['key']}: {where} renders to invalid Python: " + f"{exc.msg} at line {exc.lineno}" + ) + continue + + # Rule 3, per rendered pair. A single-pass framework gets + # ZERO calls (the SDK forces 1 and only warns, rule 14); + # every other pair gets exactly ONE, equal to the table. + # Audit BOTH modes. Skipping flags-on left a `cycles(99)` + # inside a flag-gated fragment invisible: verified families + # are backstopped by rule 13, but the UNVERIFIED one is + # not, so there the pre-fill silently changes. + single = framework in set( + table.get("single_pass_frameworks", ()) + ) + # `aggregation_strategy` was bound to the table by the + # old whole-file rule 3 and the per-pair move DROPPED it, + # so flipping it in families.json alone passed. Rule 13 + # catches the notebook side; nothing bound the TABLE, which + # this file's own header calls the single source. + for method, expected in ( + ("cycles", str(fam["cycles"])), + ("epochs", str(fam["epochs"])), + ( + "aggregation_strategy", + f'"{fam["aggregation_strategy"]}"', + ), + ): + got = called_with_all(text, method) + if single and method in ("cycles", "epochs"): + if got: + errors.append( + f"{fam['key']}: {where} calls " + f"training.{method}({got}) on a " + f"single-pass framework, which forces it " + f"to 1 and only warns β€” the value is " + f"silently dropped" + ) + elif len(got) != 1: + errors.append( + f"{fam['key']}: {where} calls " + f"training.{method}() {len(got)} times " + f"({got}); exactly one is required, because " + f"Python applies the last" + ) + elif got[0] != expected: + errors.append( + f"{fam['key']}: {where} has " + f"training.{method}({got[0]}) but " + f"families.json says {expected}" + ) + + # Rule 14b β€” the mirrored framework set must match the SDK's own. + declared_single_pass = set(table.get("single_pass_frameworks", ())) + floor = table.get("sdk_version_floor") + if not floor: + errors.append( + "families.json has no `sdk_version_floor`, so the mirror " + "cross-check cannot tell a current SDK from one that predates " + "these templates." + ) + else: + derived, version, problem = derive_single_pass_frameworks(floor) + if problem: + msg = ( + f"single_pass_frameworks NOT cross-checked ({problem}). It " + f"mirrors `tracebloc.training.plan._SURVIVAL_FRAMEWORKS`." + ) + if STRICT: + errors.append( + msg + " TRACEBLOC_CHECK_STRICT=1, and a mirror verified " + "nowhere that gates is the whole defect." + ) + else: + notes.append(msg + " Set TRACEBLOC_CHECK_STRICT=1 to make this fail.") + elif derived != declared_single_pass: + errors.append( + f"single_pass_frameworks in families.json is " + f"{sorted(declared_single_pass)} but tracebloc {version} has " + f"_SURVIVAL_FRAMEWORKS = {sorted(derived)}. The mirror has " + f"drifted β€” update families.json, and re-check which families " + f"need cycles/epochs gated on framework." + ) + else: + notes.append( + f"single_pass_frameworks matches tracebloc {version} " + f"(floor {floor}): {sorted(derived)}." + ) + + # Rule 16 β€” every setter call reachable by Start, and after the link. + for fam in families: + nb = loaded.get(fam["key"]) + if nb is None: + continue + link_at = settings_at = None + for i, cell in enumerate(nb["cells"]): + if cell["cell_type"] != "code": + continue + src = "".join(cell["source"]) + if "link_model_dataset" in src: + link_at = i if link_at is None else link_at + if is_settings_cell(cell): + settings_at = i + continue + if is_fragment(cell): + continue + # Match a CALL, not the substring "training." β€” the tokenizer cell's + # prose ends "not at training." and a substring test flags it. + calls = sorted(set(SETTER_CALL.findall(src))) + if calls: + errors.append( + f"{fam['key']}: {fam['template']} cell {i} calls " + f"training.{'/'.join(calls)}() outside the settings cell " + f"and outside any settings fragment. Start executes only " + f"the settings cell, so these are inert. Move them into " + f"the settings cell, or into a gated fragment." + ) + if settings_at is None: + continue + if link_at is None: + errors.append( + f"{fam['key']}: {fam['template']} never assigns `training` via " + f"link_model_dataset()" + ) + elif link_at > settings_at: + errors.append( + f"{fam['key']}: {fam['template']} has its settings cell " + f"(cell {settings_at}) BEFORE the link (cell {link_at}), so " + f"`training` is unassigned when the settings run." + ) + + # Rule 17 β€” no internal reference in a cell a peer will read. + for fam in families: + nb = loaded.get(fam["key"]) + if nb is None: + continue + for i, cell in enumerate(nb["cells"]): + text = ALLOWED_DOC_URL.sub("", "".join(cell["source"])) + found = sorted(set(INTERNAL_REF.findall(text))) + if found: + errors.append( + f"{fam['key']}: {fam['template']} cell {i} carries " + f"internal reference(s) {found}. These templates render " + f"into a peer's notebook, so this is shown outside the " + f"org β€” describe the situation instead of citing a tracker." + ) + + # Rule 3b β€” EXACTLY ONE settings cell per template. + # + # Nothing asserted this. `render()`, `settings_source()` and rule 16 each + # take the LAST match silently, so a second marker cell inserted BEFORE the + # real one carrying `cycles(99)` passed with exit 0 while Start applied 99. + for fam in families: + nb = loaded.get(fam["key"]) + if nb is None: + continue + found = [i for i, c in enumerate(nb["cells"]) if is_settings_cell(c)] + if len(found) != 1: + errors.append( + f"{fam['key']}: {fam['template']} has {len(found)} settings " + f"cells (indices {found}); exactly one is required. Every rule " + f"and render() silently take the last, so a second one is a " + f"pre-fill nobody audits." + ) + + # Rule 15 β€” the custom-loss offering must reach EVERY category that takes + # one, not merely appear somewhere in the file. + # + # The first version of this rule asked `any("training.loss_function" in + # src)`, which is presence, not coverage β€” so re-introducing the very + # defect it was written to stop (gating vision's loss cell to + # object_detection alone, hiding it from the other three) PASSED. A rule + # that cannot fail on its own motivating case is the shape this whole + # checker exists to catch, and it was in the checker. + no_loss = set(table.get("no_custom_loss_families", ())) + for fam in families: + nb = loaded.get(fam["key"]) + if nb is None: + continue + offered_to = set() + for cell in nb["cells"]: + if cell["cell_type"] != "code": + continue + if "training.loss_function" not in "".join(cell["source"]): + continue + offered_to |= set(gate_audience(cell, "category", fam["categories"])) + + if fam["key"] in no_loss: + if offered_to: + errors.append( + f"{fam['key']}: {fam['template']} offers a custom loss to " + f"{sorted(offered_to)}, but this family's objective is " + f"intrinsic β€” the SDK refuses a loss.py at upload, so the " + f"cell advertises a rejection." + ) + continue + + missing = set(fam["categories"]) - offered_to + if missing: + where = "nowhere in the template" if not offered_to else ( + f"only to {sorted(offered_to)}" + ) + errors.append( + f"{fam['key']}: {fam['template']} offers a custom loss " + f"{where}, so {sorted(missing)} never see it although they " + f"accept one. Put the call in the settings cell, or in " + f"fragments whose gates cover every category in the family." + ) + + # Rule 14 β€” cycles/epochs must not reach a single-pass framework. + single_pass = set(table.get("single_pass_frameworks", ())) + for fam in families: + nb = loaded.get(fam["key"]) + if nb is None: + continue + fam_forced = set(fam.get("frameworks", ())) & single_pass + for i, cell in enumerate(nb["cells"]): + if cell["cell_type"] != "code": + continue + src = "".join(cell["source"]) + calls = [ + m for m in ("cycles", "epochs") + if re.search(rf"^\s*training\.{m}\(", src, flags=re.MULTILINE) + ] + if not calls: + continue + audience = set(gate_audience(cell, "framework", fam.get("frameworks", ()))) + reached = audience & single_pass + if reached: + errors.append( + f"{fam['key']}: {fam['template']} cell {i} offers " + f"training.{'/'.join(calls)}() to {sorted(reached)}, which " + f"force both to 1 and only warn β€” the pre-fill would be " + f"silently unhonoured there. Gate the cell on " + f"framework, as the family's own pairs include " + f"{sorted(fam_forced)}." + ) + + # Rule 13 β€” D9's "ships only after a COMPLETED run" made mechanical. + try: + with open(EVIDENCE) as fh: + evidence = json.load(fh)["runs"] + except (OSError, KeyError, json.JSONDecodeError) as exc: + errors.append(f"verification-dev.json unreadable: {exc}") + evidence = {} + by_family = {v["family"]: v for v in evidence.values()} + for fam in families: + key = fam["key"] + rec = by_family.get(key) + run = (rec or {}).get("rounds", {}).get("2_full_settings_cell") or {} + # Compare EVERY literal the rendered settings cell sets against the + # record, through EVIDENCE_FIELD_MAP, and treat a missing field as a + # failure rather than as agreement. + mismatches = [] + nb_fam = loaded.get(key) + if run.get("status") != "COMPLETED": + mismatches.append(f"status={run.get('status')!r}") + elif nb_fam is not None: + src = settings_source(nb_fam) or "" + # `code_only`, matching `called_with_all` below. Discovering on the RAW + # source found commented setters that `called_with_all` then returned nothing + # for, so the comparison loop never ran and the stale COMPLETED run kept + # licensing the cell. Rule 13 asks "is this pre-fill APPLIED?", which + # `code_only`'s docstring names as the live-code-only question. + for setter in sorted(set(SETTER_CALL.findall(code_only(src)))): + if setter not in EVIDENCE_FIELD_MAP: + mismatches.append( + f"{setter}: not in EVIDENCE_FIELD_MAP, so nothing " + f"says whether the run verified it" + ) + continue + field = EVIDENCE_FIELD_MAP[setter] + if field is None: + continue + if field not in run: + mismatches.append( + f"{setter}: record has no '{field}' field" + ) + continue + for literal in called_with_all(src, setter): + # A `{{ placeholder }}` is CONTEXT-derived -- the SDK + # computes validation_split from the dataset, the picker + # supplies sequence_length -- so one run's value cannot + # confirm or refute it as a pre-fill. Only literals the + # template fixes are the template's claim to verify. + if PLACEHOLDER.search(literal): + continue + if not _literal_matches(setter, literal, run[field]): + mismatches.append( + f"{setter}({literal}) vs recorded " + f"{field}={run[field]!r}" + ) + if not run.get("experiment"): + mismatches.append("record names no experiment") + verified = not mismatches + declared = fam.get("unverified") + if verified and declared: + errors.append( + f"{key}: marked unverified ('{declared}') but " + f"verification-dev.json has a COMPLETED run " + f"({run.get('experiment')}) matching its pre-fills. Drop the " + f"marker." + ) + elif not verified and not declared: + why = "no record at all" if not run else ( + "; ".join(mismatches[:6]) + ( + f" (+{len(mismatches) - 6} more)" if len(mismatches) > 6 else "" + ) + ) if mismatches else ( + f"recorded run {run.get('experiment')} is " + f"{run.get('status')} at cycles={run.get('cycles')}, " + f"epochs={run.get('epochs')}" + ) + errors.append( + f"{key}: pre-fills cycles={fam['cycles']}, " + f"epochs={fam['epochs']} are not backed by a COMPLETED " + f"full-settings-cell run ({why}). Re-run on dev and refresh " + f"verification-dev.json, or mark the family `unverified` with " + f"the ticket that explains why." + ) + + # Rule 4, other half β€” nothing left unclaimed. + missing = ALL_CATEGORIES - set(seen_categories) + if missing: + errors.append( + f"categories with no family template: {sorted(missing)}" + ) + + for n in notes: + print(f"note: {n}") + if notes: + print() + + if errors: + print(f"{len(errors)} problem(s):\n") + for e in errors: + print(f" - {e}") + return 1 + + print( + f"OK β€” {len(families)} families, " + f"{len(seen_categories)} categories, all D9 rules hold." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_templates_mutations.py b/scripts/check_templates_mutations.py new file mode 100755 index 0000000..20ec4d1 --- /dev/null +++ b/scripts/check_templates_mutations.py @@ -0,0 +1,700 @@ +#!/usr/bin/env python3 +"""Mutation harness for `check_templates.py`: every rule, seen to fail. + +Run from the repo root: + + python3 scripts/check_templates_mutations.py + +Each mutation copies the templates and the checker into a temp dir, applies one +change that realises a WRONG ANSWER, runs the real checker as a subprocess, and +asserts it exits non-zero β€” and, where the mutation is mode- or +identity-specific, that the message names the right thing rather than merely +that something fired. + +## Why this file exists + +Until now the suite was PROSE. Review notes said "all twelve spot-checked rules +fire", with no artefact anyone could run. That is the same defect the checker +itself exists to catch, one level up: a claim about enforcement with nothing +executing it. + +The concrete cost, on this very code: a slice-based edit while rewriting +rule 14b **deleted rule 16 outright**. Nothing noticed, because only the +mutations for the rules being touched were re-run. It surfaced by luck β€” an +unrelated rule-16 mutation happened to come back green during a later spot +check. A committed harness turns that luck into a failing test, which is the +whole argument for this file (start-training#91, review). + +Two disciplines it encodes, both learned the hard way here: + +* **A mutation must sit where the code path it tests actually differs.** Rule + 12's mutations once lived in the vision template, which has no flag-gated + fragments β€” so its flags-off and flags-on renders are byte-identical and no + mutation there could tell the two modes apart. The mode-specific ones live in + `tabular_timeseries` for that reason. +* **Assert the identity or count of what fires, not that something fired.** A + flags-off break must be reported in BOTH modes; a flags-on break in one. Had + these asserted only "non-zero", a single-mode loop would still have passed. +""" + +from __future__ import annotations + +import functools +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +#: The checker this harness exercises. Imported for its PREDICATES (see +#: `_checker`) as well as run as a subprocess for its verdict -- one definition of +#: "which cell is the settings cell", not two. +CHECKER = os.path.join(ROOT, "scripts", "check_templates.py") +TEMPLATES = os.path.join("notebooks", "templates") + + +# --- mutation helpers ------------------------------------------------------ + +def _load(work, name): + with open(os.path.join(work, TEMPLATES, name)) as fh: + return json.load(fh) + + +def _save(work, name, doc): + with open(os.path.join(work, TEMPLATES, name), "w") as fh: + json.dump(doc, fh, indent=1) + fh.write("\n") + + +def _sub(work, name, old, new): + """Replace `old` with `new` in every source line of a notebook.""" + nb = _load(work, name) + hit = False + for cell in nb["cells"]: + fixed = [line.replace(old, new) for line in cell["source"]] + hit = hit or fixed != cell["source"] + cell["source"] = fixed + if not hit: + raise AssertionError(f"mutation target not found in {name}: {old!r}") + _save(work, name, nb) + + +@functools.lru_cache(maxsize=1) +def _checker(): + """The checker module itself, so its predicates are not re-implemented here. + + This harness ran the checker as a SUBPROCESS and separately re-implemented one of + its predicates, which is the fake-proof generator Bugbot named: `_settings_cell` + keyed on `training.experiment_name` appearing in any code cell, while the checker + requires `SETTINGS_MARKER`. Two answers to "which cell is the settings cell" means + a mutation can edit a DIFFERENT cell than the rule under test reads, and then pass + for the wrong reason. + + Not hypothetical: the two already disagree on shipped content -- + `traceblocTrainingGuide.ipynb` cell 17 mentions that setter and is NOT a settings + cell, so the legacy predicate matches it and `is_settings_cell` does not. The six + family templates agree today, which is exactly why this was invisible. + """ + spec = importlib.util.spec_from_file_location("_checker", CHECKER) + module = importlib.util.module_from_spec(spec) + sys.modules["_checker"] = module + spec.loader.exec_module(module) + return module + + +def _settings_cell(nb): + checker = _checker() + for cell in nb["cells"]: + if checker.is_settings_cell(cell): + return cell + raise AssertionError("no settings cell") + + +def _append_cell(work, name, source, meta=None): + nb = _load(work, name) + nb["cells"].append( + { + "cell_type": "code", + "metadata": {"tracebloc": meta} if meta else {}, + "execution_count": None, + "outputs": [], + "source": source, + } + ) + _save(work, name, nb) + + +def _table(work, fn): + p = os.path.join(work, TEMPLATES, "families.json") + with open(p) as fh: + doc = json.load(fh) + fn(doc) + with open(p, "w") as fh: + json.dump(doc, fh, indent=2) + fh.write("\n") + + +def _fam(doc, key): + return next(f for f in doc["families"] if f["key"] == key) + + +# --- the mutations --------------------------------------------------------- +# (rule, description, mutate, expect_in_output) +# +# `expect_in_output` is a substring the failure MUST name. It is not decoration: +# asserting only "exited non-zero" would let a mutation pass because some other +# rule happened to fire, which is how a deleted rule hides. + +MUTATIONS = [ + ("1", "cycles*epochs over the 20-pass ceiling", + lambda w: _table(w, lambda d: _fam(d, "vision_from_scratch").__setitem__("cycles", 21)), + "> 20"), + ("2", "epochs>1 with a non-drift-correcting strategy", + lambda w: _table(w, lambda d: _fam(d, "nlp_finetune").__setitem__("epochs", 2)), + "no drift correction"), + ("3a", "notebook cycles edited, table not", + lambda w: _sub(w, "tabular_timeseries.ipynb", "training.cycles(15)", "training.cycles(3)"), + "families.json says"), + ("3b", "the pre-fill set TWICE (Python applies the last)", + lambda w: _append_to_settings(w, "vision_from_scratch.ipynb", "\ntraining.cycles(99)"), + "times"), + ("4", "a category with no family template", + lambda w: _table(w, lambda d: _fam(d, "vision_from_scratch")["categories"].remove("keypoint_detection")), + "no family template"), + ("5", "a template that calls start()", + lambda w: _append_cell(w, "survival.ipynb", ["training.start()"]), + "Start is a button"), + ("6", "a dead setter reintroduced", + lambda w: _append_to_settings(w, "vision_from_scratch.ipynb", "\ntraining.horizontal_flip(True)"), + "dead setter"), + ("7", "a Colab-specific line", + lambda w: _append_cell(w, "embeddings.ipynb", ["# https://colab.research.google.com/x"]), + "Colab-specific"), + ("8a", "a cell gated on a category outside its family", + lambda w: _append_cell(w, "nlp_finetune.ipynb", ["training.seed(0)"], + {"applies_to": {"category": ["image_classification"]}, "settings_fragment": True}), + "not in this family"), + ("8b", "a cell gated on a framework the family does not ship", + lambda w: _append_cell(w, "nlp_finetune.ipynb", ["training.seed(0)"], + {"applies_to": {"framework": ["sklearn"]}, "settings_fragment": True}), + "not in this family"), + ("9a", "a settings_fragment whose applies_to is {} (gates nothing)", + lambda w: _append_cell(w, "vision_from_scratch.ipynb", ['training.optimizer("adam")'], + {"applies_to": {}, "settings_fragment": True}), + "gates nothing"), + ("9c", "a gate key present with an EMPTY list β€” renders for no pair", + lambda w: _empty_gate(w), + "EMPTY"), + # The discriminator for the SHARED settings-cell predicate. A decoy cell that + # merely mentions the setter must not become the mutation target; under the legacy + # predicate it did, and this case went quiet (Bugbot). + ("3i", "a decoy cell mentioning the setter, then the real pre-fill edited", + lambda w: _decoy_then_append(w, "vision_from_scratch.ipynb", "\ntraining.cycles(99)"), + "times"), + ("3c", "a second cycles() written in a form the narrow pattern missed", + lambda w: _append_to_settings(w, "vision_from_scratch.ipynb", "\ntraining . cycles (99)"), + "times"), + ("3d", "the real cycles() COMMENTED OUT, comment left in place", + lambda w: _sub(w, "vision_from_scratch.ipynb", + "training.cycles(20)", "# training.cycles(20)"), + # Rule 3 is per-pair now, so a commented-out pre-fill reads as ZERO calls + # for that pair rather than "never calls" for the file. Updated to the + # message the rule actually emits, not loosened to match anything. + "training.cycles() 0 times"), + ("3f", "a family GAINS a framework with its own gated federation fragment", + lambda w: _add_framework(w), + None), # must PASS: the old whole-file rule 3 blocked this outright + # SYNTHETIC id on purpose: test data should not carry a real private + # tracker reference just to prove the rule fires. + ("17a", "a cross-repo reference in a peer-rendered cell", + lambda w: _append_cell(w, "embeddings.ipynb", ["# see example-repo#4242 for context"]), + "internal reference"), + ("17b", "an RFC id in a peer-rendered cell", + lambda w: _append_cell(w, "embeddings.ipynb", ["# per RFC-9999"]), + "internal reference"), + ("17c", "a non-public internal host", + lambda w: _append_cell(w, "embeddings.ipynb", ["# https://internal.tracebloc.io/x"]), + "internal reference"), + ("17d", "the PUBLIC docs host must NOT be flagged", + lambda w: _append_cell(w, "embeddings.ipynb", ["# https://docs.tracebloc.io/x"]), + None), + # 17e/17f pin the ALLOWED_DOC_URL strip that lets the real anchors through. + # 17e is the case the strip exists for and 17d does not cover: a docs URL + # whose fragment is a bare NUMBER (`#1-optimizer`), which INTERNAL_REF's + # `#N` branch would otherwise read as a cross-reference. Delete the + # strip and this legitimate link turns red β€” every template carries these. + ("17e", "a docs URL with a NUMERIC fragment must NOT be flagged", + lambda w: _append_cell(w, "embeddings.ipynb", + ["# https://docs.tracebloc.io/join-use-case/hyperparameters#1-optimizer"]), + None), + # 17f is the other edge: the strip must remove ONLY the allowed URL, not + # swallow a real `#N` sitting beside one. A too-greedy strip would + # pass this tree and let an actual leak render to a peer. + ("17f", "a #N beside a docs URL is still flagged", + lambda w: _append_cell(w, "embeddings.ipynb", + ["# https://docs.tracebloc.io/join-use-case/hyperparameters#1-optimizer" + " β€” but example-repo#4242 is internal"]), + "internal reference"), + ("8d-a", "metadata.tracebloc is a bool on a FRAGMENT cell", + lambda w: _nondict_tracebloc(w, fragment=True), + "not an object"), + ("8d-b", "metadata.tracebloc is a bool on a PLAIN code cell", + lambda w: _nondict_tracebloc(w, fragment=False), + "not an object"), + ("8e", "an unknown applies_to key ALONGSIDE a valid one", + lambda w: _typo_alongside(w), + "applies_to keys"), + ("3g", "aggregation_strategy flipped in the TABLE only", + lambda w: _table(w, lambda d: _fam(d, "tabular_timeseries").__setitem__("aggregation_strategy", "fedprox")), + "aggregation_strategy"), + ("3h", "cycles(99) hidden in a FLAG-gated fragment (flags-on only)", + lambda w: _append_cell(w, "nlp_finetune.ipynb", ["training.cycles(99)"], + {"applies_to": {"dataset_flag": "allow_feature_modification"}, + "settings_fragment": True}), + "flags on"), + ("3e", "a SECOND settings cell inserted BEFORE the real one", + lambda w: _second_settings_cell(w), + "settings cells"), + ("9d", "a gate that admits every render the family produces", + lambda w: _append_cell(w, "vision_from_scratch.ipynb", ["training.seed(0)"], + {"applies_to": {"framework": ["pytorch"]}, "settings_fragment": True}), + "admits EVERY render"), + # `expect=None` β€” must STAY OK. The same gate PLUS a dataset_flag, which render() + # drops on the flags-off pass, so it is not a no-op. Rule 9d walked only + # (category, framework) and called this dead (Bugbot), which is the defect rule 9d + # was itself created to fix, one axis along. + ("9d-b", "an all-framework gate that ALSO sets dataset_flag β€” render drops it", + lambda w: _append_cell(w, "vision_from_scratch.ipynb", ["training.seed(0)"], + {"applies_to": {"framework": ["pytorch"], + "dataset_flag": "allow_feature_modification"}, + "settings_fragment": True}), + None), + ("8f-a", "a gate axis that is null (used to TypeError out of gate_audience)", + lambda w: _malformed_axis(w, None), + "not list"), + ("8f-b", "a gate axis that is a bare STRING (used to iterate its characters)", + lambda w: _malformed_axis(w, "vision"), + "not list"), + ("8f-c", "a null dataset_flag (its shape was unpinned by 8f-a/8f-b alone)", + lambda w: _malformed_axis(w, None, axis="dataset_flag"), + "not str"), + ("8f-d", "a framework axis that is a bare STRING", + lambda w: _malformed_axis(w, "pytorch", axis="framework"), + "not list"), + ("8c", "applies_to that is a LIST, not an object (used to AttributeError)", + lambda w: _nondict_applies(w), + "not an object"), + ("15b", "a loss fragment gated on category: [] β€” offered to NOBODY", + lambda w: _empty_gated_loss(w), + "custom loss"), + ("13e", "a recorded preprocessing pre-fill changed (handle_missing_values)", + lambda w: _sub(w, "tabular_timeseries.ipynb", + "training.handle_missing_values(True)", + "training.handle_missing_values(False)"), + "not backed by a COMPLETED"), + ("13f", "a recorded preprocessing pre-fill changed (encoding_strategy)", + lambda w: _sub(w, "tabular_timeseries.ipynb", + 'training.encoding_strategy("label")', + 'training.encoding_strategy("onehot")'), + "not backed by a COMPLETED"), + # NO 13g for `scaler`. Deliberately absent, with the reason, because a + # mutation that cannot realise a wrong answer is worse than none: + # * survival sets a LITERAL scaler, but survival is declared + # `unverified` in the table, so rule 13 correctly reports nothing + # for it and the mutation would always "pass"; + # * tabular_timeseries sets `{{ scaler }}`, a context-derived + # placeholder rule 13 skips by design. + # So the `scaler -> tabular_scaler` map entry is currently unexercisable. + # It is kept because it is correct the moment survival gains a COMPLETED + # run, and this comment is here so nobody reads its absence as coverage. + ("9b", "a typo'd dataset_flag", + lambda w: _retag_flag(w, "tabular_timeseries.ipynb", "allow_feature_modificaton"), + "not a flag the platform has"), + ("10", "a half-written placeholder, as an f-string produces", + lambda w: _sub(w, "embeddings.ipynb", "{{ sequence_length }}", "{ sequence_length }"), + "half-written placeholder"), + ("11", "a setter offered to a category that refuses it", + lambda w: _append_to_settings(w, "vision_from_scratch.ipynb", "\ntraining.enable_lora(True)"), + "which refuse it"), + ("12a", "a break reachable only with dataset flags OFF", + lambda w: _sub(w, "tabular_timeseries.ipynb", 'training.optimizer("sgd")', 'training.optimizer("sgd"'), + "flags off"), + ("12b", "a break reachable only with dataset flags ON", + lambda w: _break_flag_fragment(w), + "flags on"), + ("13a", "evidence: callbacks blanked", + lambda w: _evidence(w, lambda d: d["runs"]["vision"]["rounds"]["2_full_settings_cell"].__setitem__("callbacks", "[]")), + "not backed by a COMPLETED"), + ("13b", "evidence: the experiment id removed", + lambda w: _evidence(w, lambda d: d["runs"]["vision"]["rounds"]["2_full_settings_cell"].pop("experiment")), + "not backed by a COMPLETED"), + ("13c", "a pre-fill changed in BOTH table and notebook, evidence untouched", + lambda w: (_table(w, lambda d: _fam(d, "vision_from_scratch").__setitem__("aggregation_strategy", "fedprox")), + _sub(w, "vision_from_scratch.ipynb", 'aggregation_strategy("fedavg")', 'aggregation_strategy("fedprox")')), + "not backed by a COMPLETED"), + ("13d", "optimizer changed in the settings cell only", + lambda w: _sub(w, "vision_from_scratch.ipynb", 'training.optimizer("sgd")', 'training.optimizer("adamw")'), + "not backed by a COMPLETED"), + # The two forms the discovery scan and `called_with_all` used to disagree about + # (@LukasWodka in review). Both mutate the SETTINGS CELL, so if rule 13 stops + # comparing a live setter to the record, or starts comparing a commented one it + # cannot read, exactly one of these goes quiet. + ("13x-a", "a live setter written with spaces around the dot β€” Start applies it", + lambda w: _sub(w, "vision_from_scratch.ipynb", 'training.optimizer("sgd")', + 'training . optimizer("adamw")'), + "not backed by a COMPLETED"), + # `expect=None` β€” this one asserts the checker STAYS OK, like 17d. + # + # An UNMAPPED setter, commented out. The discovery scan used to read the RAW source, + # so it found this comment, looked the name up in EVIDENCE_FIELD_MAP, missed, and + # reported "not in EVIDENCE_FIELD_MAP, so nothing says whether the run verified it" + # about a line Python never executes. Scanning `code_only` there makes it invisible, + # which is correct: rule 13 asks whether a pre-fill is APPLIED. + # + # An earlier version of this case commented out a MAPPED setter and also changed the + # live value, and it was a FAKE PROOF β€” caught by the value mismatch even with the + # old two-pattern code, so it said nothing about the comment behaviour. Verified by + # reverting: this form goes from a spurious failure to OK, the old form was caught + # either way. + ("13x-b", "an UNMAPPED setter, commented out β€” must not be read as a live call", + lambda w: _sub(w, "vision_from_scratch.ipynb", 'training.optimizer("sgd")', + 'training.optimizer("sgd")\n # training.warmup_ratio(0.1)'), + None), + ("14", "cycles offered to a single-pass framework", + lambda w: _widen_gate(w), + "force both to 1"), + ("15", "the custom-loss offering removed from a family that accepts one", + lambda w: _strip_loss(w, "survival.ipynb"), + "custom loss"), + ("14b-a", "the SDK version floor removed from families.json", + lambda w: _table(w, lambda d: d.pop("sdk_version_floor")), + "no `sdk_version_floor`"), + # Needs an importable SDK: with none, rule 14b legitimately skips the + # cross-check, so drift is undetectable and this mutation would report a + # false NOT CAUGHT. Marked NEEDS_SDK rather than dropped, and SKIPPED + # LOUDLY when unavailable β€” a mutation quietly not run is the same defect + # as a rule quietly not checked. CI installs the SDK, so it runs there. + ("14b-b!", "single_pass_frameworks drifted from the SDK's own set", + lambda w: _table(w, lambda d: d.__setitem__("single_pass_frameworks", ["sklearn"])), + "has _SURVIVAL_FRAMEWORKS"), + ("16", "a setter in a standalone cell, inert under Start", + lambda w: _append_cell(w, "embeddings.ipynb", ['training.optimizer("adam")']), + "outside the settings cell"), +] + + +def _decoy_then_append(work, name, text): + """Insert a decoy cell that MENTIONS the settings setter, then mutate the real one. + + The discriminator for using the checker's own `is_settings_cell`. Under the legacy + predicate ("`training.experiment_name` in any code cell") the decoy is found FIRST, + so `_append_to_settings` edits a cell the rule under test never reads and the + mutation goes quiet -- a mutation that cannot realise a wrong answer. Under the + shared predicate the decoy has no SETTINGS_MARKER, so the real settings cell is + still the target. + """ + nb = _load(work, name) + settings_at = next( + i for i, c in enumerate(nb["cells"]) if _checker().is_settings_cell(c) + ) + nb["cells"].insert( + settings_at, + { + "cell_type": "code", + "metadata": {}, + "execution_count": None, + "outputs": [], + "source": ["# prose about training.experiment_name, not a settings cell\n"], + }, + ) + _save(work, name, nb) + _append_to_settings(work, name, text) + + +def _append_to_settings(work, name, text): + nb = _load(work, name) + _settings_cell(nb)["source"].append(text) + _save(work, name, nb) + + +def _evidence(work, fn): + p = os.path.join(work, TEMPLATES, "verification-dev.json") + with open(p) as fh: + doc = json.load(fh) + fn(doc) + with open(p, "w") as fh: + json.dump(doc, fh, indent=2) + + +def _retag_flag(work, name, value): + nb = _load(work, name) + for cell in nb["cells"]: + applies = ((cell.get("metadata") or {}).get("tracebloc") or {}).get("applies_to") or {} + if "dataset_flag" in applies: + applies["dataset_flag"] = value + _save(work, name, nb) + return + raise AssertionError("no dataset_flag gate to retag") + + +def _break_flag_fragment(work): + nb = _load(work, "tabular_timeseries.ipynb") + for cell in nb["cells"]: + applies = ((cell.get("metadata") or {}).get("tracebloc") or {}).get("applies_to") or {} + if "dataset_flag" in applies: + cell["source"].append("\ntraining.seed(0") + _save(work, "tabular_timeseries.ipynb", nb) + return + raise AssertionError("no flag-gated fragment to break") + + +def _add_framework(work): + """Give tabular a second framework with its own federation fragment. + + Expected to PASS. The whole-file rule 3 counted one `cycles` call per + framework-gated fragment and failed, so the rule BLOCKED a family from + ever gaining a framework β€” a check preventing a correct change. Kept as a + NEGATIVE mutation so the restructure cannot regress. + """ + _table(work, lambda d: _fam(d, "tabular_timeseries")["frameworks"].append("xgboost")) + _append_cell( + work, "tabular_timeseries.ipynb", + ["training.cycles(15)\n", "training.epochs(1)"], + {"applies_to": {"framework": ["xgboost"]}, "settings_fragment": True}, + ) + + +def _second_settings_cell(work): + nb = _load(work, "vision_from_scratch.ipynb") + marker = ( + "# ======================================================================\n" + "# Settings \u2014 the complete plan for this run, as plain SDK calls.\n" + ) + nb["cells"].insert(3, { + "cell_type": "code", "metadata": {}, "execution_count": None, + "outputs": [], "source": [marker, 'training.experiment_name("x")\n', + "training.cycles(99)"], + }) + _save(work, "vision_from_scratch.ipynb", nb) + + +def _malformed_axis(work, value, axis="category"): + """Put a wrong-typed value on a gate axis, on a real template cell. + + `category: null` used to raise TypeError out of `gate_audience` and rule 9d -- + a crash instead of a rule error, which the checker's own comment says its shared + helpers exist to prevent. `category: "vision"` was worse: a bare string ITERATES, + so the audience became ['v','i','s','i','o','n'] and the cell read as offered to + categories that do not exist -- a silent wrong answer (Bugbot reported the null; + the string turned up while reproducing it). + + PARAMETRISED ON THE AXIS, because the first version of this helper hardcoded + `category` and both its cases varied only that one. @LukasWodka mutation-proved the + gap: change `"dataset_flag": str` to `object` in AXIS_VALUE_TYPES and the clean tree + still passes with both category cases green, so `dataset_flag: null` was a silent OK + again -- the exact false green this fix removes. An instrument has to vary along + every axis it claims to cover. + """ + nb = _load(work, "vision_from_scratch.ipynb") + cell = nb["cells"][0] + cell.setdefault("metadata", {}).setdefault("tracebloc", {}) + cell["metadata"]["tracebloc"]["applies_to"] = {axis: value} + _save(work, "vision_from_scratch.ipynb", nb) + + +def _nondict_tracebloc(work, fragment): + """Set metadata.tracebloc to a bool, on a fragment or a plain cell. + + Both placements matter: `settings_source()` reached it on one and the + rules 8/9 loop on the other, and the loop's falsy-guard meant a fixed + accessor would have turned the traceback into a SILENTLY UNGATED cell. + """ + nb = _load(work, "tabular_timeseries.ipynb") + for cell in nb["cells"]: + if cell["cell_type"] != "code": + continue + tb = (cell.get("metadata") or {}).get("tracebloc") or {} + if bool(tb.get("settings_fragment")) == fragment: + cell.setdefault("metadata", {})["tracebloc"] = True + _save(work, "tabular_timeseries.ipynb", nb) + return + raise AssertionError(f"no {'fragment' if fragment else 'plain'} cell found") + + +def _typo_alongside(work): + nb = _load(work, "tabular_timeseries.ipynb") + for cell in nb["cells"]: + applies = ((cell.get("metadata") or {}).get("tracebloc") or {}).get("applies_to") + if isinstance(applies, dict) and "framework" in applies: + applies["categor"] = ["tabular_classification"] + _save(work, "tabular_timeseries.ipynb", nb) + return + raise AssertionError("no framework gate to add a typo beside") + + +def _nondict_applies(work): + nb = _load(work, "tabular_timeseries.ipynb") + for cell in nb["cells"]: + tb = (cell.get("metadata") or {}).get("tracebloc") or {} + if "applies_to" in tb: + tb["applies_to"] = ["pytorch"] + _save(work, "tabular_timeseries.ipynb", nb) + return + raise AssertionError("no applies_to to corrupt") + + +def _empty_gated_loss(work): + """Strip vision's loss offering and re-add it gated on an EMPTY category + list. The pod renders that for nobody, so rule 15's coverage must FAIL -- + an earlier fix made the checker read it as offered to the whole family, + which is the pod's meaning inverted.""" + _strip_loss(work, "vision_from_scratch.ipynb") + _append_cell( + work, "vision_from_scratch.ipynb", + ['# training.loss_function({"type": "custom", "value": "loss.py"})'], + {"applies_to": {"category": []}, "settings_fragment": True}, + ) + + +def _empty_gate(work): + nb = _load(work, "tabular_timeseries.ipynb") + for cell in nb["cells"]: + applies = ((cell.get("metadata") or {}).get("tracebloc") or {}).get("applies_to") or {} + if "category" in applies: + applies["category"] = [] + _save(work, "tabular_timeseries.ipynb", nb) + return + raise AssertionError("no category gate to empty") + + +def _widen_gate(work): + nb = _load(work, "tabular_timeseries.ipynb") + for cell in nb["cells"]: + if "training.cycles(15)" in "".join(cell["source"]): + cell["metadata"]["tracebloc"]["applies_to"]["framework"] = ["pytorch", "sklearn"] + _save(work, "tabular_timeseries.ipynb", nb) + return + raise AssertionError("no framework-gated federation fragment") + + +def _strip_loss(work, name): + nb = _load(work, name) + for cell in nb["cells"]: + src = "".join(cell["source"]) + if "training.loss_function" in src: + head = src.partition("# --- Custom loss")[0] + cell["source"] = [f"{line}\n" for line in head.rstrip().split("\n")] + _save(work, name, nb) + return + raise AssertionError("no custom-loss offering to strip") + + +# --- runner ---------------------------------------------------------------- + +SDK_AVAILABLE = ( + subprocess.run( + [sys.executable, "-c", "import tracebloc.training.plan"], + capture_output=True, + ).returncode + == 0 +) + + +def run_checker(work, strict=False): + env = dict(os.environ) + if strict: + env["TRACEBLOC_CHECK_STRICT"] = "1" + else: + env.pop("TRACEBLOC_CHECK_STRICT", None) + proc = subprocess.run( + [sys.executable, os.path.join("scripts", "check_templates.py")], + cwd=work, capture_output=True, text=True, env=env, + ) + return proc.returncode, proc.stdout + proc.stderr + + +def main() -> int: + failures = [] + + with tempfile.TemporaryDirectory() as base: + clean = os.path.join(base, "clean") + shutil.copytree(os.path.join(ROOT, TEMPLATES), os.path.join(clean, TEMPLATES)) + shutil.copytree(os.path.join(ROOT, "scripts"), os.path.join(clean, "scripts")) + + # Control: the tree as committed must PASS. A suite whose baseline is + # already red proves nothing about any mutation. + code, out = run_checker(clean) + if code != 0: + print("CONTROL FAILED β€” the committed tree does not pass:\n" + out) + return 1 + print(f"control: clean tree passes (exit {code})") + + skipped = [] + for rule, desc, mutate, expect in MUTATIONS: + needs_sdk = rule.endswith("!") + if needs_sdk and not SDK_AVAILABLE: + skipped.append(f"rule {rule}: {desc} (needs an importable SDK)") + print(f" rule {rule:<4} SKIPPED {desc} β€” no importable SDK") + continue + work = os.path.join(base, f"m{rule.rstrip('!')}") + shutil.copytree(clean, work) + try: + mutate(work) + except AssertionError as exc: + failures.append(f"rule {rule}: mutation could not be applied β€” {exc}") + print(f" rule {rule:<4} UNAPPLIED {desc}") + continue + code, out = run_checker(work, strict=needs_sdk) + if expect is None: + # A NEGATIVE mutation: a legitimate change the checker must + # ACCEPT. Without these, tightening a rule until it rejects + # everything looks like progress. + if code == 0: + print(f" rule {rule:<4} accepted {desc}") + else: + failures.append( + f"rule {rule}: checker REJECTED a legitimate change " + f"({desc})\n{out.strip()[:400]}" + ) + print(f" rule {rule:<4} FALSE FAIL {desc}") + continue + if code == 0: + failures.append( + f"rule {rule}: checker PASSED a tree mutated to be wrong " + f"({desc})" + ) + print(f" rule {rule:<4} NOT CAUGHT {desc}") + elif expect not in out: + failures.append( + f"rule {rule}: checker failed, but no message named " + f"{expect!r} β€” something else fired ({desc})" + ) + print(f" rule {rule:<4} WRONG RULE {desc}") + else: + print(f" rule {rule:<4} caught {desc}") + + if skipped: + print( + f"\n{len(skipped)} mutation(s) SKIPPED β€” they need an importable " + f"tracebloc, which CI installs:" + ) + for sk in skipped: + print(f" - {sk}") + + if failures: + print(f"\n{len(failures)} mutation(s) not caught:\n") + for f in failures: + print(f" - {f}") + return 1 + print( + f"\nOK β€” {len(MUTATIONS) - len(skipped)} of {len(MUTATIONS)} mutations " + f"run, every one caught by the rule that owns it." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main())