From 9cf2e21013c0715527f746a78bb2387e9ccf4ee9 Mon Sep 17 00:00:00 2001 From: drbh Date: Thu, 26 Mar 2026 11:38:28 -0400 Subject: [PATCH] feat: remove init from python cli --- kernels/src/kernels/cli/__init__.py | 56 ++++--- kernels/src/kernels/cli/init.py | 237 ---------------------------- kernels/tests/test_init.py | 72 --------- 3 files changed, 31 insertions(+), 334 deletions(-) delete mode 100644 kernels/src/kernels/cli/init.py delete mode 100644 kernels/tests/test_init.py diff --git a/kernels/src/kernels/cli/__init__.py b/kernels/src/kernels/cli/__init__.py index 09e8a361..0159660a 100644 --- a/kernels/src/kernels/cli/__init__.py +++ b/kernels/src/kernels/cli/__init__.py @@ -7,7 +7,6 @@ from huggingface_hub import ModelCard, ModelCardData from kernels.cli.doc import generate_readme_for_kernel -from kernels.cli.init import parse_kernel_name, run_init from kernels.cli.kernel_card_utils import ( DESCRIPTION, KERNEL_CARD_TEMPLATE_PATH, @@ -22,7 +21,6 @@ from kernels.compat import tomllib from kernels.lockfile import KernelLock, get_kernel_locks from kernels.utils import ( - KNOWN_BACKENDS, install_kernel, install_kernel_all_variants, ) @@ -78,7 +76,9 @@ def main(): versions_parser.add_argument("repo_id", type=str, help="The kernel repo ID") versions_parser.set_defaults(func=kernel_versions) - upload_parser = subparsers.add_parser("upload", help="Upload kernels to the Hub") + upload_parser = subparsers.add_parser( + "upload", help="(Deprecated) Upload kernels to the Hub. Use `kernel-builder upload` instead." + ) upload_parser.add_argument( "kernel_dir", type=Path, @@ -223,33 +223,14 @@ def main(): init_parser = subparsers.add_parser( "init", - help="Initialize a new kernel project from template", + help="(Removed) Use `kernel-builder init` instead", ) init_parser.add_argument( "kernel_name", - type=parse_kernel_name, + nargs="?", help="Name of the kernel repo (e.g., drbh/my-kernel)", ) - init_parser.add_argument( - "--template-repo", - type=str, - default="kernels-community/template", - help="HuggingFace repo ID for the template", - ) - init_parser.add_argument( - "--backends", - nargs="+", - choices={"all"} | KNOWN_BACKENDS, - default=["metal"] if sys.platform == "darwin" else ["cuda"], - metavar="BACKEND", - help=f"Backends to enable (all, {', '.join(KNOWN_BACKENDS)}). Defaults: cuda on Linux/Windows, metal on macOS.", - ) - init_parser.add_argument( - "--overwrite", - action="store_true", - help="Overwrite existing directory if it exists", - ) - init_parser.set_defaults(func=run_init) + init_parser.set_defaults(func=_init_removed) fill_card_parser = subparsers.add_parser( "fill-card", @@ -330,6 +311,20 @@ def lock_kernels(args): def upload_kernels(args): + import warnings + + warnings.warn( + "`kernels upload` is deprecated and will be removed in version 0.14. " + "Please use `kernel-builder upload` instead.", + DeprecationWarning, + stacklevel=1, + ) + # Also print to stderr for visibility in CLI usage + print( + "Warning: `kernels upload` is deprecated and will be removed in version 0.14.\n" + "Please use `kernel-builder upload` instead.\n", + file=sys.stderr, + ) upload_kernels_dir( Path(args.kernel_dir).resolve(), repo_id=args.repo_id, @@ -372,6 +367,17 @@ def default(self, o): return super().default(o) +def _init_removed(args): + print( + "Error: `kernels init` has been removed.\n\n" + "Please use `kernel-builder init` instead:\n\n" + " kernel-builder init \n\n" + "For more information, see: https://github.com/huggingface/kernels", + file=sys.stderr, + ) + sys.exit(1) + + def check_kernel( *, macos: str, manylinux: str, python_abi: str, repo_id: str, revision: str ): diff --git a/kernels/src/kernels/cli/init.py b/kernels/src/kernels/cli/init.py deleted file mode 100644 index 2a568705..00000000 --- a/kernels/src/kernels/cli/init.py +++ /dev/null @@ -1,237 +0,0 @@ -import argparse -import os -import re -import shutil -import subprocess -import sys -from argparse import Namespace -from pathlib import Path -from typing import NamedTuple - -import tomlkit -from huggingface_hub import snapshot_download -from huggingface_hub.utils import disable_progress_bars - -from kernels.cli.kernel_card_utils import KERNEL_CARD_TEMPLATE_PATH -from kernels.utils import KNOWN_BACKENDS - -SYSTEM_CARD_FILENAME = "CARD.md" - -KERNEL_NAME_PATTERN = re.compile(r"^[a-z][-a-z0-9]*[a-z0-9]$") - - -def validate_kernel_name(name: str) -> None: - """Validate kernel name matches ^[a-z][-a-z0-9]*[a-z0-9]$.""" - if not KERNEL_NAME_PATTERN.match(name): - raise argparse.ArgumentTypeError( - f"Invalid kernel name `{name}`. Name must:\n" - "- Start with a lowercase letter (a-z)\n" - "- Contain only lowercase letters, digits, and dashes\n" - "- End with a lowercase letter or digit\n" - "- Be at least 2 characters long\n" - "Examples: `my-kernel`, `relu2d`, `flash-attention`" - ) - - -def parse_kernel_name(value: str) -> NamedTuple: - parts = value.split("/") - if len(parts) != 2 or not all(parts): # validate format - raise argparse.ArgumentTypeError("must be /") - owner, name = parts - - if "/" in name or "\\" in name: # validate kernel name - raise argparse.ArgumentTypeError("repo name cannot contain path separators") - - # Display name uses dashes (for repo name, directory, build.toml name) - display_name = name.replace("_", "-") - # Normalized name uses underscores (for Python package names) - normalized_name = name.lower().replace("-", "_") - - RepoInfo = NamedTuple( - "RepoInfo", - [("name", str), ("normalized_name", str), ("owner", str), ("repo_id", str)], - ) - return RepoInfo( - name=display_name, - normalized_name=normalized_name, - owner=owner, - repo_id=f"{owner}/{display_name}", - ) - - -def run_init(args: Namespace) -> None: - kernel_name = args.kernel_name.name - normalized_name = args.kernel_name.normalized_name - repo_id = args.kernel_name.repo_id - backends = KNOWN_BACKENDS if "all" in args.backends else set(args.backends) - - # Target directory - target_dir = Path.cwd() / kernel_name - - if args.overwrite: - if target_dir.exists(): - shutil.rmtree(target_dir) - - if target_dir.exists() and any(target_dir.iterdir()): - print( - f"Error: Directory already exists and is not empty: {target_dir}", - file=sys.stderr, - ) - sys.exit(1) - - # Suppress progress bars for cleaner output (files are often cached) - disable_progress_bars() - - print(f"Downloading template from {args.template_repo}...", file=sys.stderr) - template_dir = Path( - snapshot_download(repo_id=args.template_repo, repo_type="model") - ) - _init_from_local_template( - template_dir, target_dir, kernel_name, normalized_name, repo_id - ) - - if backends: - _update_build_backends(target_dir / "build.toml", backends) - - # replacement logic - # - rocm uses cuda source so we need to replace the rocm with cuda - if "rocm" in backends: - backends.remove("rocm") - backends.add("cuda") - - _remove_backend_dirs(target_dir, backends) - - # Generate the card template - _initialize_card(target_dir) - - # Initialize git repo (required for Nix flakes) - subprocess.run(["git", "init"], cwd=target_dir, check=True, capture_output=True) - subprocess.run(["git", "add", "."], cwd=target_dir, check=True, capture_output=True) - - print(f"Initialized kernel project: {target_dir}") - _print_tree(target_dir) - print("\nNext steps:\n") - print(f"cd {kernel_name}") - print("cachix use huggingface") - print("nix run -L --max-jobs 1 --cores 8 .#build-and-copy") - print("uv run example.py") - - -def _initialize_card(target_dir: Path) -> None: - card_path = target_dir / SYSTEM_CARD_FILENAME - shutil.copy(KERNEL_CARD_TEMPLATE_PATH, card_path) - - -def _init_from_local_template( - template_dir: Path, - target_dir: Path, - kernel_name: str, - normalized_name: str, - repo_id: str, -) -> None: - # Placeholder mappings - replacements = { - "__KERNEL_NAME__": kernel_name, - "__KERNEL_NAME_NORMALIZED__": normalized_name, - "__REPO_ID__": repo_id, - } - - # Create target directory - target_dir.mkdir(parents=True, exist_ok=True) - - # Walk template directory - for root, dirs, files in os.walk(template_dir): - # Skip .git directory - if ".git" in dirs: - dirs.remove(".git") - - rel_root = Path(root).relative_to(template_dir) - - # Compute target root with placeholder replacement in path - target_root_str = str(rel_root) - for old, new in replacements.items(): - target_root_str = target_root_str.replace(old, new) - target_root = target_dir / target_root_str - - # Create directories - for dir_name in dirs: - new_dir_name = dir_name - for old, new in replacements.items(): - new_dir_name = new_dir_name.replace(old, new) - (target_root / new_dir_name).mkdir(parents=True, exist_ok=True) - - # Copy and process files - for file_name in files: - - src_path = Path(root) / file_name - - # Replace placeholders in filename - new_file_name = file_name - for old, new in replacements.items(): - new_file_name = new_file_name.replace(old, new) - - dst_path = target_root / new_file_name - - # Read, replace placeholders, write - try: - content = src_path.read_text() - for old, new in replacements.items(): - content = content.replace(old, new) - dst_path.parent.mkdir(parents=True, exist_ok=True) - dst_path.write_text(content) - except UnicodeDecodeError: - # Binary file, just copy - shutil.copy2(src_path, dst_path) - - -def _print_tree(directory: Path, prefix: str = "") -> None: - entries = sorted(directory.iterdir(), key=lambda x: (x.is_file(), x.name)) - entries = [e for e in entries if e.name != ".git"] - - for i, entry in enumerate(entries): - is_last = i == len(entries) - 1 - connector = "└── " if is_last else "├── " - print(f"{prefix}{connector}{entry.name}") - - if entry.is_dir(): - extension = " " if is_last else "│ " - _print_tree(entry, prefix + extension) - - -def _update_build_backends(build_toml_path: Path, backends: set[str]) -> None: - if not build_toml_path.exists(): - return - - with open(build_toml_path, "rb") as f: - build_contents = tomlkit.parse(f.read()) - - # update backends - if "general" not in build_contents: - return - build_contents["general"]["backends"] = list(backends) # type: ignore[index] - - # update kernel sections - if "kernel" in build_contents: - kernel_table = build_contents["kernel"] - remove_kernels = [] - for name, cfg in kernel_table.items(): # type: ignore[union-attr] - if isinstance(cfg, dict) and cfg.get("backend") not in set(backends): - remove_kernels.append(name) - for name in remove_kernels: - del kernel_table[name] # type: ignore[union-attr] - - # write back to file - with open(build_toml_path, "wb") as f: - f.write(tomlkit.dumps(build_contents).encode("utf-8")) - - -def _remove_backend_dirs(target_dir: Path, backends: set[str]) -> None: - keep = set(backends) - known = set(KNOWN_BACKENDS) - for entry in target_dir.iterdir(): - if not entry.is_dir(): - continue - for backend in known - keep: - if entry.name.endswith(f"_{backend}"): - shutil.rmtree(entry) - break diff --git a/kernels/tests/test_init.py b/kernels/tests/test_init.py deleted file mode 100644 index e26efd6d..00000000 --- a/kernels/tests/test_init.py +++ /dev/null @@ -1,72 +0,0 @@ -import argparse -import os -import tempfile -from pathlib import Path - -from kernels.cli.init import SYSTEM_CARD_FILENAME, parse_kernel_name, run_init -from kernels.utils import KNOWN_BACKENDS - - -def e2e_init(backends: list[str]) -> None: - kernel_name = "testuser/test-kernel" - template_repo = "kernels-community/template" - args = argparse.Namespace( - kernel_name=parse_kernel_name(kernel_name), - template_repo=template_repo, - backends=backends, - overwrite=False, - ) - expected_dir_name = "test-kernel" # directory uses original name (hyphenated) - expected_python_name = "test_kernel" # backend dirs use python name (underscored) - expected_backend_dirs = { - Path(f"{expected_python_name}_{backend}") for backend in args.backends - } - - # Replacement logic - # special case for "rocm" backend since it uses "cuda" source - if "rocm" in args.backends: - expected_backend_dirs.remove(Path(f"{expected_python_name}_rocm")) - expected_backend_dirs.add(Path(f"{expected_python_name}_cuda")) - if "all" in args.backends: - expected_backend_dirs = { - Path(f"{expected_python_name}_{backend}") for backend in KNOWN_BACKENDS - } - # special case for "rocm" backend since it uses "cuda" source - expected_backend_dirs.remove(Path(f"{expected_python_name}_rocm")) - expected_backend_dirs.add(Path(f"{expected_python_name}_cuda")) - - # TODO: neuron/npu are not yet supported in the template - expected_backend_dirs.discard(Path(f"{expected_python_name}_neuron")) - expected_backend_dirs.discard(Path(f"{expected_python_name}_npu")) - - with tempfile.TemporaryDirectory() as tmpdir: - cwd = Path.cwd() - os.chdir(tmpdir) - try: - run_init(args) - - # make sure target dir was created - target_dir = Path(tmpdir) / expected_dir_name - if not target_dir.exists(): - raise AssertionError(f"Target directory was not created: {target_dir}") - - # check that expected backend dirs were created - for expected_backend_dir in expected_backend_dirs: - if not target_dir.joinpath(expected_backend_dir).exists(): - raise AssertionError( - f"Expected backend directory was not created: {expected_backend_dir}" - ) - - # check that CARD.md was created - card_path = target_dir / SYSTEM_CARD_FILENAME - if not card_path.exists(): - raise AssertionError(f"Card template was not created: {card_path}") - - finally: - os.chdir(cwd) - - -def test_end_to_end_init() -> None: - e2e_init(backends=["cuda", "rocm"]) - e2e_init(backends=["metal", "cpu"]) - e2e_init(backends=["all"])