From 4d375555df9141bbc4a478421984339ec2fbcc38 Mon Sep 17 00:00:00 2001 From: Vladimir Sitnikov Date: Fri, 24 Apr 2026 14:47:04 +0300 Subject: [PATCH 1/3] Prototype repository-driven OCI package resolution # Conflicts: # src/apm_cli/commands/install.py # src/apm_cli/deps/lockfile.py # src/apm_cli/models/apm_package.py # src/apm_cli/models/dependency/__init__.py --- src/apm_cli/commands/uninstall/engine.py | 21 ++- src/apm_cli/deps/lockfile.py | 47 +++++- src/apm_cli/deps/oci_registry.py | 145 +++++++++++++++++++ src/apm_cli/models/apm_package.py | 75 ++++++++-- src/apm_cli/models/dependency/__init__.py | 2 + src/apm_cli/models/dependency/requirement.py | 119 +++++++++++++++ src/apm_cli/repositories/config.py | 77 ++++++++++ src/apm_cli/repositories/resolver.py | 112 ++++++++++++++ tests/unit/test_repository_resolution.py | 128 ++++++++++++++++ 9 files changed, 705 insertions(+), 21 deletions(-) create mode 100644 src/apm_cli/deps/oci_registry.py create mode 100644 src/apm_cli/models/dependency/requirement.py create mode 100644 src/apm_cli/repositories/config.py create mode 100644 src/apm_cli/repositories/resolver.py create mode 100644 tests/unit/test_repository_resolution.py diff --git a/src/apm_cli/commands/uninstall/engine.py b/src/apm_cli/commands/uninstall/engine.py index 42025b910..0281a2843 100644 --- a/src/apm_cli/commands/uninstall/engine.py +++ b/src/apm_cli/commands/uninstall/engine.py @@ -9,17 +9,22 @@ from ...utils.paths import portable_relpath from ...deps.lockfile import LockFile -from ...models.apm_package import APMPackage, DependencyReference +from ...models.apm_package import APMPackage, DependencyReference, PackageRequirement from ...integration.mcp_integrator import MCPIntegrator def _parse_dependency_entry(dep_entry): """Parse a dependency entry from apm.yml into a DependencyReference.""" - if isinstance(dep_entry, DependencyReference): + if isinstance(dep_entry, (DependencyReference, PackageRequirement)): return dep_entry if isinstance(dep_entry, str): - return DependencyReference.parse(dep_entry) + try: + return PackageRequirement.parse(dep_entry) + except Exception: + return DependencyReference.parse(dep_entry) if isinstance(dep_entry, builtins.dict): + if "name" in dep_entry and "git" not in dep_entry: + return PackageRequirement.from_dict(dep_entry) return DependencyReference.parse_from_dict(dep_entry) raise ValueError(f"Unsupported dependency entry type: {type(dep_entry).__name__}") @@ -70,6 +75,8 @@ def _dry_run_uninstall(packages_to_remove, apm_modules_dir, logger): logger.progress(f" - {pkg} from apm.yml") try: dep_ref = _parse_dependency_entry(pkg) + if not hasattr(dep_ref, "get_install_path"): + continue package_path = dep_ref.get_install_path(apm_modules_dir) except (ValueError, TypeError, AttributeError, KeyError): pkg_str = pkg if isinstance(pkg, str) else str(pkg) @@ -117,6 +124,10 @@ def _remove_packages_from_disk(packages_to_remove, apm_modules_dir, logger): for package in packages_to_remove: try: dep_ref = _parse_dependency_entry(package) + if not hasattr(dep_ref, "get_install_path"): + logger.progress(f"Removed {package} from manifest (OCI dependency)") + removed += 1 + continue package_path = dep_ref.get_install_path(apm_modules_dir) except (PathTraversalError,) as e: logger.error(f"Refusing to remove {package}: {e}") @@ -156,7 +167,7 @@ def _cleanup_transitive_orphans(lockfile, packages_to_remove, apm_modules_dir, a for pkg in packages_to_remove: try: ref = _parse_dependency_entry(pkg) - removed_repo_urls.add(ref.repo_url) + removed_repo_urls.add(getattr(ref, "repo_url", getattr(ref, "package", str(pkg)))) except (ValueError, TypeError, AttributeError, KeyError): removed_repo_urls.add(pkg) @@ -186,7 +197,7 @@ def _cleanup_transitive_orphans(lockfile, packages_to_remove, apm_modules_dir, a ref = _parse_dependency_entry(dep_str) remaining_deps.add(ref.get_unique_key()) except (ValueError, TypeError, AttributeError, KeyError): - remaining_deps.add(dep_str) + remaining_deps.add(str(dep_str)) except Exception: pass diff --git a/src/apm_cli/deps/lockfile.py b/src/apm_cli/deps/lockfile.py index f8b518eb9..61d69e6c7 100644 --- a/src/apm_cli/deps/lockfile.py +++ b/src/apm_cli/deps/lockfile.py @@ -23,7 +23,13 @@ class LockedDependency: repo_url: str host: Optional[str] = None port: Optional[int] = None # Non-standard SSH/HTTPS port (e.g. 7999 for Bitbucket DC) + source_type: Optional[str] = None # e.g. "oci" + repository_name: Optional[str] = None # logical repository chosen during resolution registry_prefix: Optional[str] = None # Registry path prefix, e.g. "artifactory/github" + oci_registry: Optional[str] = None # Registry alias used in apm.yml/config + oci_repository: Optional[str] = None # OCI repository path, e.g. acme/security-pack + oci_tag: Optional[str] = None + oci_digest: Optional[str] = None resolved_commit: Optional[str] = None resolved_ref: Optional[str] = None version: Optional[str] = None @@ -45,6 +51,10 @@ class LockedDependency: def get_unique_key(self) -> str: """Returns unique key for this dependency.""" + if self.source_type == "oci": + registry = self.oci_registry or "default" + repository = self.oci_repository or self.repo_url + return f"oci:{registry}:{repository}" if self.source == "local" and self.local_path: return self.local_path if self.is_virtual and self.virtual_path: @@ -58,8 +68,20 @@ def to_dict(self) -> Dict[str, Any]: result["host"] = self.host if self.port: result["port"] = self.port + if self.source_type: + result["source_type"] = self.source_type + if self.repository_name: + result["repository_name"] = self.repository_name if self.registry_prefix: result["registry_prefix"] = self.registry_prefix + if self.oci_registry: + result["oci_registry"] = self.oci_registry + if self.oci_repository: + result["oci_repository"] = self.oci_repository + if self.oci_tag: + result["oci_tag"] = self.oci_tag + if self.oci_digest: + result["oci_digest"] = self.oci_digest if self.resolved_commit: result["resolved_commit"] = self.resolved_commit if self.resolved_ref: @@ -131,8 +153,14 @@ def from_dict(cls, data: Dict[str, Any]) -> "LockedDependency": return cls( repo_url=data["repo_url"], host=data.get("host"), + source_type=data.get("source_type"), + repository_name=data.get("repository_name"), port=port, registry_prefix=data.get("registry_prefix"), + oci_registry=data.get("oci_registry"), + oci_repository=data.get("oci_repository"), + oci_tag=data.get("oci_tag"), + oci_digest=data.get("oci_digest"), resolved_commit=data.get("resolved_commit"), resolved_ref=data.get("resolved_ref"), version=data.get("version"), @@ -177,6 +205,24 @@ def from_dependency_ref( is set to the URL path prefix (e.g. ``"artifactory/github"``), ensuring correct auth routing on subsequent installs. """ + if getattr(dep_ref, "dependency_kind", None) == "package_requirement": + return cls( + repo_url=dep_ref.repo_url, + host=getattr(dep_ref, "resolved_host", None), + source_type=getattr(dep_ref, "resolved_source_type", None), + repository_name=getattr(dep_ref, "resolved_repository", None), + resolved_commit=resolved_commit, + resolved_ref=getattr(dep_ref, "resolved_ref", None) or dep_ref.reference, + version=dep_ref.reference, + depth=depth, + resolved_by=resolved_by, + is_dev=is_dev, + oci_repository=dep_ref.repo_url if getattr(dep_ref, "resolved_source_type", None) == "oci" else None, + oci_registry=getattr(dep_ref, "resolved_repository", None) if getattr(dep_ref, "resolved_source_type", None) == "oci" else None, + oci_tag=dep_ref.reference if getattr(dep_ref, "resolved_source_type", None) == "oci" else None, + oci_digest=getattr(dep_ref, "resolved_digest", None) if getattr(dep_ref, "resolved_source_type", None) == "oci" else None, + ) + if registry_config is not None: host = registry_config.host registry_prefix = registry_config.prefix @@ -217,7 +263,6 @@ def to_dependency_ref(self) -> DependencyReference: allow_insecure=self.allow_insecure, ) - @dataclass class LockFile: """APM lock file for reproducible dependency resolution.""" diff --git a/src/apm_cli/deps/oci_registry.py b/src/apm_cli/deps/oci_registry.py new file mode 100644 index 000000000..0ff70dd2d --- /dev/null +++ b/src/apm_cli/deps/oci_registry.py @@ -0,0 +1,145 @@ +"""OCI package retrieval for APM dependencies.""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +import tarfile +import tempfile +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Optional + +from ..models.apm_package import ( + APMPackage, + GitReferenceType, + PackageInfo, + PackageType, + ResolvedReference, + validate_apm_package, +) + + +@dataclass(frozen=True) +class OCIPullResult: + """Result of pulling an OCI-backed APM package.""" + + package_path: Path + resolved_reference: str + resolved_digest: Optional[str] = None + + +class OCIRegistryClient: + """Fetches OCI-backed APM package artifacts using the ORAS CLI.""" + + def pull_package( + self, + resolved_reference: str, + target_path: Path, + requirement=None, + ) -> PackageInfo: + """Pull an OCI artifact into *target_path* and validate it as an APM package. + + Expected OCI payload: + - one ``*.tar.gz`` archive + - archive contains raw APM package sources with ``apm.yml`` at the root + or inside a single top-level directory + """ + if target_path.exists() and any(target_path.iterdir()): + shutil.rmtree(target_path, ignore_errors=True) + target_path.mkdir(parents=True, exist_ok=True) + + pull_dir = Path(tempfile.mkdtemp(prefix="apm-oci-pull-")) + try: + self._pull_artifact(resolved_reference, pull_dir) + archive_path = self._locate_package_archive(pull_dir) + package_root = self._extract_package_archive(archive_path, target_path) + finally: + shutil.rmtree(pull_dir, ignore_errors=True) + + validation = validate_apm_package(package_root) + if not validation.is_valid: + issues = ( + "; ".join(getattr(err, "message", str(err)) for err in validation.errors) + if validation.errors else "unknown validation error" + ) + raise RuntimeError(f"OCI artifact {resolved_reference} is not a valid APM package: {issues}") + + package = APMPackage.from_apm_yml(package_root / "apm.yml") + package.package_path = package_root + package.source = resolved_reference + + return PackageInfo( + package=package, + install_path=package_root, + resolved_reference=ResolvedReference( + original_ref=resolved_reference, + ref_type=GitReferenceType.BRANCH, + resolved_commit=None, + ref_name=getattr(requirement, "version", None), + ), + installed_at=datetime.now().isoformat(), + dependency_ref=requirement, + package_type=validation.package_type or PackageType.APM_PACKAGE, + ) + + def _pull_artifact(self, resolved_reference: str, output_dir: Path) -> None: + """Pull the OCI artifact to *output_dir* using ORAS.""" + try: + proc = subprocess.run( + ["oras", "pull", resolved_reference, "--output", str(output_dir)], + capture_output=True, + text=True, + check=False, + ) + except FileNotFoundError as exc: + raise RuntimeError( + "OCI support requires the 'oras' CLI to be installed and available on PATH." + ) from exc + + if proc.returncode != 0: + stderr = (proc.stderr or proc.stdout or "").strip() + raise RuntimeError(f"Failed to pull OCI artifact {resolved_reference}: {stderr}") + + @staticmethod + def _locate_package_archive(output_dir: Path) -> Path: + """Find the pulled raw-package archive in ORAS output.""" + archives = sorted(output_dir.rglob("*.tar.gz")) + if len(archives) == 1: + return archives[0] + if not archives: + raise RuntimeError( + "OCI artifact did not contain a raw APM package archive (*.tar.gz)." + ) + raise RuntimeError( + "OCI artifact contained multiple package archives; expected exactly one *.tar.gz." + ) + + @staticmethod + def _extract_package_archive(archive_path: Path, target_path: Path) -> Path: + """Extract a raw-package archive and return the package root.""" + with tarfile.open(archive_path, "r:gz") as tar: + for member in tar.getmembers(): + if member.name.startswith("/") or ".." in Path(member.name).parts: + raise RuntimeError( + f"Refusing to extract unsafe archive entry: {member.name}" + ) + if member.issym() or member.islnk(): + raise RuntimeError( + f"Refusing to extract symlink/hardlink from OCI archive: {member.name}" + ) + if sys.version_info >= (3, 12): + tar.extractall(target_path, filter="data") + else: + tar.extractall(target_path) # noqa: S202 + + if (target_path / "apm.yml").exists(): + return target_path + for child in sorted(target_path.iterdir()): + if child.is_dir() and (child / "apm.yml").exists(): + return child + raise RuntimeError( + "Extracted OCI archive did not contain an APM package root with apm.yml." + ) diff --git a/src/apm_cli/models/apm_package.py b/src/apm_cli/models/apm_package.py index 7e84c6069..0a096ed8c 100644 --- a/src/apm_cli/models/apm_package.py +++ b/src/apm_cli/models/apm_package.py @@ -16,6 +16,7 @@ GitReferenceType, MCPDependency, RemoteRef, + PackageRequirement, ResolvedReference, parse_git_reference, ) @@ -35,6 +36,7 @@ "GitReferenceType", "MCPDependency", "RemoteRef", + "PackageRequirement", "ResolvedReference", "parse_git_reference", # Backward-compatible re-exports from .validation @@ -69,8 +71,8 @@ class APMPackage: license: Optional[str] = None source: Optional[str] = None # Source location (for dependencies) resolved_commit: Optional[str] = None # Resolved commit SHA (for dependencies) - dependencies: Optional[Dict[str, List[Union[DependencyReference, str, dict]]]] = None # Mixed types for APM/MCP/inline - dev_dependencies: Optional[Dict[str, List[Union[DependencyReference, str, dict]]]] = None + dependencies: Optional[Dict[str, List[Union[DependencyReference, PackageRequirement, str, dict]]]] = None # Mixed types for APM/MCP/inline + dev_dependencies: Optional[Dict[str, List[Union[DependencyReference, PackageRequirement, str, dict]]]] = None scripts: Optional[Dict[str, str]] = None package_path: Optional[Path] = None # Local path to package target: Optional[Union[str, List[str]]] = None # Target agent(s): single string or list (applies to compile and install) @@ -127,12 +129,22 @@ def from_apm_yml(cls, apm_yml_path: Path) -> "APMPackage": for dep_entry in dep_list: if isinstance(dep_entry, str): try: - parsed_deps.append(DependencyReference.parse(dep_entry)) + if _should_parse_as_requirement(dep_entry): + parsed_deps.append(PackageRequirement.parse(dep_entry)) + else: + parsed_deps.append(DependencyReference.parse(dep_entry)) except ValueError as e: raise ValueError(f"Invalid APM dependency '{dep_entry}': {e}") elif isinstance(dep_entry, dict): try: - parsed_deps.append(DependencyReference.parse_from_dict(dep_entry)) + if "git" in dep_entry or ( + "path" in dep_entry and "name" not in dep_entry + ): + parsed_deps.append(DependencyReference.parse_from_dict(dep_entry)) + elif "name" in dep_entry: + parsed_deps.append(PackageRequirement.from_dict(dep_entry)) + else: + parsed_deps.append(DependencyReference.parse_from_dict(dep_entry)) except ValueError as e: raise ValueError(f"Invalid APM dependency {dep_entry}: {e}") dependencies[dep_type] = parsed_deps @@ -162,12 +174,22 @@ def from_apm_yml(cls, apm_yml_path: Path) -> "APMPackage": for dep_entry in dep_list: if isinstance(dep_entry, str): try: - parsed_deps.append(DependencyReference.parse(dep_entry)) + if _should_parse_as_requirement(dep_entry): + parsed_deps.append(PackageRequirement.parse(dep_entry)) + else: + parsed_deps.append(DependencyReference.parse(dep_entry)) except ValueError as e: raise ValueError(f"Invalid dev APM dependency '{dep_entry}': {e}") elif isinstance(dep_entry, dict): try: - parsed_deps.append(DependencyReference.parse_from_dict(dep_entry)) + if "git" in dep_entry or ( + "path" in dep_entry and "name" not in dep_entry + ): + parsed_deps.append(DependencyReference.parse_from_dict(dep_entry)) + elif "name" in dep_entry: + parsed_deps.append(PackageRequirement.from_dict(dep_entry)) + else: + parsed_deps.append(DependencyReference.parse_from_dict(dep_entry)) except ValueError as e: raise ValueError(f"Invalid dev APM dependency {dep_entry}: {e}") dev_dependencies[dep_type] = parsed_deps @@ -212,13 +234,15 @@ def from_apm_yml(cls, apm_yml_path: Path) -> "APMPackage": _apm_yml_cache[resolved] = result return result - def get_apm_dependencies(self) -> List[DependencyReference]: - """Get list of APM dependencies.""" + def get_apm_dependencies(self) -> List[Union[DependencyReference, "PackageRequirement"]]: + """Get list of manifest APM dependencies.""" if not self.dependencies or 'apm' not in self.dependencies: return [] - # Filter to only return DependencyReference objects - return [dep for dep in self.dependencies['apm'] if isinstance(dep, DependencyReference)] - + return [ + dep for dep in self.dependencies['apm'] + if isinstance(dep, (DependencyReference, PackageRequirement)) + ] + def get_mcp_dependencies(self) -> List["MCPDependency"]: """Get list of MCP dependencies.""" if not self.dependencies or 'mcp' not in self.dependencies: @@ -227,14 +251,17 @@ def get_mcp_dependencies(self) -> List["MCPDependency"]: if isinstance(dep, MCPDependency)] def has_apm_dependencies(self) -> bool: - """Check if this package has APM dependencies.""" + """Check if this package has any APM dependencies.""" return bool(self.get_apm_dependencies()) - def get_dev_apm_dependencies(self) -> List[DependencyReference]: + def get_dev_apm_dependencies(self) -> List[Union[DependencyReference, "PackageRequirement"]]: """Get list of dev APM dependencies.""" if not self.dev_dependencies or 'apm' not in self.dev_dependencies: return [] - return [dep for dep in self.dev_dependencies['apm'] if isinstance(dep, DependencyReference)] + return [ + dep for dep in self.dev_dependencies['apm'] + if isinstance(dep, (DependencyReference, PackageRequirement)) + ] def get_dev_mcp_dependencies(self) -> List["MCPDependency"]: """Get list of dev MCP dependencies.""" @@ -288,4 +315,22 @@ def has_primitives(self) -> bool: if hooks_dir.exists() and any(hooks_dir.glob("*.json")): return True - return False \ No newline at end of file + return False + + +def _should_parse_as_requirement(dep_entry: str) -> bool: + """Return True when a string dependency should be resolved via repositories.""" + value = dep_entry.strip() + if not value: + return False + if DependencyReference.is_local_path(value): + return False + if "://" in value or value.startswith("git@") or value.startswith("//"): + return False + + raw = value.split("#", 1)[0] + first_segment = raw.split("/", 1)[0] + # Explicit host-qualified refs stay on the direct VCS path. + if "." in first_segment: + return False + return True diff --git a/src/apm_cli/models/dependency/__init__.py b/src/apm_cli/models/dependency/__init__.py index 538ac9aae..df12deb58 100644 --- a/src/apm_cli/models/dependency/__init__.py +++ b/src/apm_cli/models/dependency/__init__.py @@ -1,6 +1,7 @@ """Dependency reference models and Git reference utilities.""" from .mcp import MCPDependency +from .requirement import PackageRequirement from .reference import DependencyReference from .types import GitReferenceType, RemoteRef, ResolvedReference, VirtualPackageType, parse_git_reference @@ -9,6 +10,7 @@ "GitReferenceType", "MCPDependency", "RemoteRef", + "PackageRequirement", "ResolvedReference", "VirtualPackageType", "parse_git_reference", diff --git a/src/apm_cli/models/dependency/requirement.py b/src/apm_cli/models/dependency/requirement.py new file mode 100644 index 000000000..f7b4a2dc6 --- /dev/null +++ b/src/apm_cli/models/dependency/requirement.py @@ -0,0 +1,119 @@ +"""Transport-agnostic APM package requirement model.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from ...utils.path_security import ensure_path_within, validate_path_segments + + +@dataclass +class PackageRequirement: + """Logical APM package requirement resolved through configured repositories.""" + + name: str + version: Optional[str] = None + repository: Optional[str] = None + alias: Optional[str] = None + + # Compatibility surface for existing install/lockfile code + is_local: bool = False + local_path: Optional[str] = None + is_virtual: bool = False + virtual_path: Optional[str] = None + host: Optional[str] = None + + # Populated by repository resolution + resolved_source_type: Optional[str] = None + resolved_repository: Optional[str] = None + resolved_ref: Optional[str] = None + resolved_digest: Optional[str] = None + resolved_host: Optional[str] = None + + dependency_kind: str = "package_requirement" + + @property + def repo_url(self) -> str: + """Compatibility alias used throughout the existing codebase.""" + return self.name + + @property + def reference(self) -> Optional[str]: + """Compatibility alias for version/ref-style pins.""" + return self.version + + @classmethod + def parse(cls, raw: str) -> "PackageRequirement": + """Parse a shorthand logical requirement like ``owner/repo#v1.2.0``.""" + if not isinstance(raw, str) or not raw.strip(): + raise ValueError("Package requirement must be a non-empty string") + value = raw.strip() + name, version = value, None + if "#" in value: + name, version = value.rsplit("#", 1) + version = version.strip() or None + name = name.strip().strip("/") + if not name or "/" not in name: + raise ValueError("Package requirement must be in 'owner/repo' form") + validate_path_segments(name, context="package name") + return cls(name=name, version=version) + + @classmethod + def from_dict(cls, entry: dict) -> "PackageRequirement": + """Parse object-style logical dependency entry.""" + name = entry.get("name") + if not isinstance(name, str) or not name.strip(): + raise ValueError("Package dependency 'name' must be a non-empty string") + validate_path_segments(name.strip().strip("/"), context="package name") + + version = entry.get("version") + if version is not None: + if not isinstance(version, str) or not version.strip(): + raise ValueError("Package dependency 'version' must be a non-empty string") + version = version.strip() + + repository = entry.get("repository") + if repository is not None: + if not isinstance(repository, str) or not repository.strip(): + raise ValueError("Package dependency 'repository' must be a non-empty string") + repository = repository.strip() + + alias = entry.get("alias") + if alias is not None: + if not isinstance(alias, str) or not alias.strip(): + raise ValueError("Package dependency 'alias' must be a non-empty string") + alias = alias.strip() + + return cls( + name=name.strip().strip("/"), + version=version, + repository=repository, + alias=alias, + ) + + def get_unique_key(self) -> str: + """Return a stable identity for duplicate detection and locking.""" + return self.name + + def get_identity(self) -> str: + """Return logical package identity without resolved transport details.""" + return self.name + + def get_display_name(self) -> str: + """Return a human-readable package name.""" + return self.alias or self.name + + def get_install_path(self, apm_modules_dir: Path) -> Path: + """Compute install path from the logical package name.""" + validate_path_segments(self.name, context="package name") + result = apm_modules_dir.joinpath(*self.name.split("/")) + ensure_path_within(result, apm_modules_dir) + return result + + def __str__(self) -> str: + result = self.name + if self.version: + result += f"#{self.version}" + return result diff --git a/src/apm_cli/repositories/config.py b/src/apm_cli/repositories/config.py new file mode 100644 index 000000000..d77bf82ea --- /dev/null +++ b/src/apm_cli/repositories/config.py @@ -0,0 +1,77 @@ +"""Repository configuration for transport-agnostic package resolution.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import List + +import yaml + + +@dataclass(frozen=True) +class RepositoryDefinition: + """A configured package repository backend.""" + + name: str + type: str + base: str + priority: int = 0 + + +DEFAULT_REPOSITORIES: List[RepositoryDefinition] = [ + RepositoryDefinition(name="github", type="git", base="https://github.com", priority=100), + RepositoryDefinition(name="gitlab", type="git", base="https://gitlab.com", priority=90), + RepositoryDefinition(name="ghcr", type="oci", base="ghcr.io/apm", priority=80), +] + + +def repositories_config_path() -> Path: + """Return the repository config path.""" + return Path.home() / ".apm" / "repositories.yml" + + +def load_repositories() -> List[RepositoryDefinition]: + """Load configured repositories, falling back to built-in defaults.""" + path = repositories_config_path() + if not path.exists(): + return list(DEFAULT_REPOSITORIES) + + try: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except Exception: + return list(DEFAULT_REPOSITORIES) + + entries = raw.get("repositories") + if not isinstance(entries, list): + return list(DEFAULT_REPOSITORIES) + + repositories: List[RepositoryDefinition] = [] + for entry in entries: + if not isinstance(entry, dict): + continue + name = entry.get("name") + type_ = entry.get("type") + base = entry.get("base") + priority = entry.get("priority", 0) + if not isinstance(name, str) or not name.strip(): + continue + if not isinstance(type_, str) or type_ not in ("git", "oci"): + continue + if not isinstance(base, str) or not base.strip(): + continue + if not isinstance(priority, int): + priority = 0 + repositories.append( + RepositoryDefinition( + name=name.strip(), + type=type_, + base=base.strip().rstrip("/"), + priority=priority, + ) + ) + + if not repositories: + return list(DEFAULT_REPOSITORIES) + + return sorted(repositories, key=lambda repo: repo.priority, reverse=True) diff --git a/src/apm_cli/repositories/resolver.py b/src/apm_cli/repositories/resolver.py new file mode 100644 index 000000000..219ddbf0f --- /dev/null +++ b/src/apm_cli/repositories/resolver.py @@ -0,0 +1,112 @@ +"""Repository-driven resolution for logical APM package requirements.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from ..deps.github_downloader import GitHubPackageDownloader +from ..deps.oci_registry import OCIRegistryClient +from ..models.dependency import DependencyReference, PackageRequirement +from ..repositories.config import RepositoryDefinition, load_repositories + + +@dataclass(frozen=True) +class ResolvedArtifact: + """A concrete artifact selected for a logical package requirement.""" + + repository_name: str + source_type: str + locator: str + host: Optional[str] = None + + +class ArtifactResolver: + """Resolve logical package requirements through configured repositories.""" + + def __init__( + self, + git_downloader: GitHubPackageDownloader, + oci_client: OCIRegistryClient, + ): + self.git_downloader = git_downloader + self.oci_client = oci_client + + def fetch_requirement(self, requirement: PackageRequirement, target_path: Path): + """Resolve and fetch a package requirement into *target_path*.""" + last_error: Optional[Exception] = None + for repository in load_repositories(): + if requirement.repository and repository.name != requirement.repository: + continue + try: + artifact = self._resolve_artifact(requirement, repository) + return self._fetch_artifact(requirement, artifact, target_path) + except Exception as exc: + last_error = exc + continue + if last_error is not None: + raise RuntimeError( + f"Failed to resolve {requirement} from configured repositories: {last_error}" + ) + if requirement.repository: + raise RuntimeError( + f"Requested repository '{requirement.repository}' is not configured for {requirement}" + ) + raise RuntimeError(f"No configured repository could resolve {requirement}") + + def _resolve_artifact( + self, requirement: PackageRequirement, repository: RepositoryDefinition + ) -> ResolvedArtifact: + """Build a concrete locator for a requirement against one repository.""" + if repository.type == "git": + locator = f"{repository.base.rstrip('/')}/{requirement.name}.git" + if requirement.version: + locator += f"#{requirement.version}" + host = locator.split("://", 1)[1].split("/", 1)[0] if "://" in locator else None + return ResolvedArtifact( + repository_name=repository.name, + source_type="git", + locator=locator, + host=host, + ) + + if repository.type == "oci": + if not requirement.version: + raise RuntimeError( + f"OCI repository '{repository.name}' requires a version for {requirement.name}" + ) + locator = f"{repository.base.rstrip('/')}/{requirement.name}:{requirement.version}" + return ResolvedArtifact( + repository_name=repository.name, + source_type="oci", + locator=locator, + ) + + raise RuntimeError(f"Unsupported repository type: {repository.type}") + + def _fetch_artifact( + self, + requirement: PackageRequirement, + artifact: ResolvedArtifact, + target_path: Path, + ): + """Fetch a concrete artifact and annotate the requirement with the result.""" + if artifact.source_type == "git": + dep_ref = DependencyReference.parse(artifact.locator) + package_info = self.git_downloader.download_package(dep_ref, target_path) + requirement.resolved_source_type = "git" + requirement.resolved_repository = artifact.repository_name + requirement.resolved_ref = artifact.locator + requirement.resolved_host = dep_ref.host + return package_info + + if artifact.source_type == "oci": + package_info = self.oci_client.pull_package(artifact.locator, target_path, requirement) + requirement.resolved_source_type = "oci" + requirement.resolved_repository = artifact.repository_name + requirement.resolved_ref = artifact.locator + requirement.resolved_digest = getattr(package_info, "resolved_digest", None) + return package_info + + raise RuntimeError(f"Unsupported artifact type: {artifact.source_type}") diff --git a/tests/unit/test_repository_resolution.py b/tests/unit/test_repository_resolution.py new file mode 100644 index 000000000..65c607923 --- /dev/null +++ b/tests/unit/test_repository_resolution.py @@ -0,0 +1,128 @@ +import tarfile + +import yaml + +from apm_cli.deps.lockfile import LockedDependency +from apm_cli.deps.oci_registry import OCIRegistryClient +from apm_cli.models.apm_package import APMPackage, PackageRequirement +from apm_cli.repositories.config import RepositoryDefinition, load_repositories +from apm_cli.repositories.resolver import ArtifactResolver + + +def test_apm_package_parses_shorthand_as_logical_requirement(tmp_path): + manifest = tmp_path / "apm.yml" + manifest.write_text( + yaml.safe_dump( + { + "name": "demo", + "version": "1.0.0", + "dependencies": { + "apm": [ + "microsoft/apm-standards#v1.2.0", + {"name": "acme/security-pack", "version": "1.0.0"}, + "gitlab.com/group/repo", + ] + }, + } + ), + encoding="utf-8", + ) + + package = APMPackage.from_apm_yml(manifest) + deps = package.get_apm_dependencies() + + assert isinstance(deps[0], PackageRequirement) + assert deps[0].name == "microsoft/apm-standards" + assert deps[0].version == "v1.2.0" + assert isinstance(deps[1], PackageRequirement) + assert deps[1].name == "acme/security-pack" + assert deps[2].host == "gitlab.com" + + +def test_default_repositories_include_git_and_oci(monkeypatch, tmp_path): + monkeypatch.setattr( + "apm_cli.repositories.config.repositories_config_path", + lambda: tmp_path / "missing-repositories.yml", + ) + repos = load_repositories() + names = [repo.name for repo in repos] + assert "github" in names + assert "gitlab" in names + assert "ghcr" in names + + +def test_locked_dependency_tracks_resolved_requirement_metadata(): + req = PackageRequirement(name="acme/security-pack", version="1.0.0") + req.resolved_source_type = "oci" + req.resolved_repository = "ghcr" + req.resolved_ref = "ghcr.io/apm/acme/security-pack:1.0.0" + req.resolved_digest = "sha256:abc" + + locked = LockedDependency.from_dependency_ref(req, None, 1, None) + round_trip = LockedDependency.from_dict(locked.to_dict()) + + assert round_trip.source_type == "oci" + assert round_trip.repository_name == "ghcr" + assert round_trip.oci_repository == "acme/security-pack" + assert round_trip.oci_tag == "1.0.0" + assert round_trip.oci_digest == "sha256:abc" + + +def test_artifact_resolver_annotates_requirement_for_git(monkeypatch, tmp_path): + class StubDownloader: + def download_package(self, dep_ref, target_path): + class Result: + resolved_reference = type( + "Resolved", (), {"ref_name": "v1.2.0", "resolved_commit": "deadbeef"} + )() + + target_path.mkdir(parents=True, exist_ok=True) + (target_path / "apm.yml").write_text("name: pkg\nversion: 1.0.0\n", encoding="utf-8") + return Result() + + class StubOCI: + def pull_package(self, resolved_reference, target_path, requirement=None): + raise AssertionError("OCI backend should not be used in this test") + + resolver = ArtifactResolver(StubDownloader(), StubOCI()) + req = PackageRequirement(name="microsoft/apm-standards", version="v1.2.0", repository="github") + monkeypatch.setattr( + "apm_cli.repositories.resolver.load_repositories", + lambda: [ + RepositoryDefinition(name="github", type="git", base="https://github.com", priority=100) + ], + ) + + resolver.fetch_requirement(req, tmp_path / "pkg") + + assert req.resolved_source_type == "git" + assert req.resolved_repository == "github" + assert req.resolved_ref.endswith("microsoft/apm-standards.git#v1.2.0") + + +def test_oci_registry_client_extracts_raw_package_tarball(tmp_path, monkeypatch): + artifact_dir = tmp_path / "artifact" + artifact_dir.mkdir() + src_dir = tmp_path / "src" + pkg_dir = src_dir / "security-pack" + pkg_dir.mkdir(parents=True) + (pkg_dir / "apm.yml").write_text("name: security-pack\nversion: 1.0.0\n", encoding="utf-8") + (pkg_dir / ".apm").mkdir() + + archive_path = artifact_dir / "security-pack.tar.gz" + with tarfile.open(archive_path, "w:gz") as tar: + tar.add(pkg_dir, arcname=pkg_dir.name) + + def fake_pull(self, resolved_reference, output_dir): + output_dir.mkdir(parents=True, exist_ok=True) + target = output_dir / archive_path.name + target.write_bytes(archive_path.read_bytes()) + + monkeypatch.setattr(OCIRegistryClient, "_pull_artifact", fake_pull) + + client = OCIRegistryClient() + req = PackageRequirement(name="acme/security-pack", version="1.0.0", repository="ghcr") + result = client.pull_package("ghcr.io/apm/acme/security-pack:1.0.0", tmp_path / "install", req) + + assert (result.install_path / "apm.yml").exists() + assert result.package.name == "security-pack" From 6a52d22b9be9659a66b3aabcc9c03c1216b59e7c Mon Sep 17 00:00:00 2001 From: Vladimir Sitnikov Date: Fri, 24 Apr 2026 18:02:21 +0300 Subject: [PATCH 2/3] Add docs for repository-driven dependencies --- docs/src/content/docs/guides/dependencies.md | 78 +++++++++++++++++-- .../content/docs/guides/private-packages.md | 25 +++++- .../content/docs/reference/lockfile-spec.md | 24 +++++- .../content/docs/reference/manifest-schema.md | 59 +++++++++++++- .../.apm/skills/apm-usage/dependencies.md | 62 ++++++++++++++- 5 files changed, 234 insertions(+), 14 deletions(-) diff --git a/docs/src/content/docs/guides/dependencies.md b/docs/src/content/docs/guides/dependencies.md index 2810bb47c..072384d72 100644 --- a/docs/src/content/docs/guides/dependencies.md +++ b/docs/src/content/docs/guides/dependencies.md @@ -8,14 +8,14 @@ Complete guide to APM package dependency management - share and reuse context co ## What Are APM Dependencies? -APM dependencies are git repositories containing `.apm/` directories with context collections (instructions, chatmodes, contexts) and agent workflows (prompts). They enable teams to: +APM dependencies are reusable APM packages that APM can fetch either directly from git or indirectly through configured repositories such as an OCI registry. They enable teams to: - **Share proven workflows** across projects and team members - **Standardize compliance and design patterns** organization-wide - **Build on tested context** instead of starting from scratch - **Maintain consistency** across multiple repositories and teams -APM supports any git-accessible host — GitHub, GitLab, Bitbucket, self-hosted instances, and more. +APM supports direct git hosts such as GitHub, GitLab, Bitbucket, Azure DevOps, and self-hosted servers, plus repository-driven resolution for logical package requirements. ## Dependency Types @@ -81,8 +81,13 @@ name: my-project version: 1.0.0 dependencies: apm: - # GitHub shorthand (default) + # Logical package requirements resolved through configured repositories - microsoft/apm-sample-package#v1.0.0 + - name: acme/security-pack + version: 1.2.0 + repository: ghcr + + # Logical requirement that will usually resolve from the default GitHub repository - github/awesome-copilot/skills/review-and-refactor # Full HTTPS git URL (any host) @@ -116,10 +121,12 @@ dependencies: KB_TOKEN: "${KB_TOKEN}" ``` -APM accepts dependencies in two forms: +APM accepts dependencies in three forms: + +**Logical requirement strings**: +- **Logical package requirement** (`owner/repo` or `owner/repo#v1.2.0`) — resolved through configured repositories -**String format** (simple cases): -- **Shorthand** (`owner/repo`) — defaults to GitHub +**Direct source strings**: - **HTTPS URL** (`https://host/owner/repo.git`) — any git host, whole repo - Custom port: `https://host:8443/owner/repo.git` — port is preserved in clone URLs - **SSH URL** (`git@host:owner/repo.git`) — any git host, whole repo @@ -130,11 +137,19 @@ APM accepts dependencies in two forms: - For nested groups + virtual paths, use the object format below - **Local path** (`./path`, `../path`, `/absolute/path`) — local filesystem package -**Object format** (when you need `path`, `ref`, or `alias` on a git URL): +**Object format**: +- **Logical requirement object** (`name` + optional `version` / `repository` / `alias`) +- **Git object** (`git` + optional `path` / `ref` / `alias`) +- **Local path object** (`path`) ```yaml dependencies: apm: + - name: acme/security-pack + version: 1.2.0 + repository: corp-oci + alias: security-pack + - git: https://gitlab.com/acme/coding-standards.git path: instructions/security # virtual sub-path inside the repo ref: v2.0 # pin to a tag, branch, or commit @@ -174,12 +189,61 @@ transitive host you want to allow. > path: file.prompt.md > ``` +## Repository-Driven Resolution + +Logical requirements keep package identity separate from transport. APM resolves them using repositories configured on the client machine in `~/.apm/repositories.yml`. + +Built-in defaults: + +| Name | Type | Base | Priority | +|------|------|------|----------| +| `github` | `git` | `https://github.com` | `100` | +| `gitlab` | `git` | `https://gitlab.com` | `90` | +| `ghcr` | `oci` | `ghcr.io/apm` | `80` | + +Example override: + +```yaml +repositories: + - name: corp-oci + type: oci + base: registry.example.com/apm + priority: 110 + + - name: github + type: git + base: https://github.com + priority: 100 +``` + +Resolution rules: + +1. If a dependency sets `repository: `, APM only tries that configured repository. +2. Otherwise APM tries repositories in descending `priority` order. +3. The first repository that resolves and fetches the package wins. +4. The resolved transport details are recorded in `apm.lock.yaml`. + +Bare `owner/repo` strings in `apm.yml` are treated as logical requirements. Use explicit git URLs or host-qualified refs such as `gitlab.com/group/repo` when you want to bypass repository resolution and point at a specific source. + +## OCI-backed Packages + +The current OCI prototype supports consuming raw APM packages from OCI registries through configured repositories. + +Expected artifact shape: + +- exactly one `*.tar.gz` file in the OCI artifact +- the archive contains raw APM package sources +- `apm.yml` is at the archive root or under one top-level directory + +This prototype currently uses the `oras` CLI to pull OCI artifacts. Publishing OCI packages and media-type enforcement are not implemented yet. + ### How Dependencies Are Stored (Canonical Format) APM normalizes every dependency entry on write — no matter how you specify a package, the stored form in `apm.yml` is always a clean, canonical string. This works like Docker's default registry convention: - **GitHub** is the default registry. The `github.com` host is stripped, leaving just `owner/repo`. - **Non-default hosts** (GitLab, Bitbucket, self-hosted) keep their FQDN: `gitlab.com/owner/repo`. +- **Logical requirement objects** are preserved as objects because they carry repository-selection metadata. | You type | Stored in apm.yml | |----------|-------------------| diff --git a/docs/src/content/docs/guides/private-packages.md b/docs/src/content/docs/guides/private-packages.md index f274af5fe..2220f3545 100644 --- a/docs/src/content/docs/guides/private-packages.md +++ b/docs/src/content/docs/guides/private-packages.md @@ -5,7 +5,7 @@ sidebar: order: 9 --- -A private APM package is just a private git repository with an `apm.yml`. There is no registry and no publish step — make the repo private, grant read access, and `apm install` handles the rest. +A private APM package can be consumed either from a private git repository or from a private OCI-backed repository configured in `~/.apm/repositories.yml`. In both cases, the package itself is still just an APM package with an `apm.yml`. ## Create the package @@ -57,6 +57,29 @@ dependencies: APM reuses the same port across protocols during clone fallback (so `ssh://host:7999/...` falls back to `https://host:7999/...`). If your host serves SSH and HTTPS on different ports and SSH is unreachable, pin the protocol that matches the port you need. +For private OCI-backed package storage, configure a repository on each client: + +```yaml +# ~/.apm/repositories.yml +repositories: + - name: corp-oci + type: oci + base: registry.example.com/apm + priority: 100 +``` + +Then reference the package logically in `apm.yml`: + +```yaml +dependencies: + apm: + - name: your-org/my-private-package + version: 1.0.0 + repository: corp-oci +``` + +Current OCI support is consume-only in this prototype. The OCI artifact is expected to contain one `*.tar.gz` with raw APM package sources. + ## Share with your team Every developer needs read access to the private repository and the appropriate token in their environment. For teams, a fine-grained PAT scoped to the organization works well — no write access required. diff --git a/docs/src/content/docs/reference/lockfile-spec.md b/docs/src/content/docs/reference/lockfile-spec.md index 1c42bc202..74f0a7683 100644 --- a/docs/src/content/docs/reference/lockfile-spec.md +++ b/docs/src/content/docs/reference/lockfile-spec.md @@ -90,6 +90,18 @@ dependencies: package_type: apm_package deployed_files: - .github/instructions/common-guidelines.instructions.md + + - repo_url: acme/security-pack + source_type: oci + repository_name: ghcr + oci_registry: ghcr + oci_repository: acme/security-pack + oci_tag: 1.2.0 + resolved_ref: ghcr.io/apm/acme/security-pack:1.2.0 + depth: 1 + package_type: apm_package + deployed_files: + - .github/instructions/security.instructions.md ``` ### 4.1 Top-Level Fields @@ -113,8 +125,10 @@ fields: |-------|------|----------|-------------| | `repo_url` | string | MUST | Source repository URL, or `_local/` for local path dependencies. | | `host` | string | MAY | Git host identifier (e.g., `github.com`). Omitted when inferrable from `repo_url`. | +| `source_type` | string | MAY | Resolved transport type. Current prototype values: `git`, `oci`. | +| `repository_name` | string | MAY | Name of the configured repository that satisfied a logical requirement. | | `resolved_commit` | string | MUST (remote) | Full 40-character commit SHA that was checked out. Required for remote (git) dependencies; MUST be omitted for local (`source: "local"`) dependencies. | -| `resolved_ref` | string | MUST (remote) | Git ref (tag, branch, SHA) that resolved to `resolved_commit`. Required for remote (git) dependencies; MUST be omitted for local (`source: "local"`) dependencies. | +| `resolved_ref` | string | MUST (remote) | Git ref (tag, branch, SHA) that resolved to `resolved_commit`, or the fully resolved OCI locator used for an OCI-backed dependency. Required for remote dependencies; MUST be omitted for local (`source: "local"`) dependencies. | | `version` | string | MAY | Semantic version of the package, if declared in its manifest. | | `virtual_path` | string | MAY | Sub-path within the repository for virtual (monorepo) packages. | | `is_virtual` | boolean | MAY | `true` if the package is a virtual sub-package. Omitted when `false`. | @@ -126,6 +140,10 @@ fields: | `deployed_files` | array of strings | MUST | Every file path APM deployed for this dependency, relative to project root. | | `source` | string | MAY | Dependency source. `"local"` for local path dependencies. Omitted for remote (git) dependencies. | | `local_path` | string | MAY | Filesystem path (relative or absolute) to the local package. Present only when `source` is `"local"`. | +| `oci_registry` | string | MAY | Logical OCI repository name used during resolution. | +| `oci_repository` | string | MAY | OCI repository path that stored the package archive. | +| `oci_tag` | string | MAY | OCI tag used when pulling the artifact. | +| `oci_digest` | string | MAY | OCI digest, when available from the transport or lock refresh. | | `is_insecure` | boolean | MAY | `true` when the dep was fetched over HTTP (unencrypted). Omitted when `false`. Presence forces re-approval on the next install: the apm.yml entry MUST carry `allow_insecure: true` and the invocation MUST pass `--allow-insecure` (or `--allow-insecure-host` for transitive deps). Absent or `false` means HTTPS/SSH. | | `allow_insecure` | boolean | MAY | `true` when the user's manifest explicitly approved the HTTP fetch with `allow_insecure: true`. Persisted alongside `is_insecure` for replay safety: a legacy lockfile with `is_insecure: true` but no `allow_insecure` fail-closes to `allow_insecure: false`, forcing re-approval. Omitted when `false`. | @@ -139,7 +157,9 @@ lists) SHOULD be omitted from the serialized output to keep the file concise. Each dependency is uniquely identified by its `repo_url`, or by the combination of `repo_url` and `virtual_path` for virtual packages. For local path dependencies (`source: "local"`), the unique key is the -`local_path` value. A conforming lock file MUST NOT contain duplicate +`local_path` value. In the current OCI prototype, implementations MAY use an +OCI-specific internal key shape such as `oci::` while +still storing the logical package identity in `repo_url`. A conforming lock file MUST NOT contain duplicate entries for the same key. ### 4.4 Content Integrity diff --git a/docs/src/content/docs/reference/manifest-schema.md b/docs/src/content/docs/reference/manifest-schema.md index dde0a3922..c67c87ea7 100644 --- a/docs/src/content/docs/reference/manifest-schema.md +++ b/docs/src/content/docs/reference/manifest-schema.md @@ -218,7 +218,7 @@ local_path_form = ("./" / "../" / "/" / "~/" / ".\\" / "..\\" / "~\\") path | Segment | Required | Pattern | Description | |---|---|---|---| -| `host` | OPTIONAL | FQDN (e.g. `gitlab.com`) | Git host. Defaults to `github.com`. | +| `host` | OPTIONAL | FQDN (e.g. `gitlab.com`) | Explicit host qualifier. When omitted, the string is treated as a logical package requirement resolved through configured repositories. When present, the string is treated as a direct host-qualified source. | | `port` | OPTIONAL | `1`–`65535` | Non-default port on `ssh://`, `https://`, `http://` clone URLs. Not expressible in SCP shorthand. | | `owner/repo` | REQUIRED | 2+ path segments of `[a-zA-Z0-9._-]+` | Repository path. GitHub uses exactly 2 segments (`owner/repo`). Non-GitHub hosts MAY use nested groups (e.g. `gitlab.com/group/sub/repo`). | | `virtual_path` | OPTIONAL | Path segments after repo | Subdirectory, file, or collection within the repo. See §4.1.3. | @@ -229,12 +229,12 @@ local_path_form = ("./" / "../" / "/" / "~/" / ".\\" / "..\\" / "~\\") path ```yaml dependencies: apm: - # GitHub shorthand (default host) — each line shows a syntax variant + # Logical package requirements - microsoft/apm-sample-package # latest (lockfile pins commit SHA) - microsoft/apm-sample-package#v1.0.0 # pinned to tag (immutable) - microsoft/apm-sample-package#main # branch ref (may change over time) - # Non-GitHub hosts (FQDN preserved) + # Direct host-qualified sources (FQDN preserved) - gitlab.com/acme/coding-standards - bitbucket.org/team/repo#main @@ -262,6 +262,29 @@ dependencies: #### 4.1.2. Object Form +APM supports two object-style forms for `dependencies.apm` entries: + +- a **logical requirement object**, resolved through configured repositories +- a **direct git/local object**, which points at an explicit source + +Logical requirement object: + +| Field | Type | Required | Pattern / Constraint | Description | +|---|---|---|---|---| +| `name` | `string` | REQUIRED | package name with at least one `/` | Logical package identity, for example `acme/security-pack`. | +| `version` | `string` | OPTIONAL | non-empty string | Version or ref-style selector used during resolution. | +| `repository` | `string` | OPTIONAL | non-empty string | Name of a configured repository in `~/.apm/repositories.yml`. | +| `alias` | `string` | OPTIONAL | `^[a-zA-Z0-9._-]+$` | Local display alias. | + +```yaml +- name: acme/security-pack + version: 1.2.0 + repository: corp-oci + alias: security-pack +``` + +Direct git/local object: + REQUIRED when the shorthand is ambiguous (e.g. nested-group repos with virtual paths). | Field | Type | Required | Pattern / Constraint | Description | @@ -286,6 +309,36 @@ Local path dependency (development only): - path: ./packages/my-shared-skills ``` +#### 4.1.2.1. Repository Configuration + +Logical requirement resolution is driven by a client-side file at `~/.apm/repositories.yml`. + +```yaml +repositories: + - name: github + type: git + base: https://github.com + priority: 100 + + - name: ghcr + type: oci + base: ghcr.io/apm + priority: 80 +``` + +| Field | Type | Required | Description | +|---|---|---|---| +| `name` | `string` | REQUIRED | Stable repository identifier used by `dependencies.apm[*].repository`. | +| `type` | `string` | REQUIRED | Repository backend type. Current values: `git`, `oci`. | +| `base` | `string` | REQUIRED | Base locator prefix for the repository. | +| `priority` | `integer` | OPTIONAL | Higher values are tried first when a dependency does not pin `repository`. | + +Built-in defaults are used when the file is absent or invalid: + +- `github` → `git` → `https://github.com` → priority `100` +- `gitlab` → `git` → `https://gitlab.com` → priority `90` +- `ghcr` → `oci` → `ghcr.io/apm` → priority `80` + #### 4.1.3. Virtual Packages A dependency MAY target a subdirectory, file, or collection within a repository rather than the whole repo. Conforming resolvers MUST classify virtual packages using the following rules, evaluated in order: diff --git a/packages/apm-guide/.apm/skills/apm-usage/dependencies.md b/packages/apm-guide/.apm/skills/apm-usage/dependencies.md index 503297591..8701d69aa 100644 --- a/packages/apm-guide/.apm/skills/apm-usage/dependencies.md +++ b/packages/apm-guide/.apm/skills/apm-usage/dependencies.md @@ -5,7 +5,11 @@ ```yaml dependencies: apm: - # GitHub shorthand + # Logical package requirements resolved via configured repositories + - microsoft/apm-sample-package#v1.0.0 + - acme/security-pack#1.2.0 + + # Logical requirement that will usually resolve from the default GitHub repository - microsoft/apm-sample-package - microsoft/apm-sample-package#v1.0.0 # pinned tag - microsoft/apm-sample-package#main # branch @@ -35,6 +39,47 @@ dependencies: - ../sibling-repo/my-package ``` +## Repository-driven requirements + +Logical requirements keep package identity separate from transport: + +```yaml +dependencies: + apm: + - name: acme/security-pack + version: 1.2.0 + + - name: acme/security-pack + version: 1.2.0 + repository: corp-oci + alias: security-pack +``` + +APM resolves those entries through `~/.apm/repositories.yml`: + +```yaml +repositories: + - name: github + type: git + base: https://github.com + priority: 100 + + - name: corp-oci + type: oci + base: registry.example.com/apm + priority: 110 +``` + +Resolution order: + +1. If `repository:` is set, APM only tries that repository. +2. Otherwise repositories are tried by descending `priority`. +3. The resolved source is written to `apm.lock.yaml`. + +Bare `owner/repo` strings in `apm.yml` are treated as logical requirements. +Use explicit git URLs or host-qualified refs such as `gitlab.com/group/repo` +when you want to bypass repository resolution and point at a specific git host. + ### Custom git ports Non-default git ports are preserved on `https://`, `http://`, and `ssh://` URLs @@ -95,6 +140,11 @@ both protocols. ## Object form (complex cases) ```yaml +- name: acme/security-pack + version: 1.2.0 + repository: corp-oci + alias: security-pack + - git: https://gitlab.com/acme/repo.git path: instructions/security # virtual sub-path ref: v2.0 # tag, branch, or SHA @@ -109,6 +159,15 @@ both protocols. - path: ./packages/my-skills # local only ``` +## OCI package expectations + +The current OCI prototype is consume-only. + +- A configured OCI repository resolves a logical package name to an OCI reference. +- The OCI artifact must contain exactly one `*.tar.gz`. +- That archive must contain raw APM package sources with `apm.yml` at the root or under one top-level directory. +- APM currently shells out to `oras` to pull the artifact. + ## Virtual package types Virtual packages reference a subset of a repository. @@ -134,6 +193,7 @@ APM normalizes dependency strings when saving to apm.yml: | `./packages/my-skills` | `./packages/my-skills` | GitHub URLs are stripped to shorthand; non-GitHub hosts keep the FQDN. +Logical requirement objects stay as objects because they carry repository-selection metadata. ## MCP dependency formats From f9b92e199111b87c9fd9c852a1b1744e84b69b79 Mon Sep 17 00:00:00 2001 From: Vladimir Sitnikov Date: Fri, 24 Apr 2026 19:48:38 +0300 Subject: [PATCH 3/3] Fix review issues in repository resolution prototype --- src/apm_cli/commands/uninstall/engine.py | 21 +++++++++++--------- src/apm_cli/deps/lockfile.py | 4 ---- src/apm_cli/deps/oci_registry.py | 9 +++++++-- src/apm_cli/models/dependency/requirement.py | 5 +++++ tests/unit/test_repository_resolution.py | 17 +++++++++++++++- 5 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/apm_cli/commands/uninstall/engine.py b/src/apm_cli/commands/uninstall/engine.py index 0281a2843..d2f930a9e 100644 --- a/src/apm_cli/commands/uninstall/engine.py +++ b/src/apm_cli/commands/uninstall/engine.py @@ -29,6 +29,15 @@ def _parse_dependency_entry(dep_entry): raise ValueError(f"Unsupported dependency entry type: {type(dep_entry).__name__}") +def _dependency_identity(dep) -> str: + """Return the stable identity used for matching and orphan detection.""" + if getattr(dep, "source", None) == "local" and getattr(dep, "local_path", None): + return dep.local_path + if getattr(dep, "is_virtual", False) and getattr(dep, "virtual_path", None): + return f"{dep.repo_url}/{dep.virtual_path}" + return dep.repo_url + + def _validate_uninstall_packages(packages, current_deps, logger): """Validate which packages can be removed and return matched/unmatched lists.""" packages_to_remove = [] @@ -75,8 +84,6 @@ def _dry_run_uninstall(packages_to_remove, apm_modules_dir, logger): logger.progress(f" - {pkg} from apm.yml") try: dep_ref = _parse_dependency_entry(pkg) - if not hasattr(dep_ref, "get_install_path"): - continue package_path = dep_ref.get_install_path(apm_modules_dir) except (ValueError, TypeError, AttributeError, KeyError): pkg_str = pkg if isinstance(pkg, str) else str(pkg) @@ -124,10 +131,6 @@ def _remove_packages_from_disk(packages_to_remove, apm_modules_dir, logger): for package in packages_to_remove: try: dep_ref = _parse_dependency_entry(package) - if not hasattr(dep_ref, "get_install_path"): - logger.progress(f"Removed {package} from manifest (OCI dependency)") - removed += 1 - continue package_path = dep_ref.get_install_path(apm_modules_dir) except (PathTraversalError,) as e: logger.error(f"Refusing to remove {package}: {e}") @@ -167,7 +170,7 @@ def _cleanup_transitive_orphans(lockfile, packages_to_remove, apm_modules_dir, a for pkg in packages_to_remove: try: ref = _parse_dependency_entry(pkg) - removed_repo_urls.add(getattr(ref, "repo_url", getattr(ref, "package", str(pkg)))) + removed_repo_urls.add(_dependency_identity(ref)) except (ValueError, TypeError, AttributeError, KeyError): removed_repo_urls.add(pkg) @@ -195,7 +198,7 @@ def _cleanup_transitive_orphans(lockfile, packages_to_remove, apm_modules_dir, a for dep_str in updated_data.get("dependencies", {}).get("apm", []) or []: try: ref = _parse_dependency_entry(dep_str) - remaining_deps.add(ref.get_unique_key()) + remaining_deps.add(_dependency_identity(ref)) except (ValueError, TypeError, AttributeError, KeyError): remaining_deps.add(str(dep_str)) except Exception: @@ -203,7 +206,7 @@ def _cleanup_transitive_orphans(lockfile, packages_to_remove, apm_modules_dir, a for dep in lockfile.get_all_dependencies(): key = dep.get_unique_key() - if key not in orphans and dep.repo_url not in removed_repo_urls: + if key not in orphans and _dependency_identity(dep) not in removed_repo_urls: remaining_deps.add(key) actual_orphans = orphans - remaining_deps diff --git a/src/apm_cli/deps/lockfile.py b/src/apm_cli/deps/lockfile.py index 61d69e6c7..71d0335c8 100644 --- a/src/apm_cli/deps/lockfile.py +++ b/src/apm_cli/deps/lockfile.py @@ -51,10 +51,6 @@ class LockedDependency: def get_unique_key(self) -> str: """Returns unique key for this dependency.""" - if self.source_type == "oci": - registry = self.oci_registry or "default" - repository = self.oci_repository or self.repo_url - return f"oci:{registry}:{repository}" if self.source == "local" and self.local_path: return self.local_path if self.is_virtual and self.virtual_path: diff --git a/src/apm_cli/deps/oci_registry.py b/src/apm_cli/deps/oci_registry.py index 0ff70dd2d..1ff31dd54 100644 --- a/src/apm_cli/deps/oci_registry.py +++ b/src/apm_cli/deps/oci_registry.py @@ -9,7 +9,7 @@ import tempfile from dataclasses import dataclass from datetime import datetime -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Optional from ..models.apm_package import ( @@ -20,6 +20,7 @@ ResolvedReference, validate_apm_package, ) +from ..utils.path_security import ensure_path_within @dataclass(frozen=True) @@ -122,7 +123,9 @@ def _extract_package_archive(archive_path: Path, target_path: Path) -> Path: """Extract a raw-package archive and return the package root.""" with tarfile.open(archive_path, "r:gz") as tar: for member in tar.getmembers(): - if member.name.startswith("/") or ".." in Path(member.name).parts: + normalized_name = member.name.replace("\\", "/") + normalized_path = PurePosixPath(normalized_name) + if normalized_name.startswith("/") or ".." in normalized_path.parts: raise RuntimeError( f"Refusing to extract unsafe archive entry: {member.name}" ) @@ -130,6 +133,8 @@ def _extract_package_archive(archive_path: Path, target_path: Path) -> Path: raise RuntimeError( f"Refusing to extract symlink/hardlink from OCI archive: {member.name}" ) + candidate = target_path.joinpath(*normalized_path.parts) + ensure_path_within(candidate, target_path) if sys.version_info >= (3, 12): tar.extractall(target_path, filter="data") else: diff --git a/src/apm_cli/models/dependency/requirement.py b/src/apm_cli/models/dependency/requirement.py index f7b4a2dc6..bb54ed9c7 100644 --- a/src/apm_cli/models/dependency/requirement.py +++ b/src/apm_cli/models/dependency/requirement.py @@ -57,6 +57,11 @@ def parse(cls, raw: str) -> "PackageRequirement": name = name.strip().strip("/") if not name or "/" not in name: raise ValueError("Package requirement must be in 'owner/repo' form") + first_segment = name.split("/", 1)[0] + if "." in first_segment: + raise ValueError( + "Host-qualified references must use direct dependency parsing" + ) validate_path_segments(name, context="package name") return cls(name=name, version=version) diff --git a/tests/unit/test_repository_resolution.py b/tests/unit/test_repository_resolution.py index 65c607923..41bd271fa 100644 --- a/tests/unit/test_repository_resolution.py +++ b/tests/unit/test_repository_resolution.py @@ -1,4 +1,5 @@ import tarfile +from urllib.parse import urlparse import yaml @@ -66,6 +67,7 @@ def test_locked_dependency_tracks_resolved_requirement_metadata(): assert round_trip.oci_repository == "acme/security-pack" assert round_trip.oci_tag == "1.0.0" assert round_trip.oci_digest == "sha256:abc" + assert round_trip.get_unique_key() == "acme/security-pack" def test_artifact_resolver_annotates_requirement_for_git(monkeypatch, tmp_path): @@ -97,7 +99,20 @@ def pull_package(self, resolved_reference, target_path, requirement=None): assert req.resolved_source_type == "git" assert req.resolved_repository == "github" - assert req.resolved_ref.endswith("microsoft/apm-standards.git#v1.2.0") + locator, fragment = req.resolved_ref.split("#", 1) + parsed = urlparse(locator) + assert parsed.hostname == "github.com" + assert parsed.path == "/microsoft/apm-standards.git" + assert fragment == "v1.2.0" + + +def test_package_requirement_rejects_host_qualified_refs(): + try: + PackageRequirement.parse("gitlab.com/group/repo") + except ValueError as exc: + assert "Host-qualified references" in str(exc) + else: + raise AssertionError("Expected host-qualified ref to be rejected") def test_oci_registry_client_extracts_raw_package_tarball(tmp_path, monkeypatch):