diff --git a/AGENTS.md b/AGENTS.md index 3add266..e265391 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,6 +8,7 @@ The v0.1 product is deliberately small: - validate a local skill directory - validate an embedded bundled skill directory tree +- resolve a public GitHub skill bundle directory tree - detect local agent hosts - resolve safe CLI install selection before writes - run the safe CLI install workflow for bundled skills @@ -19,7 +20,7 @@ The v0.1 product is deliberately small: - return structured reports - enforce parity through shared golden cases -Do not turn kitup into a marketplace, registry client, package manager, MCP server, GUI, agent runtime integration layer, or arbitrary skill installer. Remote installs, GitHub source installs, marketplace search, plugin systems, `AGENTS.md` mutation, script execution, and symlink-first workflows are outside v0.1 unless this boundary and golden cases are changed first. +Do not turn kitup into a marketplace, registry client, package manager, MCP server, GUI, agent runtime integration layer, or arbitrary skill installer. Only first-party public GitHub bundle resolution is inside v0.1 behavior. Other remote installs, private GitHub auth, marketplace search, plugin systems, `AGENTS.md` mutation, script execution, and symlink-first workflows are outside v0.1 unless this boundary and golden cases are changed first. ## Product Promise @@ -55,7 +56,8 @@ When docs and executable checks disagree, fix the source of truth or surface the Before non-trivial work, classify the request as one of: -- v0.1 bundled-skill source, selection, and installer behavior +- v0.1 bundled-skill bundle, selection, and installer behavior +- public GitHub bundle behavior - host adapter data correction - fixture or parity improvement - documentation of existing behavior @@ -63,9 +65,9 @@ Before non-trivial work, classify the request as one of: Only the first four are normal work. For post-v0.1 scope, do not implement product surface until the boundary is explicitly changed in docs and golden cases. -Every new source, selection, or installer behavior needs a golden case. A behavior that cannot be expressed in `testdata/cases` is not ready to become SDK behavior. +Every new bundle input, selection, or installer behavior needs a golden case. A behavior that cannot be expressed in `testdata/cases` is not ready to become SDK behavior. -Do not add speculative abstractions for future registries, package managers, remote sources, marketplace metadata, auth, or dependency resolution. Build the shallowest implementation that satisfies the existing case matrix. +Do not add speculative abstractions for future registries, package managers, custom providers, marketplace metadata, auth, or dependency resolution. Build the shallowest implementation that satisfies the existing case matrix. ## Host Adapter Rules @@ -101,7 +103,7 @@ Content hashes must be deterministic across TypeScript, Go, and Rust. Hash bundl Reports are API contracts. Return structured `installed`, `updated`, `skipped`, `conflicts`, and `errors` data instead of relying on logs. -Bundled skill sources are directory trees. `SKILL.md` must live at the bundle root, but references, scripts, assets, and other regular files are part of the same source and must be validated, hashed, and copied as a tree. +Skill bundles are directory trees. `SKILL.md` must live at the bundle root, but references, scripts, assets, and other regular files are part of the same bundle and must be validated, hashed, and copied as a tree. ## Multi-Language Parity diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4679e62..c4a7a5c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,6 +6,7 @@ Keep changes inside the v0.1 boundary: - validate local skill directories - validate embedded skill directory trees +- resolve public GitHub skill bundle directory trees - detect agent hosts - resolve safe CLI install selection - run the safe CLI install workflow @@ -15,7 +16,7 @@ Keep changes inside the v0.1 boundary: - return structured reports - keep TypeScript, Go, and Rust behavior aligned through golden cases -Do not add marketplace, registry, remote install, script execution, MCP server, GUI, or agent runtime behavior unless the product boundary changes first. +Do not add marketplace, registry, private remote install, custom provider, script execution, MCP server, GUI, or agent runtime behavior unless the product boundary changes first. ## Setup @@ -73,7 +74,7 @@ Host support is data-first. ## SDK Behavior Changes -Every source, selection, or installer behavior needs a golden case in `testdata/cases`. +Every bundle input, selection, or installer behavior needs a golden case in `testdata/cases`. Before opening a pull request: diff --git a/README.md b/README.md index faea6fd..5823b23 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # kitup -Shared installer SDK for bundled Agent Skills. +Shared installer SDK for bundled and public GitHub Agent Skills. -CLI authors ship a skill with their tool. `kitup` models that bundled skill as a directory tree, resolves safe agent targets, validates `SKILL.md`, copies the full tree into the right host directories, and writes ownership metadata so updates stay safe. +CLI authors ship or point to a skill. `kitup` models that skill as a directory tree, resolves safe agent targets, validates `SKILL.md`, copies the full tree into the right host directories, and writes ownership metadata so updates stay safe. ```text mycli skill install @@ -17,7 +17,7 @@ mycli skill install - resolve user and project skill directories - resolve safe CLI install selection before writing - validate bundled skills -- install from a local directory or embedded bundle tree +- install from a local directory, embedded bundle tree, or public GitHub bundle directory - copy, update, and uninstall kitup-owned installs - refuse unsafe overwrite conflicts - return structured install reports @@ -26,6 +26,7 @@ mycli skill install - not a skill marketplace - not a remote registry +- not a private GitHub auth client - not a replacement for user-facing skill discovery tools ## Usage @@ -97,6 +98,26 @@ await runBundledSkillInstall({ }); ``` +For public GitHub directories, configure the GitHub bundle in the embedding CLI. The user-facing install command stays the same: + +```ts +import { githubBundle, runBundledSkillInstall } from "@kitup/sdk"; + +await runBundledSkillInstall({ + appId: "mycli", + skillBundle: githubBundle({ + owner: "acme", + repo: "mycli-skills", + path: "skills/mycli", + ref: "v1.2.3", + }), + scope: "user", + agents: ["codex"], +}); +``` + +Go, Rust, and Cobra use the same `skillBundle` shape; see [API](docs/API.md) for language-specific constructors. + ### Go Install: diff --git a/docs/API.md b/docs/API.md index b1def15..bd7bd02 100644 --- a/docs/API.md +++ b/docs/API.md @@ -4,7 +4,7 @@ The core flow is: -1. normalize a bundled skill directory tree +1. resolve a local directory tree, embedded files, or public GitHub bundle 2. resolve safe target agent selection for CLI workflows 3. validate `SKILL.md` 4. copy, update, skip, or report conflicts @@ -20,6 +20,7 @@ import { detectHosts, directoryBundle, filesBundle, + githubBundle, installBundledSkill, installFlagError, installWorkflowError, @@ -45,6 +46,21 @@ const report = await installBundledSkill({ }); ``` +Public GitHub bundle call: + +```ts +const report = await installBundledSkill({ + appId: "mycli", + skillBundle: githubBundle({ + owner: "acme", + repo: "mycli-skills", + path: "skills/mycli", + ref: "v1.2.3", + }), + scope: "user", +}); +``` + Implemented functions: - `loadHostSpec(hostsFile?)` @@ -56,6 +72,7 @@ Implemented functions: - `computeBundleContentHash(bundle, cwd?)` - `directoryBundle(path)` - `filesBundle(files)` +- `githubBundle(options)` - `parseInstallFlags(flags)` - `agentSelectorFromFlags(values)` - `parseScopeFlag(value)` @@ -87,6 +104,21 @@ report, err := kitup.InstallBundledSkill(kitup.InstallOptions{ }) ``` +Public GitHub bundle call: + +```go +report, err := kitup.InstallBundledSkill(kitup.InstallOptions{ + AppID: "mycli", + SkillBundle: kitup.GitHubBundle(kitup.GitHubBundleOptions{ + Owner: "acme", + Repo: "mycli-skills", + Path: "skills/mycli", + Ref: "v1.2.3", + }), + Scope: kitup.UserScope, +}) +``` + Implemented functions: - `LoadHostSpec(hostsFile string)` @@ -99,6 +131,7 @@ Implemented functions: - `DirectoryBundle(path)` - `FSBundle(fsys, root)` - `FilesBundle(files)` +- `GitHubBundle(opts)` - `ParseInstallFlags(flags)` - `AgentSelectorFromFlags(values)` - `ParseScopeFlag(value)` @@ -133,6 +166,23 @@ let report = kitup::install_bundled_skill(&kitup::InstallOptions { })?; ``` +Public GitHub bundle call: + +```rust +let report = kitup::install_bundled_skill(&kitup::InstallOptions { + base: kitup::BaseOptions::default(), + app_id: "mycli".to_string(), + skill_bundle: kitup::github_bundle(kitup::GitHubBundleOptions { + owner: "acme".to_string(), + repo: "mycli-skills".to_string(), + path: "skills/mycli".to_string(), + ref_name: "v1.2.3".to_string(), + }), + scope: kitup::Scope::User, + agents: kitup::AgentSelector::Auto, +})?; +``` + Implemented functions: - `load_host_spec(hosts_file)` @@ -144,6 +194,7 @@ Implemented functions: - `compute_bundle_content_hash(bundle)` - `directory_bundle(path)` - `files_bundle(files)` +- `github_bundle(options)` - `parse_install_flags(flags)` - `agent_selector_from_flags(values, errors)` - `parse_scope_flag(value, errors)` @@ -163,12 +214,20 @@ Implemented functions: Install options use the same concepts across languages: - `appId` / `AppID` / `app_id`: owner id written to `.kitup.json` -- `skillBundle` / `SkillBundle` / `skill_bundle`: bundled skill directory tree source +- `skillBundle` / `SkillBundle` / `skill_bundle`: local directory, embedded files, or public GitHub bundle - `scope`: `user` or `project` - `agents`: `"auto"`, `"*"`, or explicit host ids - `home`, `cwd`, `hostsFile`: optional test and embedding overrides -Bundle sources must use root-relative POSIX paths. SDKs reject empty paths, absolute paths, `..`, duplicate files, and backslash paths. SDKs exclude `.kitup.json`, `.git`, `.DS_Store`, swap files, and editor backups before validation, hashing, and copy. +Bundle file paths must use root-relative POSIX paths. SDKs reject empty paths, absolute paths, `..`, duplicate files, and backslash paths. SDKs exclude `.kitup.json`, `.git`, `.DS_Store`, swap files, and editor backups before validation, hashing, and copy. + +The first non-local bundle constructor is GitHub only: + +- TypeScript: `githubBundle({ owner, repo, path, ref })` +- Go: `GitHubBundle(GitHubBundleOptions{Owner, Repo, Path, Ref})` +- Rust: `github_bundle(GitHubBundleOptions { owner, repo, path, ref_name })` + +GitHub bundle resolution downloads only files under the configured directory path, requires `SKILL.md` at that bundle root, records the requested ref and resolved commit, and writes GitHub provenance into `.kitup.json`. It does not search GitHub, install dependencies, execute scripts, handle private auth, or install whole repositories by default. The embedding CLI owns command names and framework attachment. `kitup` owns standard install flag semantics, selector mapping, user-facing workflow text, summary rendering, confirmation, dry-run planning, workflow exit classification, and execution. For user-facing commands, call `runBundledSkillInstall` / `RunBundledSkillInstall` / `run_bundled_skill_install` with values from the shared flag parsing helpers. diff --git a/examples/rust/Cargo.lock b/examples/rust/Cargo.lock index 1ac6df3..6589b45 100644 --- a/examples/rust/Cargo.lock +++ b/examples/rust/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "block-buffer" version = "0.10.4" @@ -11,6 +17,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -46,6 +62,32 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -56,6 +98,120 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "itoa" version = "1.0.18" @@ -69,6 +225,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "ureq", ] [[package]] @@ -85,12 +242,45 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + [[package]] name = "memchr" version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -109,6 +299,55 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "untrusted", + "windows-sys", +] + +[[package]] +name = "rustls" +version = "0.23.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "serde" version = "1.0.228" @@ -163,6 +402,30 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.118" @@ -174,6 +437,27 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "typenum" version = "1.20.1" @@ -186,12 +470,237 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.8", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/go/kitup.go b/go/kitup.go index a4f866a..0b181af 100644 --- a/go/kitup.go +++ b/go/kitup.go @@ -9,6 +9,8 @@ import ( "fmt" "io" "io/fs" + "net/http" + "net/url" "os" pathpkg "path" "path/filepath" @@ -271,11 +273,19 @@ type SkillFile struct { } type SkillBundle struct { - kind string - dir string - fsys fs.FS - root string - files []SkillFile + kind string + dir string + fsys fs.FS + root string + files []SkillFile + github GitHubBundleOptions +} + +type GitHubBundleOptions struct { + Owner string + Repo string + Path string + Ref string } func DirectoryBundle(dir string) SkillBundle { @@ -290,6 +300,10 @@ func FilesBundle(files []SkillFile) SkillBundle { return SkillBundle{kind: "files", files: files} } +func GitHubBundle(opts GitHubBundleOptions) SkillBundle { + return SkillBundle{kind: "github", github: opts} +} + type TargetGroup struct { HostIDs []string SkillName string @@ -352,11 +366,21 @@ type InstallWorkflowReport struct { } type metadata struct { - SchemaVersion int `json:"schemaVersion"` - AppID string `json:"appId"` - SkillName string `json:"skillName"` - Source string `json:"source"` - Hash string `json:"hash"` + SchemaVersion int `json:"schemaVersion"` + AppID string `json:"appId"` + SkillName string `json:"skillName"` + Source string `json:"source"` + Hash string `json:"hash"` + SourceID string `json:"sourceId,omitempty"` + Version string `json:"version,omitempty"` + Provenance map[string]string `json:"provenance,omitempty"` +} + +type bundleMetadata struct { + Source string + SourceID string + Version string + Provenance map[string]string } type bundleFile struct { @@ -773,9 +797,13 @@ func UninstallBundledSkill(opts UninstallOptions) (UninstallReport, error) { } func installOrPlan(opts InstallOptions, write bool) (InstallReport, error) { - bundle, err := readSkillBundle(opts.SkillBundle) + bundle, bundleMeta, err := resolveSkillBundle(opts.SkillBundle) if err != nil { - return emptyInstallReport([]map[string]any{{"reason": "invalid-skill-bundle"}}), nil + reason := "invalid-skill-bundle" + if opts.SkillBundle.kind == "github" { + reason = "bundle-resolve-failed" + } + return emptyInstallReport([]map[string]any{{"reason": reason}}), nil } skill := validateNormalizedSkill(bundle) if !skill.Valid { @@ -793,7 +821,7 @@ func installOrPlan(opts InstallOptions, write bool) (InstallReport, error) { switch { case !present: if write { - if err := copyManagedSkill(bundle, target.TargetDir, opts.AppID, skill.SkillName, hash); err != nil { + if err := copyManagedSkill(bundle, target.TargetDir, opts.AppID, skill.SkillName, hash, bundleMeta); err != nil { return report, err } } @@ -806,7 +834,7 @@ func installOrPlan(opts InstallOptions, write bool) (InstallReport, error) { report.Skipped = append(report.Skipped, withReason(result, "unchanged")) default: if write { - if err := replaceManagedSkill(bundle, target.TargetDir, opts.AppID, skill.SkillName, hash); err != nil { + if err := replaceManagedSkill(bundle, target.TargetDir, opts.AppID, skill.SkillName, hash, bundleMeta); err != nil { return report, err } } @@ -816,17 +844,17 @@ func installOrPlan(opts InstallOptions, write bool) (InstallReport, error) { return report, nil } -func copyManagedSkill(bundle normalizedSkillBundle, targetDir, appID, skillName, hash string) error { +func copyManagedSkill(bundle normalizedSkillBundle, targetDir, appID, skillName, hash string, bundleMeta bundleMetadata) error { if err := os.RemoveAll(targetDir); err != nil { return err } if err := copySkillBundle(bundle, targetDir); err != nil { return err } - return writeMetadata(targetDir, appID, skillName, hash) + return writeMetadata(targetDir, appID, skillName, hash, bundleMeta) } -func replaceManagedSkill(bundle normalizedSkillBundle, targetDir, appID, skillName, hash string) error { +func replaceManagedSkill(bundle normalizedSkillBundle, targetDir, appID, skillName, hash string, bundleMeta bundleMetadata) error { suffix := ".kitup-" + time.Now().Format("20060102150405.000000000") tmp := targetDir + suffix backup := targetDir + suffix + "-backup" @@ -834,7 +862,7 @@ func replaceManagedSkill(bundle normalizedSkillBundle, targetDir, appID, skillNa if err := copySkillBundle(bundle, tmp); err != nil { return err } - if err := writeMetadata(tmp, appID, skillName, hash); err != nil { + if err := writeMetadata(tmp, appID, skillName, hash, bundleMeta); err != nil { _ = os.RemoveAll(tmp) return err } @@ -872,8 +900,21 @@ func copySkillBundle(bundle normalizedSkillBundle, dest string) error { return nil } -func writeMetadata(targetDir, appID, skillName, hash string) error { - data, err := json.MarshalIndent(metadata{SchemaVersion: 1, AppID: appID, SkillName: skillName, Source: "bundled", Hash: hash}, "", " ") +func writeMetadata(targetDir, appID, skillName, hash string, bundleMeta bundleMetadata) error { + meta := metadata{ + SchemaVersion: 1, + AppID: appID, + SkillName: skillName, + Source: bundleMeta.Source, + Hash: hash, + SourceID: bundleMeta.SourceID, + Version: bundleMeta.Version, + Provenance: bundleMeta.Provenance, + } + if meta.Source == "" { + meta.Source = "bundled" + } + data, err := json.MarshalIndent(meta, "", " ") if err != nil { return err } @@ -1180,6 +1221,137 @@ func parseFrontmatter(content string) map[string]string { return fields } +func resolveSkillBundle(bundle SkillBundle) (normalizedSkillBundle, bundleMetadata, error) { + switch bundle.kind { + case "github": + return resolveGitHubBundle(bundle.github) + default: + normalized, err := readSkillBundle(bundle) + if err != nil { + return normalizedSkillBundle{}, bundleMetadata{}, err + } + return normalized, bundleMetadata{Source: "bundled"}, nil + } +} + +func resolveGitHubBundle(opts GitHubBundleOptions) (normalizedSkillBundle, bundleMetadata, error) { + root := trimGitHubPath(opts.Path) + if opts.Owner == "" || opts.Repo == "" || root == "" || opts.Ref == "" { + return normalizedSkillBundle{}, bundleMetadata{}, errors.New("invalid github bundle") + } + apiBase := envBaseURL("KITUP_GITHUB_API_BASE_URL", "https://api.github.com") + rawBase := envBaseURL("KITUP_GITHUB_RAW_BASE_URL", "https://raw.githubusercontent.com") + var commit struct { + Sha string `json:"sha"` + Commit struct { + Tree struct { + Sha string `json:"sha"` + } `json:"tree"` + } `json:"commit"` + } + if err := getJSON(apiBase+"/repos/"+escapePathPart(opts.Owner)+"/"+escapePathPart(opts.Repo)+"/commits/"+escapePathPart(opts.Ref), &commit); err != nil { + return normalizedSkillBundle{}, bundleMetadata{}, err + } + if commit.Sha == "" || commit.Commit.Tree.Sha == "" { + return normalizedSkillBundle{}, bundleMetadata{}, errors.New("invalid github commit") + } + var tree struct { + Tree []struct { + Path string `json:"path"` + Type string `json:"type"` + Mode string `json:"mode"` + } `json:"tree"` + } + if err := getJSON(apiBase+"/repos/"+escapePathPart(opts.Owner)+"/"+escapePathPart(opts.Repo)+"/git/trees/"+escapePathPart(commit.Commit.Tree.Sha)+"?recursive=1", &tree); err != nil { + return normalizedSkillBundle{}, bundleMetadata{}, err + } + prefix := root + "/" + files := []SkillFile{} + for _, item := range tree.Tree { + if item.Type != "blob" || !strings.HasPrefix(item.Path, prefix) { + continue + } + contents, err := getBytes(rawBase + "/" + escapePathPart(opts.Owner) + "/" + escapePathPart(opts.Repo) + "/" + escapePathPart(commit.Sha) + "/" + escapePath(item.Path)) + if err != nil { + return normalizedSkillBundle{}, bundleMetadata{}, err + } + mode := fs.FileMode(0o644) + if item.Mode == "100755" { + mode = 0o755 + } + files = append(files, SkillFile{Path: strings.TrimPrefix(item.Path, prefix), Contents: contents, Mode: mode}) + } + if len(files) == 0 { + return normalizedSkillBundle{}, bundleMetadata{}, errors.New("github bundle path not found") + } + bundle, err := normalizeSkillFiles(files) + if err != nil { + return normalizedSkillBundle{}, bundleMetadata{}, err + } + return bundle, bundleMetadata{ + Source: "github", + SourceID: "github:" + opts.Owner + "/" + opts.Repo + "/" + root, + Version: opts.Ref, + Provenance: map[string]string{ + "owner": opts.Owner, + "repo": opts.Repo, + "path": root, + "ref": opts.Ref, + "resolvedCommit": commit.Sha, + }, + }, nil +} + +func envBaseURL(name, fallback string) string { + value := strings.TrimRight(os.Getenv(name), "/") + if value == "" { + return fallback + } + return value +} + +func getJSON(url string, value any) error { + data, err := getBytes(url) + if err != nil { + return err + } + return json.Unmarshal(data, value) +} + +func getBytes(value string) ([]byte, error) { + request, err := http.NewRequest(http.MethodGet, value, nil) + if err != nil { + return nil, err + } + request.Header.Set("User-Agent", "kitup") + client := http.Client{Timeout: 30 * time.Second} + response, err := client.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode > 299 { + return nil, fmt.Errorf("github request failed: %s", value) + } + return io.ReadAll(response.Body) +} + +func trimGitHubPath(value string) string { + return strings.Trim(value, "/") +} + +func escapePath(value string) string { + parts := strings.Split(value, "/") + for index, part := range parts { + parts[index] = escapePathPart(part) + } + return strings.Join(parts, "/") +} + +func escapePathPart(value string) string { + return url.PathEscape(value) +} + func readSkillBundle(bundle SkillBundle) (normalizedSkillBundle, error) { switch bundle.kind { case "directory": @@ -1196,6 +1368,9 @@ func readSkillBundle(bundle SkillBundle) (normalizedSkillBundle, error) { return normalizeSkillFiles(files) case "files": return normalizeSkillFiles(bundle.files) + case "github": + normalized, _, err := resolveGitHubBundle(bundle.github) + return normalized, err default: return normalizedSkillBundle{}, errors.New("missing skill bundle") } diff --git a/go/kitup_test.go b/go/kitup_test.go index 7b9b00b..87e0930 100644 --- a/go/kitup_test.go +++ b/go/kitup_test.go @@ -3,6 +3,8 @@ package kitup import ( "bytes" "encoding/json" + "net/http" + "net/http/httptest" "os" "path/filepath" "reflect" @@ -183,6 +185,9 @@ func setupGiven(t *testing.T, tc goldenCase, home, workspace string) { if meta, ok := tc.Given["metadata"].(map[string]any); ok { writeMetadataFixture(t, tc, home, workspace, meta) } + if github, ok := tc.Given["github"].(map[string]any); ok { + startGitHubFixture(t, github) + } } func assertExpectedFiles(t *testing.T, tc goldenCase, home, workspace string) { @@ -209,16 +214,7 @@ func assertExpectedMetadata(t *testing.T, tc goldenCase, home, workspace string) for key, value := range meta["fields"].(map[string]any) { equal(t, actual[key], value) } - hash := meta["hash"].(string) - if hash == "from-skill-bundle-dir" { - var err error - hash, err = ComputeBundleContentHash(DirectoryBundle(repoPathFromCase(tc.Options["skillBundleDir"].(string)))) - must(t, err) - } else if hash == "from-skill-files" { - var err error - hash, err = ComputeBundleContentHash(FilesBundle(skillFiles(tc.Options["skillFiles"].([]any)))) - must(t, err) - } + hash := expectedBundleHash(t, tc, meta["hash"].(string)) equal(t, actual["hash"], hash) } @@ -242,12 +238,7 @@ func assertExpectedWriteCounts(t *testing.T, tc goldenCase, report any, home, wo } func writeMetadataFixture(t *testing.T, tc goldenCase, home, workspace string, meta map[string]any) { - hash := meta["hash"].(string) - if hash == "from-skill-bundle-dir" { - var err error - hash, err = ComputeBundleContentHash(DirectoryBundle(caseSkillBundleDir(tc))) - must(t, err) - } + hash := expectedBundleHash(t, tc, meta["hash"].(string)) fields := map[string]any{} for key, value := range meta["fields"].(map[string]any) { fields[key] = value @@ -256,6 +247,25 @@ func writeMetadataFixture(t *testing.T, tc goldenCase, home, workspace string, m writeFixtureFile(t, expandString(meta["path"].(string), home, workspace), fields) } +func expectedBundleHash(t *testing.T, tc goldenCase, marker string) string { + switch marker { + case "from-skill-bundle-dir": + hash, err := ComputeBundleContentHash(DirectoryBundle(caseSkillBundleDir(tc))) + must(t, err) + return hash + case "from-skill-files": + hash, err := ComputeBundleContentHash(FilesBundle(skillFiles(tc.Options["skillFiles"].([]any)))) + must(t, err) + return hash + case "from-github-bundle": + hash, err := ComputeBundleContentHash(FilesBundle(githubSkillFiles(tc))) + must(t, err) + return hash + default: + return marker + } +} + func assertSelection(t *testing.T, actual, expected map[string]any) { if expectedCount, ok := expected["selectedCount"]; ok { equal(t, float64(len(actual["selectedHostIds"].([]any))), expectedCount) @@ -344,6 +354,14 @@ func skillBundleFromOptions(opts map[string]any) SkillBundle { if dir, ok := opts["skillBundleDir"].(string); ok { return DirectoryBundle(repoPathFromCase(dir)) } + if bundle, ok := opts["githubBundle"].(map[string]any); ok { + return GitHubBundle(GitHubBundleOptions{ + Owner: bundle["owner"].(string), + Repo: bundle["repo"].(string), + Path: bundle["path"].(string), + Ref: bundle["ref"].(string), + }) + } return SkillBundle{} } @@ -359,6 +377,65 @@ func skillFiles(values []any) []SkillFile { return files } +func githubSkillFiles(tc goldenCase) []SkillFile { + bundle := tc.Options["githubBundle"].(map[string]any) + root := strings.Trim(bundle["path"].(string), "/") + "/" + github := tc.Given["github"].(map[string]any) + rawFiles := github["files"].(map[string]any) + files := []SkillFile{} + for path, contents := range rawFiles { + if strings.HasPrefix(path, root) { + files = append(files, SkillFile{Path: strings.TrimPrefix(path, root), Contents: []byte(contents.(string))}) + } + } + return files +} + +func startGitHubFixture(t *testing.T, github map[string]any) { + owner := github["owner"].(string) + repo := github["repo"].(string) + ref := github["ref"].(string) + commit := github["commit"].(string) + treeSha := github["treeSha"].(string) + files := github["files"].(map[string]any) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/repos/" + owner + "/" + repo + "/commits/" + ref: + writeResponseJSON(t, w, map[string]any{"sha": commit, "commit": map[string]any{"tree": map[string]any{"sha": treeSha}}}) + case "/repos/" + owner + "/" + repo + "/git/trees/" + treeSha: + tree := []map[string]string{} + for path := range files { + mode := "100644" + if strings.HasSuffix(path, ".sh") { + mode = "100755" + } + tree = append(tree, map[string]string{"path": path, "type": "blob", "mode": mode}) + } + writeResponseJSON(t, w, map[string]any{"tree": tree}) + default: + prefix := "/" + owner + "/" + repo + "/" + commit + "/" + if strings.HasPrefix(r.URL.Path, prefix) { + path := strings.TrimPrefix(r.URL.Path, prefix) + if contents, ok := files[path]; ok { + _, _ = w.Write([]byte(contents.(string))) + return + } + } + http.NotFound(w, r) + } + })) + t.Setenv("KITUP_GITHUB_API_BASE_URL", server.URL) + t.Setenv("KITUP_GITHUB_RAW_BASE_URL", server.URL) + t.Cleanup(server.Close) +} + +func writeResponseJSON(t *testing.T, w http.ResponseWriter, value any) { + w.Header().Set("content-type", "application/json") + data, err := json.Marshal(value) + must(t, err) + _, _ = w.Write(data) +} + func boolValue(value any) bool { boolean, _ := value.(bool) return boolean diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 9e698a9..ac410ec 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "block-buffer" version = "0.10.4" @@ -11,6 +17,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -46,6 +62,32 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -56,6 +98,120 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "itoa" version = "1.0.18" @@ -69,6 +225,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "ureq", ] [[package]] @@ -77,12 +234,45 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + [[package]] name = "memchr" version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -101,6 +291,55 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "untrusted", + "windows-sys", +] + +[[package]] +name = "rustls" +version = "0.23.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "serde" version = "1.0.228" @@ -155,6 +394,30 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.118" @@ -166,6 +429,27 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "typenum" version = "1.20.1" @@ -178,12 +462,237 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.8", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index eb74c74..94b1cfb 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -14,3 +14,4 @@ categories = ["development-tools"] serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.10" +ureq = { version = "2", default-features = false, features = ["tls"] } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 3306df5..8f636b9 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -7,7 +7,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::fs; use std::io::{self, BufRead, Write}; use std::path::{Path, PathBuf}; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum Scope { @@ -177,6 +177,15 @@ pub struct SkillFile { pub enum SkillBundle { Directory(PathBuf), Files(Vec), + GitHub(GitHubBundleOptions), +} + +#[derive(Clone, Debug)] +pub struct GitHubBundleOptions { + pub owner: String, + pub repo: String, + pub path: String, + pub ref_name: String, } pub fn directory_bundle(path: impl Into) -> SkillBundle { @@ -187,6 +196,10 @@ pub fn files_bundle(files: Vec) -> SkillBundle { SkillBundle::Files(files) } +pub fn github_bundle(options: GitHubBundleOptions) -> SkillBundle { + SkillBundle::GitHub(options) +} + #[derive(Clone, Debug)] pub struct TargetGroup { pub host_ids: Vec, @@ -300,6 +313,14 @@ struct NormalizedSkillBundle { by_path: BTreeMap, } +#[derive(Clone, Debug)] +struct BundleMetadata { + source: String, + source_id: Option, + version: Option, + provenance: BTreeMap, +} + pub fn parse_install_flags(flags: InstallFlagValues) -> ParsedInstallFlags { let mut errors = Vec::new(); let scope = parse_scope_flag(flags.scope.as_deref(), &mut errors); @@ -878,11 +899,16 @@ pub fn uninstall_bundled_skill(options: &UninstallOptions) -> io::Result io::Result { - let bundle = match read_skill_bundle(&options.skill_bundle) { - Ok(bundle) => bundle, + let (bundle, bundle_metadata) = match resolve_skill_bundle(&options.skill_bundle) { + Ok(value) => value, Err(_) => { + let reason = if matches!(options.skill_bundle, SkillBundle::GitHub(_)) { + "bundle-resolve-failed" + } else { + "invalid-skill-bundle" + }; return Ok(install_report(vec![json!({ - "reason": "invalid-skill-bundle" + "reason": reason })])); } }; @@ -908,6 +934,7 @@ fn install_or_plan(options: &InstallOptions, write: bool) -> io::Result io::Result io::Result<()> { let _ = fs::remove_dir_all(target_dir); copy_skill_bundle(bundle, target_dir)?; - write_metadata(target_dir, app_id, skill_name, hash) + write_metadata(target_dir, app_id, skill_name, hash, bundle_metadata) } fn replace_managed_skill( @@ -960,6 +989,7 @@ fn replace_managed_skill( app_id: &str, skill_name: &str, hash: &str, + bundle_metadata: &BundleMetadata, ) -> io::Result<()> { let suffix = format!( ".kitup-{}", @@ -972,7 +1002,7 @@ fn replace_managed_skill( let backup = PathBuf::from(format!("{}{}-backup", target_dir.display(), suffix)); let _ = fs::remove_dir_all(&tmp); copy_skill_bundle(bundle, &tmp)?; - write_metadata(&tmp, app_id, skill_name, hash)?; + write_metadata(&tmp, app_id, skill_name, hash, bundle_metadata)?; fs::rename(target_dir, &backup)?; if let Err(error) = fs::rename(&tmp, target_dir) { let _ = fs::remove_dir_all(&tmp); @@ -999,14 +1029,30 @@ fn copy_skill_bundle(bundle: &NormalizedSkillBundle, dest: &Path) -> io::Result< Ok(()) } -fn write_metadata(target_dir: &Path, app_id: &str, skill_name: &str, hash: &str) -> io::Result<()> { - let data = serde_json::to_vec_pretty(&json!({ +fn write_metadata( + target_dir: &Path, + app_id: &str, + skill_name: &str, + hash: &str, + bundle_metadata: &BundleMetadata, +) -> io::Result<()> { + let mut value = json!({ "schemaVersion": 1, "appId": app_id, "skillName": skill_name, - "source": "bundled", + "source": bundle_metadata.source, "hash": hash - }))?; + }); + if let Some(source_id) = &bundle_metadata.source_id { + value["sourceId"] = json!(source_id); + } + if let Some(version) = &bundle_metadata.version { + value["version"] = json!(version); + } + if !bundle_metadata.provenance.is_empty() { + value["provenance"] = json!(bundle_metadata.provenance); + } + let data = serde_json::to_vec_pretty(&value)?; let mut data = data; data.push(b'\n'); fs::write(target_dir.join(".kitup.json"), data) @@ -1358,10 +1404,180 @@ fn valid_skill_name(name: &str) -> bool { true } +fn resolve_skill_bundle( + bundle: &SkillBundle, +) -> io::Result<(NormalizedSkillBundle, BundleMetadata)> { + match bundle { + SkillBundle::GitHub(options) => resolve_github_bundle(options), + _ => Ok(( + read_skill_bundle(bundle)?, + BundleMetadata { + source: "bundled".to_string(), + source_id: None, + version: None, + provenance: BTreeMap::new(), + }, + )), + } +} + +fn resolve_github_bundle( + options: &GitHubBundleOptions, +) -> io::Result<(NormalizedSkillBundle, BundleMetadata)> { + let root = trim_github_path(&options.path); + if options.owner.is_empty() + || options.repo.is_empty() + || root.is_empty() + || options.ref_name.is_empty() + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid github bundle", + )); + } + let api_base = env_base_url("KITUP_GITHUB_API_BASE_URL", "https://api.github.com"); + let raw_base = env_base_url( + "KITUP_GITHUB_RAW_BASE_URL", + "https://raw.githubusercontent.com", + ); + let commit: Value = get_json(&format!( + "{}/repos/{}/{}/commits/{}", + api_base, + escape_path_part(&options.owner), + escape_path_part(&options.repo), + escape_path_part(&options.ref_name) + ))?; + let resolved_commit = commit["sha"].as_str().unwrap_or("").to_string(); + let tree_sha = commit["commit"]["tree"]["sha"] + .as_str() + .unwrap_or("") + .to_string(); + if resolved_commit.is_empty() || tree_sha.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid github commit", + )); + } + let tree: Value = get_json(&format!( + "{}/repos/{}/{}/git/trees/{}?recursive=1", + api_base, + escape_path_part(&options.owner), + escape_path_part(&options.repo), + escape_path_part(&tree_sha) + ))?; + let prefix = format!("{root}/"); + let mut files = Vec::new(); + for item in tree["tree"].as_array().into_iter().flatten() { + let path = item["path"].as_str().unwrap_or(""); + if item["type"].as_str() != Some("blob") || !path.starts_with(&prefix) { + continue; + } + let contents = get_bytes(&format!( + "{}/{}/{}/{}/{}", + raw_base, + escape_path_part(&options.owner), + escape_path_part(&options.repo), + escape_path_part(&resolved_commit), + escape_path(path) + ))?; + let mode = if item["mode"].as_str() == Some("100755") { + Some(0o755) + } else { + Some(0o644) + }; + files.push(SkillFile { + path: path.strip_prefix(&prefix).unwrap().to_string(), + contents, + mode, + }); + } + if files.is_empty() { + return Err(io::Error::new( + io::ErrorKind::NotFound, + "github bundle path not found", + )); + } + let mut provenance = BTreeMap::new(); + provenance.insert("owner".to_string(), options.owner.clone()); + provenance.insert("repo".to_string(), options.repo.clone()); + provenance.insert("path".to_string(), root.clone()); + provenance.insert("ref".to_string(), options.ref_name.clone()); + provenance.insert("resolvedCommit".to_string(), resolved_commit); + Ok(( + normalize_skill_files(files)?, + BundleMetadata { + source: "github".to_string(), + source_id: Some(format!( + "github:{}/{}/{}", + options.owner, options.repo, root + )), + version: Some(options.ref_name.clone()), + provenance, + }, + )) +} + +fn env_base_url(name: &str, fallback: &str) -> String { + std::env::var(name) + .unwrap_or_else(|_| fallback.to_string()) + .trim_end_matches('/') + .to_string() +} + +fn get_json(url: &str) -> io::Result { + serde_json::from_slice(&get_bytes(url)?).map_err(io::Error::other) +} + +fn get_bytes(url: &str) -> io::Result> { + let agent = ureq::AgentBuilder::new() + .timeout(Duration::from_secs(30)) + .build(); + let response = agent + .get(url) + .set("User-Agent", "kitup") + .call() + .map_err(io::Error::other)?; + if response.status() < 200 || response.status() > 299 { + return Err(io::Error::other(format!("github request failed: {url}"))); + } + let mut reader = response.into_reader(); + let mut data = Vec::new(); + std::io::Read::read_to_end(&mut reader, &mut data)?; + Ok(data) +} + +fn trim_github_path(value: &str) -> String { + value.trim_matches('/').to_string() +} + +fn escape_path(value: &str) -> String { + value + .split('/') + .map(escape_path_part) + .collect::>() + .join("/") +} + +fn escape_path_part(value: &str) -> String { + let mut out = String::new(); + for byte in value.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + out.push(byte as char); + } else { + out.push_str(&format!("%{byte:02X}")); + } + } + out +} + fn read_skill_bundle(bundle: &SkillBundle) -> io::Result { match bundle { SkillBundle::Directory(root) => normalize_skill_files(read_directory_bundle_files(root)?), SkillBundle::Files(files) => normalize_skill_files(files.clone()), + SkillBundle::GitHub(options) => { + let (bundle, _) = resolve_github_bundle(options)?; + Ok(bundle) + } } } diff --git a/rust/tests/golden.rs b/rust/tests/golden.rs index e5f06c8..9a214a6 100644 --- a/rust/tests/golden.rs +++ b/rust/tests/golden.rs @@ -1,16 +1,19 @@ use kitup::{ classify_install_workflow_exit, compute_bundle_content_hash, detect_hosts, directory_bundle, - files_bundle, install_bundled_skill, load_host_spec, parse_install_flags, plan_bundled_skill, - resolve_hosts, resolve_install_selection, run_bundled_skill_install_with_io, - uninstall_bundled_skill, update_bundled_skill, validate_skill_bundle, AgentSelector, - BaseOptions, InstallFlagValues, InstallOptions, InstallSelectionOptions, - InstallWorkflowOptions, ParsedInstallFlags, Scope, SkillBundle, SkillFile, UninstallOptions, + files_bundle, github_bundle, install_bundled_skill, load_host_spec, parse_install_flags, + plan_bundled_skill, resolve_hosts, resolve_install_selection, + run_bundled_skill_install_with_io, uninstall_bundled_skill, update_bundled_skill, + validate_skill_bundle, AgentSelector, BaseOptions, GitHubBundleOptions, InstallFlagValues, + InstallOptions, InstallSelectionOptions, InstallWorkflowOptions, ParsedInstallFlags, Scope, + SkillBundle, SkillFile, UninstallOptions, }; use serde::Deserialize; use serde_json::{json, Map, Value}; use std::fs; -use std::io::Cursor; +use std::io::{Cursor, Read}; +use std::net::{TcpListener, TcpStream}; use std::path::{Path, PathBuf}; +use std::thread; use std::time::{SystemTime, UNIX_EPOCH}; #[derive(Deserialize)] @@ -301,6 +304,9 @@ fn setup_given(case: &GoldenCase, home: &Path, workspace: &Path) { if let Some(metadata) = case.given.get("metadata").and_then(Value::as_object) { write_metadata_fixture(case, home, workspace, metadata); } + if let Some(github) = case.given.get("github").and_then(Value::as_object) { + start_github_fixture(github); + } } fn assert_expected_files(case: &GoldenCase, home: &Path, workspace: &Path) { @@ -339,18 +345,7 @@ fn assert_expected_metadata(case: &GoldenCase, home: &Path, workspace: &Path) { for (key, value) in metadata["fields"].as_object().unwrap() { assert_json_eq(&actual[key], value.clone()); } - let mut hash = metadata["hash"].as_str().unwrap().to_string(); - if hash == "from-skill-bundle-dir" { - hash = compute_bundle_content_hash(&directory_bundle(repo_path( - case.options["skillBundleDir"].as_str().unwrap(), - ))) - .unwrap(); - } else if hash == "from-skill-files" { - hash = compute_bundle_content_hash(&files_bundle(skill_files( - case.options["skillFiles"].as_array().unwrap(), - ))) - .unwrap(); - } + let hash = expected_bundle_hash(case, metadata["hash"].as_str().unwrap()); assert_eq!(actual["hash"], hash); } @@ -448,10 +443,7 @@ fn write_metadata_fixture( metadata: &Map, ) { let mut fields = metadata["fields"].as_object().unwrap().clone(); - let mut hash = metadata["hash"].as_str().unwrap().to_string(); - if hash == "from-skill-bundle-dir" { - hash = compute_bundle_content_hash(&directory_bundle(case_skill_bundle_dir(case))).unwrap(); - } + let hash = expected_bundle_hash(case, metadata["hash"].as_str().unwrap()); fields.insert("hash".to_string(), json!(hash)); write_fixture_file( &expand_string(metadata["path"].as_str().unwrap(), home, workspace), @@ -459,6 +451,22 @@ fn write_metadata_fixture( ); } +fn expected_bundle_hash(case: &GoldenCase, marker: &str) -> String { + match marker { + "from-skill-bundle-dir" => { + compute_bundle_content_hash(&directory_bundle(case_skill_bundle_dir(case))).unwrap() + } + "from-skill-files" => compute_bundle_content_hash(&files_bundle(skill_files( + case.options["skillFiles"].as_array().unwrap(), + ))) + .unwrap(), + "from-github-bundle" => { + compute_bundle_content_hash(&files_bundle(github_skill_files(case))).unwrap() + } + _ => marker.to_string(), + } +} + fn write_fixture_file(path: &Path, value: &Value) { fs::create_dir_all(path.parent().unwrap()).unwrap(); if let Some(text) = value.as_str() { @@ -511,6 +519,14 @@ fn skill_bundle_from_options(options: &Map) -> SkillBundle { if let Some(dir) = options.get("skillBundleDir").and_then(Value::as_str) { return directory_bundle(repo_path(dir)); } + if let Some(bundle) = options.get("githubBundle").and_then(Value::as_object) { + return github_bundle(GitHubBundleOptions { + owner: bundle["owner"].as_str().unwrap().to_string(), + repo: bundle["repo"].as_str().unwrap().to_string(), + path: bundle["path"].as_str().unwrap().to_string(), + ref_name: bundle["ref"].as_str().unwrap().to_string(), + }); + } files_bundle(Vec::new()) } @@ -528,6 +544,108 @@ fn skill_files(values: &[Value]) -> Vec { .collect() } +fn github_skill_files(case: &GoldenCase) -> Vec { + let bundle = case.options["githubBundle"].as_object().unwrap(); + let root = format!("{}/", bundle["path"].as_str().unwrap().trim_matches('/')); + let files = case.given["github"]["files"].as_object().unwrap(); + files + .iter() + .filter_map(|(path, contents)| { + path.strip_prefix(&root).map(|relative| SkillFile { + path: relative.to_string(), + contents: contents.as_str().unwrap().as_bytes().to_vec(), + mode: None, + }) + }) + .collect() +} + +fn start_github_fixture(github: &Map) { + let owner = github["owner"].as_str().unwrap().to_string(); + let repo = github["repo"].as_str().unwrap().to_string(); + let ref_name = github["ref"].as_str().unwrap().to_string(); + let commit = github["commit"].as_str().unwrap().to_string(); + let tree_sha = github["treeSha"].as_str().unwrap().to_string(); + let files = github["files"].as_object().unwrap().clone(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let base = format!("http://{}", addr); + std::env::set_var("KITUP_GITHUB_API_BASE_URL", &base); + std::env::set_var("KITUP_GITHUB_RAW_BASE_URL", &base); + thread::spawn(move || { + for stream in listener.incoming().flatten() { + handle_github_fixture_connection( + stream, &owner, &repo, &ref_name, &commit, &tree_sha, &files, + ); + } + }); +} + +fn handle_github_fixture_connection( + mut stream: TcpStream, + owner: &str, + repo: &str, + ref_name: &str, + commit: &str, + tree_sha: &str, + files: &Map, +) { + let mut buffer = [0; 4096]; + let Ok(size) = stream.read(&mut buffer) else { + return; + }; + let request = String::from_utf8_lossy(&buffer[..size]); + let path = request + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .unwrap_or("/"); + let path = path.split('?').next().unwrap_or(path); + let commit_path = format!("/repos/{owner}/{repo}/commits/{ref_name}"); + let tree_path = format!("/repos/{owner}/{repo}/git/trees/{tree_sha}"); + if path == commit_path { + write_http_json( + &mut stream, + json!({ "sha": commit, "commit": { "tree": { "sha": tree_sha } } }), + ); + return; + } + if path == tree_path { + let tree: Vec<_> = files + .keys() + .map(|path| { + json!({ + "path": path, + "type": "blob", + "mode": if path.ends_with(".sh") { "100755" } else { "100644" } + }) + }) + .collect(); + write_http_json(&mut stream, json!({ "tree": tree })); + return; + } + let raw_prefix = format!("/{owner}/{repo}/{commit}/"); + if let Some(file) = path.strip_prefix(&raw_prefix) { + if let Some(contents) = files.get(file).and_then(Value::as_str) { + write_http(&mut stream, "200 OK", "application/octet-stream", contents); + return; + } + } + write_http(&mut stream, "404 Not Found", "text/plain", "not found"); +} + +fn write_http_json(stream: &mut TcpStream, value: Value) { + write_http(stream, "200 OK", "application/json", &value.to_string()); +} + +fn write_http(stream: &mut TcpStream, status: &str, content_type: &str, body: &str) { + let response = format!( + "HTTP/1.1 {status}\r\ncontent-type: {content_type}\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", + body.len() + ); + let _ = std::io::Write::write_all(stream, response.as_bytes()); +} + fn scope(value: &str) -> Scope { match value { "user" => Scope::User, diff --git a/scripts/check.mjs b/scripts/check.mjs index 0852910..37d82c8 100755 --- a/scripts/check.mjs +++ b/scripts/check.mjs @@ -160,6 +160,10 @@ function validateCases(cases, hosts) { "workflow-many-detected-yes-installs", "workflow-non-tty-no-agent-error", "workflow-zero-detected-yes-error", + "github-bundle-install", + "github-bundle-dry-run", + "github-bundle-resolve-failure", + "github-bundle-unchanged", ]) { assert(caseIds.has(id), `missing golden case: ${id}`); } diff --git a/testdata/cases/bundled-skill-install.json b/testdata/cases/bundled-skill-install.json index c51928a..06b67d8 100644 --- a/testdata/cases/bundled-skill-install.json +++ b/testdata/cases/bundled-skill-install.json @@ -1926,6 +1926,233 @@ ] } } + }, + { + "id": "github-bundle-install", + "operation": "install", + "description": "Installs a public GitHub bundle with provenance metadata while keeping the user-facing install flow unchanged.", + "options": { + "appId": "example-cli", + "githubBundle": { + "owner": "acme", + "repo": "mycli-skills", + "path": "skills/github-basic", + "ref": "main" + }, + "scope": "user", + "agents": [ + "codex" + ], + "home": "$HOME", + "cwd": "$WORKSPACE" + }, + "given": { + "dirs": [ + "$HOME/.codex" + ], + "files": {}, + "github": { + "owner": "acme", + "repo": "mycli-skills", + "ref": "main", + "commit": "abc123", + "treeSha": "tree123", + "files": { + "skills/github-basic/SKILL.md": "---\nname: github-basic\ndescription: GitHub sourced skill.\n---\n", + "skills/github-basic/references/guide.md": "GitHub guide.\n", + "other/SKILL.md": "---\nname: ignored\ndescription: Ignored skill.\n---\n" + } + } + }, + "expected": { + "filesPresent": [ + "$HOME/.agents/skills/github-basic/SKILL.md", + "$HOME/.agents/skills/github-basic/references/guide.md" + ], + "filesAbsent": [ + "$HOME/.agents/skills/github-basic/other/SKILL.md" + ], + "metadata": { + "path": "$HOME/.agents/skills/github-basic/.kitup.json", + "hash": "from-github-bundle", + "fields": { + "appId": "example-cli", + "skillName": "github-basic", + "source": "github", + "sourceId": "github:acme/mycli-skills/skills/github-basic", + "version": "main", + "provenance": { + "owner": "acme", + "repo": "mycli-skills", + "path": "skills/github-basic", + "ref": "main", + "resolvedCommit": "abc123" + } + } + } + } + }, + { + "id": "github-bundle-dry-run", + "operation": "run-install-workflow", + "description": "Plans a GitHub bundle install without writing files in dry-run mode.", + "options": { + "appId": "example-cli", + "githubBundle": { + "owner": "acme", + "repo": "mycli-skills", + "path": "skills/github-basic", + "ref": "main" + }, + "scope": "user", + "agents": [ + "codex" + ], + "yes": true, + "dryRun": true, + "home": "$HOME", + "cwd": "$WORKSPACE" + }, + "given": { + "dirs": [ + "$HOME/.codex" + ], + "files": {}, + "github": { + "owner": "acme", + "repo": "mycli-skills", + "ref": "main", + "commit": "abc123", + "treeSha": "tree123", + "files": { + "skills/github-basic/SKILL.md": "---\nname: github-basic\ndescription: GitHub sourced skill.\n---\n" + } + } + }, + "expected": { + "workflow": { + "canceled": false, + "dryRun": true + }, + "filesAbsent": [ + "$HOME/.agents/skills/github-basic/SKILL.md", + "$HOME/.agents/skills/github-basic/.kitup.json" + ] + } + }, + { + "id": "github-bundle-resolve-failure", + "operation": "install", + "description": "Reports a structured error when the configured GitHub bundle path has no files.", + "options": { + "appId": "example-cli", + "githubBundle": { + "owner": "acme", + "repo": "mycli-skills", + "path": "skills/missing", + "ref": "main" + }, + "scope": "user", + "agents": [ + "codex" + ], + "home": "$HOME", + "cwd": "$WORKSPACE" + }, + "given": { + "dirs": [ + "$HOME/.codex" + ], + "files": {}, + "github": { + "owner": "acme", + "repo": "mycli-skills", + "ref": "main", + "commit": "abc123", + "treeSha": "tree123", + "files": { + "skills/github-basic/SKILL.md": "---\nname: github-basic\ndescription: GitHub sourced skill.\n---\n" + } + } + }, + "expected": { + "report": { + "installed": [], + "updated": [], + "skipped": [], + "conflicts": [], + "errors": [ + { + "reason": "bundle-resolve-failed" + } + ] + }, + "filesAbsent": [ + "$HOME/.agents/skills/github-basic/SKILL.md" + ] + } + }, + { + "id": "github-bundle-unchanged", + "operation": "install", + "description": "Skips a GitHub bundle install when kitup ownership and content hash are unchanged.", + "options": { + "appId": "example-cli", + "githubBundle": { + "owner": "acme", + "repo": "mycli-skills", + "path": "skills/github-basic", + "ref": "main" + }, + "scope": "user", + "agents": [ + "codex" + ], + "home": "$HOME", + "cwd": "$WORKSPACE" + }, + "given": { + "dirs": [ + "$HOME/.codex" + ], + "files": {}, + "metadata": { + "path": "$HOME/.agents/skills/github-basic/.kitup.json", + "hash": "from-github-bundle", + "fields": { + "schemaVersion": 1, + "appId": "example-cli", + "skillName": "github-basic", + "source": "github" + } + }, + "github": { + "owner": "acme", + "repo": "mycli-skills", + "ref": "main", + "commit": "abc123", + "treeSha": "tree123", + "files": { + "skills/github-basic/SKILL.md": "---\nname: github-basic\ndescription: GitHub sourced skill.\n---\n" + } + } + }, + "expected": { + "report": { + "installed": [], + "updated": [], + "skipped": [ + { + "hostId": "codex", + "skillName": "github-basic", + "targetDir": "$HOME/.agents/skills/github-basic", + "reason": "unchanged" + } + ], + "conflicts": [], + "errors": [] + } + } } ] } diff --git a/ts/src/index.ts b/ts/src/index.ts index 1a6c367..91c890e 100644 --- a/ts/src/index.ts +++ b/ts/src/index.ts @@ -68,7 +68,16 @@ export interface SkillFile { } export type SkillBundle = - { kind: "directory"; path: string } | { kind: "files"; files: SkillFile[] }; + | { kind: "directory"; path: string } + | { kind: "files"; files: SkillFile[] } + | { kind: "github"; options: GitHubBundleOptions }; + +export interface GitHubBundleOptions { + owner: string; + repo: string; + path: string; + ref: string; +} export interface InstallOptions extends BaseOptions { appId: string; @@ -157,7 +166,11 @@ export type UnsupportedScopeError = { reason: "unsupported-scope"; }; export type SkillError = { reason: SkillInfo["errorCode"] }; -export type TargetError = UnknownHostError | UnsupportedScopeError | SkillError; +export type BundleError = { + reason: "bundle-resolve-failed"; +}; +export type TargetError = + UnknownHostError | UnsupportedScopeError | SkillError | BundleError; export interface InstallReport { installed: TargetResult[]; @@ -215,11 +228,13 @@ interface InstallMetadata { schemaVersion: 1; appId: string; skillName: string; - source: "bundled"; + source: "bundled" | "github"; hash: string; + sourceId?: string; + version?: string; + provenance?: Record; } -const source = "bundled"; const defaultAgents: AgentSelector = "auto"; interface BundleFile { @@ -234,6 +249,13 @@ interface NormalizedSkillBundle { byPath: Map; } +interface BundleMetadata { + source: InstallMetadata["source"]; + sourceId?: string; + version?: string; + provenance?: Record; +} + export function directoryBundle(path: string): SkillBundle { return { kind: "directory", path }; } @@ -242,6 +264,10 @@ export function filesBundle(files: SkillFile[]): SkillBundle { return { kind: "files", files }; } +export function githubBundle(options: GitHubBundleOptions): SkillBundle { + return { kind: "github", options }; +} + export function parseInstallFlags( flags: InstallFlagValues, ): ParsedInstallFlags { @@ -770,11 +796,118 @@ async function readSkillBundle( bundle: SkillBundle, cwd = process.cwd(), ): Promise { + return (await resolveSkillBundle(bundle, cwd)).bundle; +} + +async function resolveSkillBundle( + bundle: SkillBundle, + cwd = process.cwd(), +): Promise<{ bundle: NormalizedSkillBundle; metadata: BundleMetadata }> { if (bundle.kind === "directory") { const dir = resolvePath(bundle.path, cwd); - return normalizeSkillFiles(await readDirectoryBundleFiles(dir), dir); + return { + bundle: normalizeSkillFiles(await readDirectoryBundleFiles(dir), dir), + metadata: { source: "bundled" }, + }; + } + if (bundle.kind === "files") { + return { + bundle: normalizeSkillFiles(bundle.files), + metadata: { source: "bundled" }, + }; } - return normalizeSkillFiles(bundle.files); + return resolveGitHubBundle(bundle.options); +} + +async function resolveGitHubBundle( + options: GitHubBundleOptions, +): Promise<{ bundle: NormalizedSkillBundle; metadata: BundleMetadata }> { + const root = trimGitHubPath(options.path); + if (!options.owner || !options.repo || !root || !options.ref) { + throw new Error("invalid github bundle"); + } + const apiBase = envBaseUrl( + "KITUP_GITHUB_API_BASE_URL", + "https://api.github.com", + ); + const rawBase = envBaseUrl( + "KITUP_GITHUB_RAW_BASE_URL", + "https://raw.githubusercontent.com", + ); + const commit: any = await getJson( + `${apiBase}/repos/${encodePathPart(options.owner)}/${encodePathPart(options.repo)}/commits/${encodePathPart(options.ref)}`, + ); + const resolvedCommit = String(commit.sha ?? ""); + const treeSha = String(commit.commit?.tree?.sha ?? ""); + if (!resolvedCommit || !treeSha) throw new Error("invalid github commit"); + + const tree: any = await getJson( + `${apiBase}/repos/${encodePathPart(options.owner)}/${encodePathPart(options.repo)}/git/trees/${encodePathPart(treeSha)}?recursive=1`, + ); + const files: SkillFile[] = []; + const prefix = `${root}/`; + for (const item of tree.tree ?? []) { + const path = String(item.path ?? ""); + if (item.type !== "blob" || !path.startsWith(prefix)) continue; + const relativePath = path.slice(prefix.length); + const url = `${rawBase}/${encodePathPart(options.owner)}/${encodePathPart(options.repo)}/${encodePathPart(resolvedCommit)}/${encodePath(path)}`; + files.push({ + path: relativePath, + contents: await getBytes(url), + mode: item.mode === "100755" ? 0o755 : 0o644, + }); + } + if (files.length === 0) throw new Error("github bundle path not found"); + const bundle = normalizeSkillFiles(files); + return { + bundle, + metadata: { + source: "github", + sourceId: `github:${options.owner}/${options.repo}/${root}`, + version: options.ref, + provenance: { + owner: options.owner, + repo: options.repo, + path: root, + ref: options.ref, + resolvedCommit, + }, + }, + }; +} + +function envBaseUrl(name: string, fallback: string) { + return (process.env[name] ?? fallback).replace(/\/+$/, ""); +} + +async function getJson(url: string) { + const response = await fetch(url, { + headers: { "User-Agent": "kitup" }, + signal: AbortSignal.timeout(30_000), + }); + if (!response.ok) throw new Error(`github request failed: ${url}`); + return response.json(); +} + +async function getBytes(url: string) { + const response = await fetch(url, { + headers: { "User-Agent": "kitup" }, + signal: AbortSignal.timeout(30_000), + }); + if (!response.ok) throw new Error(`github download failed: ${url}`); + return new Uint8Array(await response.arrayBuffer()); +} + +function trimGitHubPath(path: string) { + return path.replace(/^\/+|\/+$/g, ""); +} + +function encodePath(path: string) { + return path.split("/").map(encodePathPart).join("/"); +} + +function encodePathPart(part: string) { + return encodeURIComponent(part); } async function readDirectoryBundleFiles( @@ -857,10 +990,21 @@ async function installOrPlan( ): Promise { const cwd = options.cwd ?? process.cwd(); let bundle: NormalizedSkillBundle; + let bundleMetadata: BundleMetadata; try { - bundle = await readSkillBundle(options.skillBundle, cwd); + ({ bundle, metadata: bundleMetadata } = await resolveSkillBundle( + options.skillBundle, + cwd, + )); } catch { - return emptyInstallReport([{ reason: "invalid-skill-bundle" }]); + return emptyInstallReport([ + { + reason: + options.skillBundle.kind === "github" + ? "bundle-resolve-failed" + : "invalid-skill-bundle", + }, + ]); } const skill = validateNormalizedSkill(bundle); if (!skill.valid || !skill.skillName) { @@ -885,6 +1029,7 @@ async function installOrPlan( options.appId, skill.skillName, hash, + bundleMetadata, ); } report.installed.push(result); @@ -902,6 +1047,7 @@ async function installOrPlan( options.appId, skill.skillName, hash, + bundleMetadata, ); } report.updated.push(result); @@ -955,10 +1101,11 @@ async function copyManagedSkill( appId: string, skillName: string, hash: string, + metadata: BundleMetadata, ) { await rm(targetDir, { recursive: true, force: true }); await copySkillBundle(bundle, targetDir); - await writeMetadata(targetDir, appId, skillName, hash); + await writeMetadata(targetDir, appId, skillName, hash, metadata); } async function replaceManagedSkill( @@ -967,13 +1114,14 @@ async function replaceManagedSkill( appId: string, skillName: string, hash: string, + metadata: BundleMetadata, ) { const suffix = `.kitup-${process.pid}-${Date.now()}`; const tmp = `${targetDir}${suffix}`; const backup = `${targetDir}${suffix}-backup`; await rm(tmp, { recursive: true, force: true }); await copySkillBundle(bundle, tmp); - await writeMetadata(tmp, appId, skillName, hash); + await writeMetadata(tmp, appId, skillName, hash, metadata); try { await rename(targetDir, backup); await rename(tmp, targetDir); @@ -1000,10 +1148,21 @@ async function writeMetadata( appId: string, skillName: string, hash: string, + metadata: BundleMetadata, ) { + const value: InstallMetadata = { + schemaVersion: 1, + appId, + skillName, + source: metadata.source, + hash, + }; + if (metadata.sourceId) value.sourceId = metadata.sourceId; + if (metadata.version) value.version = metadata.version; + if (metadata.provenance) value.provenance = metadata.provenance; await writeFile( join(targetDir, ".kitup.json"), - `${JSON.stringify({ schemaVersion: 1, appId, skillName, source, hash }, null, 2)}\n`, + `${JSON.stringify(value, null, 2)}\n`, ); } diff --git a/ts/test/golden.test.ts b/ts/test/golden.test.ts index c26a6c7..e9ae0d7 100644 --- a/ts/test/golden.test.ts +++ b/ts/test/golden.test.ts @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { createServer } from "node:http"; import { mkdtemp, mkdir, @@ -17,6 +18,7 @@ import { detectHosts, directoryBundle, filesBundle, + githubBundle, installBundledSkill, loadHostSpec, planBundledSkill, @@ -42,11 +44,13 @@ for (const testCase of cases) { await mkdir(home, { recursive: true }); await mkdir(workspace, { recursive: true }); + const cleanups: Array<() => Promise | void> = []; try { - await setupGiven(testCase, home, workspace); + await setupGiven(testCase, home, workspace, cleanups); await runCase(testCase, home, workspace); passed++; } finally { + for (const cleanup of cleanups.reverse()) await cleanup(); await rm(root, { recursive: true, force: true }); } } @@ -167,7 +171,12 @@ async function runCase(testCase: any, home: string, workspace: string) { await assertExpectedMetadata(testCase, home, workspace); } -async function setupGiven(testCase: any, home: string, workspace: string) { +async function setupGiven( + testCase: any, + home: string, + workspace: string, + cleanups: Array<() => Promise | void>, +) { for (const dir of testCase.given.dirs ?? []) { await mkdir(expandValue(dir, home, workspace), { recursive: true }); } @@ -190,6 +199,9 @@ async function setupGiven(testCase: any, home: string, workspace: string) { workspace, testCase.given.metadata, ); + + if (testCase.given.github) + await startGitHubFixture(testCase.given.github, cleanups); } async function assertExpectedFiles( @@ -216,16 +228,7 @@ async function assertExpectedMetadata( const actual = JSON.parse(await readFile(path, "utf8")); for (const [key, value] of Object.entries(expected.fields)) assert.deepEqual(actual[key], value); - const expectedHash = - expected.hash === "from-skill-bundle-dir" - ? await computeBundleContentHash( - directoryBundle(resolveRepoPath(testCase.options.skillBundleDir)), - ) - : expected.hash === "from-skill-files" - ? await computeBundleContentHash( - filesBundle(testCase.options.skillFiles), - ) - : expected.hash; + const expectedHash = await expectedBundleHash(testCase, expected.hash); assert.equal(actual.hash, expectedHash); } @@ -253,15 +256,85 @@ async function writeMetadataFixture( metadata: any, ) { const path = expandValue(metadata.path, home, workspace); - const hash = - metadata.hash === "from-skill-bundle-dir" - ? await computeBundleContentHash( - directoryBundle(caseSkillBundleDir(testCase)), - ) - : metadata.hash; + const hash = await expectedBundleHash(testCase, metadata.hash); await writeFixtureFile(path, { ...metadata.fields, hash }); } +async function expectedBundleHash(testCase: any, marker: string) { + if (marker === "from-skill-bundle-dir") + return computeBundleContentHash( + directoryBundle(caseSkillBundleDir(testCase)), + ); + if (marker === "from-skill-files") + return computeBundleContentHash(filesBundle(testCase.options.skillFiles)); + if (marker === "from-github-bundle") + return computeBundleContentHash(filesBundle(githubSkillFiles(testCase))); + return marker; +} + +async function startGitHubFixture( + github: any, + cleanups: Array<() => Promise | void>, +) { + const previousApi = process.env.KITUP_GITHUB_API_BASE_URL; + const previousRaw = process.env.KITUP_GITHUB_RAW_BASE_URL; + const server = createServer((request, response) => { + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + const path = decodeURIComponent(url.pathname); + const commitPath = `/repos/${github.owner}/${github.repo}/commits/${github.ref}`; + const treePath = `/repos/${github.owner}/${github.repo}/git/trees/${github.treeSha}`; + if (path === commitPath) { + writeJson(response, { + sha: github.commit, + commit: { tree: { sha: github.treeSha } }, + }); + return; + } + if (path === treePath) { + writeJson(response, { + tree: Object.keys(github.files).map((file) => ({ + path: file, + type: "blob", + mode: file.endsWith(".sh") ? "100755" : "100644", + })), + }); + return; + } + const rawPrefix = `/${github.owner}/${github.repo}/${github.commit}/`; + if (path.startsWith(rawPrefix)) { + const file = path.slice(rawPrefix.length); + if (github.files[file] !== undefined) { + response.writeHead(200, { "content-type": "application/octet-stream" }); + response.end(github.files[file]); + return; + } + } + response.writeHead(404); + response.end("not found"); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("bad server"); + const base = `http://127.0.0.1:${address.port}`; + process.env.KITUP_GITHUB_API_BASE_URL = base; + process.env.KITUP_GITHUB_RAW_BASE_URL = base; + cleanups.push(async () => { + restoreEnv("KITUP_GITHUB_API_BASE_URL", previousApi); + restoreEnv("KITUP_GITHUB_RAW_BASE_URL", previousRaw); + await new Promise((resolve) => server.close(() => resolve())); + }); +} + +function writeJson(response: any, value: any) { + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify(value)); +} + +function restoreEnv(name: string, value: string | undefined) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; +} + async function writeFixtureFile(path: string, value: any) { await mkdir(dirname(path), { recursive: true }); await writeFile( @@ -300,6 +373,8 @@ function expandOptions(options: any, home: string, workspace: string) { ); if (expanded.skillFiles) expanded.skillBundle = filesBundle(expanded.skillFiles); + if (expanded.githubBundle) + expanded.skillBundle = githubBundle(expanded.githubBundle); return expanded; } @@ -330,6 +405,17 @@ function caseSkillBundleDir(testCase: any) { ); } +function githubSkillFiles(testCase: any) { + const root = testCase.options.githubBundle.path.replace(/^\/+|\/+$/g, ""); + const prefix = `${root}/`; + return Object.entries(testCase.given.github.files) + .filter(([path]) => path.startsWith(prefix)) + .map(([path, contents]) => ({ + path: path.slice(prefix.length), + contents: contents as string, + })); +} + function assertSelection(actual: any, expected: any) { const normalized = { ...actual }; if (expected.selectedCount !== undefined) {