diff --git a/README.md b/README.md index 294868bd..5ec39216 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![npm downloads](https://img.shields.io/npm/dm/skilld?color=yellow)](https://npm.chart.dev/skilld) [![license](https://img.shields.io/npm/l/skilld?color=yellow)](https://github.com/harlan-zw/skilld/blob/main/LICENSE) -> Generate AI agent skills from your NPM dependencies. +> Generate AI agent skills from your NPM dependencies and Rust crates. ## Why? @@ -116,6 +116,9 @@ skilld # Add skills for specific package(s) skilld add vue nuxt pinia +# Add skills for a Rust crate +skilld add crate:serde + # Update outdated skills skilld update skilld update tailwindcss @@ -148,7 +151,7 @@ skilld config | Command | Description | |---------|-------------| | `skilld` | Interactive wizard (first run) or status menu (existing skills) | -| `skilld add ` | Add skills for package(s), space or comma-separated | +| `skilld add ` | Add skills for package(s), space/comma-separated (`npm`, `crate:`, or `owner/repo`) | | `skilld update [pkg]` | Update outdated skills (all or specific) | | `skilld search ` | Search indexed docs (`-p` to filter by package) | | `skilld list` | List installed skills (`--json` for machine-readable output) | diff --git a/src/commands/sync-shared.ts b/src/commands/sync-shared.ts index b4be4b38..1a76c3d0 100644 --- a/src/commands/sync-shared.ts +++ b/src/commands/sync-shared.ts @@ -1076,6 +1076,7 @@ export async function selectLlmConfig(presetModel?: OptimizeModel, message?: str export interface EnhanceOptions { packageName: string + cachePackageName?: string version: string skillDir: string dirName?: string @@ -1099,7 +1100,7 @@ export interface EnhanceOptions { } export async function enhanceSkillWithLLM(opts: EnhanceOptions): Promise { - const { packageName, version, skillDir, dirName, model, resolved, relatedSkills, hasIssues, hasDiscussions, hasReleases, hasChangelog, docsType, hasShippedDocs: shippedDocs, pkgFiles, force, debug, sections, customPrompt, packages, features, eject } = opts + const { packageName, cachePackageName, version, skillDir, dirName, model, resolved, relatedSkills, hasIssues, hasDiscussions, hasReleases, hasChangelog, docsType, hasShippedDocs: shippedDocs, pkgFiles, force, debug, sections, customPrompt, packages, features, eject } = opts // Eject mode: search index isn't built, so don't include search hints in prompts const effectiveFeatures = eject && features ? { ...features, search: false } as FeaturesConfig : features @@ -1108,7 +1109,7 @@ export async function enhanceSkillWithLLM(opts: EnhanceOptions): Promise { const docFiles = listReferenceFiles(skillDir) const hasGithub = hasIssues || hasDiscussions const { optimized, wasOptimized, usage, cost, warnings, error, debugLogsDir } = await optimizeDocs({ - packageName, + packageName: cachePackageName || packageName, skillDir, model, version, diff --git a/src/commands/sync.ts b/src/commands/sync.ts index 894b55d7..4c9c75f3 100644 --- a/src/commands/sync.ts +++ b/src/commands/sync.ts @@ -1,5 +1,5 @@ import type { AgentType, OptimizeModel } from '../agent/index.ts' -import type { ProjectState } from '../core/skills.ts' +import type { ProjectState, SkillEntry } from '../core/skills.ts' import type { GitSkillSource } from '../sources/git-skills.ts' import type { ResolveAttempt } from '../sources/index.ts' import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' @@ -37,6 +37,7 @@ import { fetchPkgDist, parsePackageSpec, readLocalDependencies, + resolveCrateDocsWithAttempts, resolvePackageDocsWithAttempts, searchNpmPackages, } from '../sources/index.ts' @@ -95,13 +96,44 @@ export interface SyncOptions { from?: string } +export function isCrateSpec(spec: string): boolean { + return spec.startsWith('crate:') +} + +function toStoragePackageName(packageName: string, isCrate: boolean): string { + return isCrate ? `@skilld-crate/${packageName}` : packageName +} + +function toIdentityPackageName(packageName: string, isCrate: boolean): string { + return isCrate ? `crate:${packageName}` : packageName +} + +function storageNameFromIdentity(identityName: string): string { + if (identityName.startsWith('crate:')) + return toStoragePackageName(identityName.slice('crate:'.length), true) + return identityName +} + +function toUpdatePackageSpec(skill: SkillEntry): string { + const packageName = skill.packageName || skill.name + if (packageName.startsWith('crate:')) + return packageName + + if (skill.name.includes('skilld-crate-') || skill.info?.source?.includes('docs.rs/')) + return `crate:${packageName}` + + return packageName +} + export async function syncCommand(state: ProjectState, opts: SyncOptions): Promise { // If packages specified, sync those if (opts.packages && opts.packages.length > 0) { - // Use parallel sync for multiple packages - if (opts.packages.length > 1) { - return syncPackagesParallel({ - packages: opts.packages, + const crateSpecs = opts.packages.filter(isCrateSpec) + const npmSpecs = opts.packages.filter(pkg => !isCrateSpec(pkg)) + + if (npmSpecs.length > 1) { + await syncPackagesParallel({ + packages: npmSpecs, global: opts.global, agent: opts.agent, model: opts.model, @@ -111,9 +143,13 @@ export async function syncCommand(state: ProjectState, opts: SyncOptions): Promi mode: opts.mode, }) } + else if (npmSpecs.length === 1) { + await syncSinglePackage(npmSpecs[0]!, opts) + } + + for (const spec of crateSpecs) + await syncSinglePackage(spec, opts) - // Single package - use original flow for cleaner output - await syncSinglePackage(opts.packages[0]!, opts) return } @@ -238,80 +274,95 @@ interface SyncConfig { } async function syncSinglePackage(packageSpec: string, config: SyncConfig): Promise { + const isCrate = isCrateSpec(packageSpec) + const normalizedSpec = isCrate ? packageSpec.slice('crate:'.length).trim() : packageSpec + if (!normalizedSpec) { + p.log.error('Invalid crate spec. Use format: crate:') + return + } + // Parse dist-tag from spec: "vue@beta" → name="vue", tag="beta" - const { name: packageName, tag: requestedTag } = parsePackageSpec(packageSpec) + const { name: parsedName, tag: requestedTag } = parsePackageSpec(normalizedSpec) + const packageName = isCrate ? parsedName.toLowerCase() : parsedName + const identityPackageName = toIdentityPackageName(packageName, isCrate) + const storagePackageName = toStoragePackageName(packageName, isCrate) const spin = timedSpinner() spin.start(`Resolving ${packageSpec}`) const cwd = process.cwd() - const localDeps = await readLocalDependencies(cwd).catch(() => []) - const localVersion = localDeps.find(d => d.name === packageName)?.version - - // Try npm first — use full spec for npm resolution (unpkg supports dist-tags) - const resolveResult = await resolvePackageDocsWithAttempts(requestedTag ? packageSpec : packageName, { - version: localVersion, - cwd, - onProgress: step => spin.message(`${packageName}: ${RESOLVE_STEP_LABELS[step]}`), - }) + const localDeps = isCrate ? [] : await readLocalDependencies(cwd).catch(() => []) + const localVersion = isCrate ? undefined : localDeps.find(d => d.name === packageName)?.version + + const resolveResult = isCrate + ? await resolveCrateDocsWithAttempts(packageName, { + version: requestedTag, + onProgress: step => spin.message(`${identityPackageName}: ${step}`), + }) + : await resolvePackageDocsWithAttempts(requestedTag ? normalizedSpec : packageName, { + version: localVersion, + cwd, + onProgress: step => spin.message(`${identityPackageName}: ${RESOLVE_STEP_LABELS[step]}`), + }) let resolved = resolveResult.package // If npm fails, check if it's a link: dep and try local resolution - if (!resolved) { + if (!resolved && !isCrate) { spin.message(`Resolving local package: ${packageName}`) resolved = await resolveLocalDep(packageName, cwd) } if (!resolved) { - // Search npm for alternatives before giving up - spin.message(`Searching npm for "${packageName}"...`) - const suggestions = await searchNpmPackages(packageName) - - if (suggestions.length > 0) { - spin.stop(`Package "${packageName}" not found on npm`) - showResolveAttempts(resolveResult.attempts) - - const selected = await p.select({ - message: 'Did you mean one of these?', - options: [ - ...suggestions.map(s => ({ - label: s.name, - value: s.name, - hint: s.description, - })), - { label: 'None of these', value: '_none_' as const }, - ], - }) + if (!isCrate) { + // Search npm for alternatives before giving up + spin.message(`Searching npm for "${packageName}"...`) + const suggestions = await searchNpmPackages(packageName) + + if (suggestions.length > 0) { + spin.stop(`Package "${packageName}" not found on npm`) + showResolveAttempts(resolveResult.attempts) + + const selected = await p.select({ + message: 'Did you mean one of these?', + options: [ + ...suggestions.map(s => ({ + label: s.name, + value: s.name, + hint: s.description, + })), + { label: 'None of these', value: '_none_' as const }, + ], + }) + + if (!p.isCancel(selected) && selected !== '_none_') + return syncSinglePackage(selected as string, config) - if (!p.isCancel(selected) && selected !== '_none_') - return syncSinglePackage(selected as string, config) - - return + return + } } - spin.stop(`Could not find docs for: ${packageName}`) + spin.stop(`Could not find docs for: ${identityPackageName}`) showResolveAttempts(resolveResult.attempts) return } - const version = localVersion || resolved.version || 'latest' + const version = isCrate ? (resolved.version || requestedTag || 'latest') : (localVersion || resolved.version || 'latest') const versionKey = getVersionKey(version) // Force: nuke cached references + search index so all existsSync guards re-fetch - if (config.force) { - forceClearCache(packageName, version) - } + if (config.force) + forceClearCache(storagePackageName, version) - const useCache = isCached(packageName, version) + const useCache = isCached(storagePackageName, version) // Download npm dist if not in node_modules (for standalone/learning use) - if (!existsSync(join(cwd, 'node_modules', packageName))) { + if (!isCrate && !existsSync(join(cwd, 'node_modules', packageName))) { spin.message(`Downloading ${packageName}@${version} dist`) await fetchPkgDist(packageName, version) } // Shipped skills: symlink directly, skip all doc fetching/caching/LLM - const shippedResult = handleShippedSkills(packageName, version, cwd, config.agent, config.global) + const shippedResult = isCrate ? null : handleShippedSkills(packageName, version, cwd, config.agent, config.global) if (shippedResult) { const shared = !config.global && getSharedSkillsDir(cwd) for (const shipped of shippedResult.shipped) { @@ -323,12 +374,12 @@ async function syncSinglePackage(packageSpec: string, config: SyncConfig): Promi return } - spin.stop(`Resolved ${packageName}@${useCache ? versionKey : version}${config.force ? ' (force)' : useCache ? ' (cached)' : ''}`) + spin.stop(`Resolved ${identityPackageName}@${useCache ? versionKey : version}${config.force ? ' (force)' : useCache ? ' (cached)' : ''}`) ensureCacheDir() const baseDir = resolveBaseDir(cwd, config.agent, config.global) - const skillDirName = config.name ? sanitizeName(config.name) : computeSkillDirName(packageName) + const skillDirName = config.name ? sanitizeName(config.name) : computeSkillDirName(storagePackageName) // Eject path override: default to ./skills/, or use specified directory const skillDir = config.eject ? typeof config.eject === 'string' @@ -339,18 +390,18 @@ async function syncSinglePackage(packageSpec: string, config: SyncConfig): Promi // ── Merge mode: skill dir already exists with a different primary package (skip in eject) ── const existingLock = config.eject ? undefined : readLock(baseDir)?.skills[skillDirName] - const isMerge = existingLock && existingLock.packageName !== packageName + const isMerge = existingLock && existingLock.packageName !== identityPackageName if (isMerge) { - spin.stop(`Merging ${packageName} into ${skillDirName}`) + spin.stop(`Merging ${identityPackageName} into ${skillDirName}`) // Create named symlink for this package - linkPkgNamed(skillDir, packageName, cwd, version) + linkPkgNamed(skillDir, storagePackageName, cwd, version) // Merge into lockfile const repoSlug = resolved.repoUrl?.match(/github\.com\/([^/]+\/[^/]+?)(?:\.git)?(?:[/#]|$)/)?.[1] writeLock(baseDir, skillDirName, { - packageName, + packageName: identityPackageName, version, repo: repoSlug, source: existingLock.source, @@ -361,9 +412,10 @@ async function syncSinglePackage(packageSpec: string, config: SyncConfig): Promi // Regenerate SKILL.md with all packages listed const updatedLock = readLock(baseDir)?.skills[skillDirName] const allPackages = parsePackages(updatedLock?.packages).map(p => ({ name: p.name })) - const relatedSkills = await findRelatedSkills(packageName, baseDir) - const pkgFiles = getPkgKeyFiles(existingLock.packageName!, cwd, existingLock.version) - const shippedDocs = hasShippedDocs(existingLock.packageName!, cwd, existingLock.version) + const relatedSkills = await findRelatedSkills(storagePackageName, baseDir) + const existingStorageName = storageNameFromIdentity(existingLock.packageName!) + const pkgFiles = getPkgKeyFiles(existingStorageName, cwd, existingLock.version) + const shippedDocs = hasShippedDocs(existingStorageName, cwd, existingLock.version) const mergeFeatures = readConfig().features ?? defaultFeatures const skillMd = generateSkillMd({ @@ -389,7 +441,7 @@ async function syncSinglePackage(packageSpec: string, config: SyncConfig): Promi if (!config.global) registerProject(cwd) - p.outro(`Merged ${packageName} into ${skillDirName}`) + p.outro(`Merged ${identityPackageName} into ${skillDirName}`) return } @@ -399,7 +451,7 @@ async function syncSinglePackage(packageSpec: string, config: SyncConfig): Promi const resSpin = timedSpinner() resSpin.start('Finding resources') const resources = await fetchAndCacheResources({ - packageName, + packageName: storagePackageName, resolved, version, useCache, @@ -424,14 +476,14 @@ async function syncSinglePackage(packageSpec: string, config: SyncConfig): Promi p.log.warn(`\x1B[33m${w}\x1B[0m`) // Create symlinks (LLM needs .skilld/ to read docs, even in eject mode) - linkAllReferences(skillDir, packageName, cwd, version, resources.docsType, undefined, features, resources.repoInfo) + linkAllReferences(skillDir, storagePackageName, cwd, version, resources.docsType, undefined, features, resources.repoInfo) // ── Phase 2: Search index (skip in eject mode — not portable) ── if (features.search && !config.eject) { const idxSpin = timedSpinner() idxSpin.start('Creating search index') await indexResources({ - packageName, + packageName: storagePackageName, version, cwd, docsToIndex: resources.docsToIndex, @@ -441,23 +493,23 @@ async function syncSinglePackage(packageSpec: string, config: SyncConfig): Promi idxSpin.stop('Search index ready') } - const pkgDir = resolvePkgDir(packageName, cwd, version) - const hasChangelog = detectChangelog(pkgDir, getCacheDir(packageName, version)) - const relatedSkills = await findRelatedSkills(packageName, baseDir) - const shippedDocs = hasShippedDocs(packageName, cwd, version) - const pkgFiles = getPkgKeyFiles(packageName, cwd, version) + const pkgDir = resolvePkgDir(storagePackageName, cwd, version) + const hasChangelog = detectChangelog(pkgDir, getCacheDir(storagePackageName, version)) + const relatedSkills = await findRelatedSkills(storagePackageName, baseDir) + const shippedDocs = hasShippedDocs(storagePackageName, cwd, version) + const pkgFiles = getPkgKeyFiles(storagePackageName, cwd, version) // Write base SKILL.md (no LLM needed) const repoSlug = resolved.repoUrl?.match(/github\.com\/([^/]+\/[^/]+?)(?:\.git)?(?:[/#]|$)/)?.[1] // Also create named symlink for this package (skip in eject mode) if (!config.eject) - linkPkgNamed(skillDir, packageName, cwd, version) + linkPkgNamed(skillDir, storagePackageName, cwd, version) // Skip lockfile in eject mode — no agent skills dir to write to if (!config.eject) { writeLock(baseDir, skillDirName, { - packageName, + packageName: identityPackageName, version, repo: repoSlug, source: resources.docSource, @@ -504,6 +556,7 @@ async function syncSinglePackage(packageSpec: string, config: SyncConfig): Promi p.log.step(getModelLabel(llmConfig.model)) await enhanceSkillWithLLM({ packageName, + cachePackageName: storagePackageName, version, skillDir, dirName: skillDirName, @@ -533,7 +586,7 @@ async function syncSinglePackage(packageSpec: string, config: SyncConfig): Promi const skilldDir = join(skillDir, '.skilld') if (existsSync(skilldDir) && !config.debug) rmSync(skilldDir, { recursive: true, force: true }) - ejectReferences(skillDir, packageName, cwd, version, resources.docsType, features, resources.repoInfo) + ejectReferences(skillDir, storagePackageName, cwd, version, resources.docsType, features, resources.repoInfo) } // Skip agent integration in eject mode (no symlinks, no gitignore, no instructions) @@ -555,7 +608,7 @@ async function syncSinglePackage(packageSpec: string, config: SyncConfig): Promi await shutdownWorker() const ejectMsg = isEject ? ' (ejected)' : '' - p.outro(config.mode === 'update' ? `Updated ${packageName}${ejectMsg}` : `Synced ${packageName} to ${relative(cwd, skillDir)}${ejectMsg}`) + p.outro(config.mode === 'update' ? `Updated ${identityPackageName}${ejectMsg}` : `Synced ${identityPackageName} to ${relative(cwd, skillDir)}${ejectMsg}`) } // ── Citty command definitions (lazy-loaded by cli.ts) ── @@ -565,7 +618,7 @@ export const addCommandDef = defineCommand({ args: { package: { type: 'positional', - description: 'Package(s) to sync (space or comma-separated, e.g., vue nuxt pinia)', + description: 'Package(s) to sync (space/comma-separated; npm, crate:, or owner/repo)', required: true, }, skill: { @@ -596,16 +649,15 @@ export const addCommandDef = defineCommand({ .filter(Boolean), )] - // Partition: git sources vs npm packages const gitSources: GitSkillSource[] = [] - const npmTokens: string[] = [] + const packageTokens: string[] = [] for (const input of rawInputs) { const git = parseGitSkillInput(input) if (git) gitSources.push(git) else - npmTokens.push(input) + packageTokens.push(input) } // Handle git sources @@ -616,9 +668,8 @@ export const addCommandDef = defineCommand({ } } - // Handle npm packages via existing flow - if (npmTokens.length > 0) { - const packages = [...new Set(npmTokens.flatMap(s => s.split(/[,\s]+/)).map(s => s.trim()).filter(Boolean))] + if (packageTokens.length > 0) { + const packages = [...new Set(packageTokens.flatMap(s => s.split(/[,\s]+/)).map(s => s.trim()).filter(Boolean))] const state = await getProjectState(cwd) p.intro(introLine({ state })) return syncCommand(state, { @@ -755,7 +806,7 @@ export const updateCommandDef = defineCommand({ return } - const packages = state.outdated.map(s => s.packageName || s.name) + const packages = state.outdated.map(toUpdatePackageSpec) return syncCommand(state, { packages, global: args.global, diff --git a/src/sources/crates.ts b/src/sources/crates.ts new file mode 100644 index 00000000..75fb6744 --- /dev/null +++ b/src/sources/crates.ts @@ -0,0 +1,200 @@ +import type { ResolveAttempt, ResolveResult, ResolvedPackage } from './types.ts' +import { fetchLlmsUrl } from './llms.ts' +import { resolveGitHubRepo } from './github.ts' +import { $fetch, createRateLimitedRunner, isLikelyCodeHostUrl, isUselessDocsUrl, normalizeRepoUrl, parseGitHubUrl } from './utils.ts' + +const VALID_CRATE_NAME = /^[a-z0-9][a-z0-9_-]*$/ +const runCratesApiRateLimited = createRateLimitedRunner(1000) + +interface CratesApiResponse { + crate?: { + id?: string + name?: string + description?: string + homepage?: string | null + documentation?: string | null + repository?: string | null + max_version?: string + newest_version?: string + max_stable_version?: string + default_version?: string + updated_at?: string + } + versions?: Array<{ + num?: string + yanked?: boolean + created_at?: string + description?: string | null + homepage?: string | null + documentation?: string | null + repository?: string | null + }> +} + +function selectCrateVersion( + data: CratesApiResponse, + requestedVersion?: string, +): { version: string, entry?: NonNullable[number] } | null { + const versions = data.versions || [] + + if (requestedVersion) { + const exact = versions.find(v => v.num === requestedVersion && !v.yanked) + if (exact?.num) + return { version: exact.num, entry: exact } + } + + const crate = data.crate + const preferred = [ + crate?.max_stable_version, + crate?.newest_version, + crate?.max_version, + crate?.default_version, + ].find(Boolean) + + if (preferred) { + const match = versions.find(v => v.num === preferred && !v.yanked) + if (match?.num) + return { version: preferred, entry: match } + if (versions.length === 0) + return { version: preferred } + } + + const firstStable = versions.find(v => !v.yanked && v.num) + if (firstStable?.num) + return { version: firstStable.num, entry: firstStable } + + return null +} + +function pickPreferredUrl(...urls: Array): string | undefined { + return urls.map(v => v?.trim()).find(v => !!v) +} + +async function fetchCratesApi(url: string): Promise { + return runCratesApiRateLimited(() => $fetch(url).catch(() => null)) +} + +export async function resolveCrateDocsWithAttempts( + crateName: string, + options: { version?: string, onProgress?: (message: string) => void } = {}, +): Promise { + const attempts: ResolveAttempt[] = [] + const onProgress = options.onProgress + const normalizedName = crateName.trim().toLowerCase() + + if (!normalizedName || !VALID_CRATE_NAME.test(normalizedName)) { + attempts.push({ + source: 'crates', + status: 'error', + message: `Invalid crate name: ${crateName}`, + }) + return { package: null, attempts } + } + + onProgress?.('crates.io metadata') + const apiUrl = `https://crates.io/api/v1/crates/${encodeURIComponent(normalizedName)}` + const data = await fetchCratesApi(apiUrl) + + if (!data?.crate) { + attempts.push({ + source: 'crates', + url: apiUrl, + status: 'not-found', + message: 'Crate not found on crates.io', + }) + return { package: null, attempts } + } + + attempts.push({ + source: 'crates', + url: apiUrl, + status: 'success', + message: `Found crate: ${data.crate.name || normalizedName}`, + }) + + const selected = selectCrateVersion(data, options.version) + if (!selected) { + attempts.push({ + source: 'crates', + url: apiUrl, + status: 'error', + message: 'No usable crate versions found', + }) + return { package: null, attempts } + } + + const version = selected.version + const versionEntry = selected.entry + const docsRsUrl = `https://docs.rs/${encodeURIComponent(normalizedName)}/${encodeURIComponent(version)}` + + const repositoryRaw = pickPreferredUrl(versionEntry?.repository, data.crate.repository) + const homepage = pickPreferredUrl(versionEntry?.homepage, data.crate.homepage) + const documentation = pickPreferredUrl(versionEntry?.documentation, data.crate.documentation) + const normalizedRepo = repositoryRaw ? normalizeRepoUrl(repositoryRaw) : undefined + const repoUrl = normalizedRepo && isLikelyCodeHostUrl(normalizedRepo) + ? normalizedRepo + : isLikelyCodeHostUrl(homepage) + ? homepage + : undefined + + let resolved: ResolvedPackage = { + name: normalizedName, + version, + releasedAt: versionEntry?.created_at || data.crate.updated_at || undefined, + description: versionEntry?.description || data.crate.description, + docsUrl: (() => { + if (documentation && !isUselessDocsUrl(documentation) && !isLikelyCodeHostUrl(documentation)) + return documentation + if (homepage && !isUselessDocsUrl(homepage) && !isLikelyCodeHostUrl(homepage)) + return homepage + return docsRsUrl + })(), + repoUrl, + } + + const gh = repoUrl ? parseGitHubUrl(repoUrl) : null + if (gh) { + onProgress?.('GitHub enrichment') + const ghResolved = await resolveGitHubRepo(gh.owner, gh.repo) + if (ghResolved) { + attempts.push({ + source: 'github-meta', + url: repoUrl, + status: 'success', + message: 'Enriched via GitHub repo metadata', + }) + resolved = { + ...ghResolved, + name: normalizedName, + version, + releasedAt: resolved.releasedAt || ghResolved.releasedAt, + description: resolved.description || ghResolved.description, + docsUrl: resolved.docsUrl || ghResolved.docsUrl, + repoUrl, + readmeUrl: ghResolved.readmeUrl || resolved.readmeUrl, + } + } + else { + attempts.push({ + source: 'github-meta', + url: repoUrl, + status: 'not-found', + message: 'GitHub enrichment failed, using crates.io metadata', + }) + } + } + + if (!resolved.llmsUrl && resolved.docsUrl) { + onProgress?.('llms.txt discovery') + resolved.llmsUrl = await fetchLlmsUrl(resolved.docsUrl).catch(() => null) ?? undefined + if (resolved.llmsUrl) { + attempts.push({ + source: 'llms.txt', + url: resolved.llmsUrl, + status: 'success', + }) + } + } + + return { package: resolved, attempts } +} diff --git a/src/sources/index.ts b/src/sources/index.ts index 303c0ebd..7d215405 100644 --- a/src/sources/index.ts +++ b/src/sources/index.ts @@ -7,6 +7,8 @@ export { fetchBlogReleases } from './blog-releases.ts' // Crawl export { fetchCrawledDocs, toCrawlPattern } from './crawl.ts' +export { resolveCrateDocsWithAttempts } from './crates.ts' + // Discussions export type { GitHubDiscussion } from './discussions.ts' diff --git a/src/sources/npm.ts b/src/sources/npm.ts index 96459565..d1ec31af 100644 --- a/src/sources/npm.ts +++ b/src/sources/npm.ts @@ -13,7 +13,7 @@ import { getCacheDir } from '../cache/version.ts' import { fetchGitDocs, fetchGitHubRepoMeta, fetchReadme, searchGitHubRepo, validateGitDocsWithLlms } from './github.ts' import { fetchLlmsTxt, fetchLlmsUrl } from './llms.ts' import { getCrawlUrl } from './package-registry.ts' -import { $fetch, isGitHubRepoUrl, isUselessDocsUrl, normalizeRepoUrl, parseGitHubUrl, parsePackageSpec } from './utils.ts' +import { $fetch, isGitHubRepoUrl, isUselessDocsUrl, normalizeRepoUrl, parseGitHubUrl, parsePackageSpec, SKILLD_USER_AGENT } from './utils.ts' /** * Search npm registry for packages matching a query. @@ -603,7 +603,7 @@ export async function fetchPkgDist(name: string, version: string): Promise null) if (!tarballRes?.ok || !tarballRes.body) diff --git a/src/sources/types.ts b/src/sources/types.ts index f6e8d119..f99148c7 100644 --- a/src/sources/types.ts +++ b/src/sources/types.ts @@ -65,7 +65,7 @@ export interface FetchedDoc { } export interface ResolveAttempt { - source: 'npm' | 'github-docs' | 'github-meta' | 'github-search' | 'llms.txt' | 'readme' + source: 'npm' | 'crates' | 'github-docs' | 'github-meta' | 'github-search' | 'llms.txt' | 'readme' url?: string status: 'success' | 'not-found' | 'error' message?: string diff --git a/src/sources/utils.ts b/src/sources/utils.ts index 161b6ba7..9fa30756 100644 --- a/src/sources/utils.ts +++ b/src/sources/utils.ts @@ -4,13 +4,37 @@ import { ofetch } from 'ofetch' +export const SKILLD_USER_AGENT = 'skilld/1.0 (+https://github.com/harlan-zw/skilld)' + export const $fetch = ofetch.create({ retry: 3, - retryDelay: 500, + retryDelay: 1000, + retryStatusCodes: [408, 429, 500, 502, 503, 504], timeout: 15_000, - headers: { 'User-Agent': 'skilld/1.0' }, + headers: { 'User-Agent': SKILLD_USER_AGENT }, }) +export function createRateLimitedRunner(intervalMs: number): (task: () => Promise) => Promise { + let queue: Promise = Promise.resolve() + let lastRunAt = 0 + + return async function runRateLimited(task: () => Promise): Promise { + const run = async (): Promise => { + const elapsed = Date.now() - lastRunAt + const waitMs = intervalMs - elapsed + if (waitMs > 0) + await new Promise(resolve => setTimeout(resolve, waitMs)) + + lastRunAt = Date.now() + return task() + } + + const request = queue.then(run, run) + queue = request.then(() => undefined, () => undefined) + return request + } +} + /** * Fetch text content from URL */ @@ -65,6 +89,18 @@ export function isGitHubRepoUrl(url: string): boolean { } } +export function isLikelyCodeHostUrl(url: string | undefined): boolean { + if (!url) + return false + try { + const parsed = new URL(url) + return ['github.com', 'www.github.com', 'gitlab.com', 'www.gitlab.com'].includes(parsed.hostname) + } + catch { + return false + } +} + /** * Parse owner/repo from GitHub URL */ diff --git a/test/e2e/crate-smoke.test.ts b/test/e2e/crate-smoke.test.ts new file mode 100644 index 00000000..c8126f90 --- /dev/null +++ b/test/e2e/crate-smoke.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest' +import { resolveCrateDocsWithAttempts } from '../../src/sources' + +describe('e2e crate smoke', () => { + it('returns crate resolution result for serde', async () => { + const result = await resolveCrateDocsWithAttempts('serde') + + if (result.package) { + expect(result.package.name).toBe('serde') + expect(result.package.version).toBeTruthy() + expect(result.package.docsUrl).toMatch(/^https:\/\/docs\.rs\/serde(?:\/|$)/) + expect(result.attempts.some(a => a.source === 'crates' && a.status === 'success')).toBe(true) + return + } + + expect(result.attempts.some(a => a.source === 'crates')).toBe(true) + expect(result.attempts.some(a => a.status === 'not-found' || a.status === 'error')).toBe(true) + }, 120_000) +}) diff --git a/test/unit/sources-crates.test.ts b/test/unit/sources-crates.test.ts new file mode 100644 index 00000000..5afb251b --- /dev/null +++ b/test/unit/sources-crates.test.ts @@ -0,0 +1,283 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mockFetch = vi.fn<( + url: string, + opts?: { responseType?: string, method?: string }, +) => Promise<{ ok?: boolean, json?: () => Promise, text?: () => Promise }>>() + +function createMockFetch() { + async function $fetch(url: string, opts?: { responseType?: string, method?: string }): Promise { + const response = await mockFetch(url, opts) + if (!response?.ok) + throw new Error('fetch failed') + if (opts?.responseType === 'text') + return response.text?.() ?? null + return response.json?.() ?? null + } + + $fetch.raw = async (url: string, opts?: { responseType?: string, method?: string }) => { + return mockFetch(url, opts) + } + + return $fetch +} + +vi.mock('ofetch', () => ({ + ofetch: { create: () => createMockFetch() }, +})) + +vi.mock('../../src/sources/github', () => ({ + resolveGitHubRepo: vi.fn(), +})) + +vi.mock('../../src/sources/llms', () => ({ + fetchLlmsUrl: vi.fn(), +})) + +const { resolveCrateDocsWithAttempts } = await import('../../src/sources/crates') + +describe('sources/crates', () => { + beforeEach(() => { + vi.resetAllMocks() + }) + + it('returns error attempt for invalid crate name', async () => { + const result = await resolveCrateDocsWithAttempts('serde!') + + expect(result.package).toBeNull() + expect(result.attempts).toEqual([ + { + source: 'crates', + status: 'error', + message: 'Invalid crate name: serde!', + }, + ]) + }) + + it('returns not-found attempt when crates.io metadata cannot be fetched', async () => { + mockFetch.mockRejectedValueOnce(new Error('network')) + + const result = await resolveCrateDocsWithAttempts('serde') + + expect(result.package).toBeNull() + expect(result.attempts).toEqual([ + { + source: 'crates', + url: 'https://crates.io/api/v1/crates/serde', + status: 'not-found', + message: 'Crate not found on crates.io', + }, + ]) + }) + + it('falls back to docs.rs when documentation/homepage are missing or repo-like', async () => { + const { fetchLlmsUrl } = await import('../../src/sources/llms') + const { resolveGitHubRepo } = await import('../../src/sources/github') + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: () => Promise.resolve({ + crate: { + name: 'serde', + max_version: '1.0.217', + repository: 'https://github.com/serde-rs/serde', + documentation: 'https://github.com/serde-rs/serde', + homepage: 'https://github.com/serde-rs/serde', + updated_at: '2025-01-01T00:00:00Z', + }, + versions: [ + { + num: '1.0.217', + yanked: false, + created_at: '2024-12-20T00:00:00Z', + }, + ], + }), + }) + + vi.mocked(resolveGitHubRepo).mockResolvedValue(null) + vi.mocked(fetchLlmsUrl).mockResolvedValue(null) + + const progress: string[] = [] + const result = await resolveCrateDocsWithAttempts('serde', { + onProgress: step => progress.push(step), + }) + + expect(result.package).toMatchObject({ + name: 'serde', + version: '1.0.217', + docsUrl: 'https://docs.rs/serde/1.0.217', + repoUrl: 'https://github.com/serde-rs/serde', + releasedAt: '2024-12-20T00:00:00Z', + }) + expect(progress).toEqual([ + 'crates.io metadata', + 'GitHub enrichment', + 'llms.txt discovery', + ]) + }) + + it('selects requested non-yanked version and keeps docs.rs versioned URL', async () => { + const { fetchLlmsUrl } = await import('../../src/sources/llms') + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: () => Promise.resolve({ + crate: { + name: 'serde', + max_stable_version: '1.0.220', + }, + versions: [ + { num: '1.0.220', yanked: false, created_at: '2025-01-10T00:00:00Z' }, + { num: '1.0.0', yanked: false, created_at: '2020-01-01T00:00:00Z' }, + ], + }), + }) + + vi.mocked(fetchLlmsUrl).mockResolvedValue(null) + + const result = await resolveCrateDocsWithAttempts('serde', { version: '1.0.0' }) + + expect(result.package).toMatchObject({ + name: 'serde', + version: '1.0.0', + docsUrl: 'https://docs.rs/serde/1.0.0', + }) + }) + + it('falls back from requested yanked version to preferred stable version', async () => { + const { fetchLlmsUrl } = await import('../../src/sources/llms') + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: () => Promise.resolve({ + crate: { + name: 'serde', + max_stable_version: '1.0.220', + newest_version: '1.0.220', + }, + versions: [ + { num: '1.0.220', yanked: false, created_at: '2025-01-10T00:00:00Z' }, + { num: '1.0.200', yanked: true, created_at: '2024-10-01T00:00:00Z' }, + ], + }), + }) + + vi.mocked(fetchLlmsUrl).mockResolvedValue(null) + + const result = await resolveCrateDocsWithAttempts('serde', { version: '1.0.200' }) + + expect(result.package?.version).toBe('1.0.220') + expect(result.package?.releasedAt).toBe('2025-01-10T00:00:00Z') + }) + + it('returns error attempt when crate exists but no usable versions are available', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: () => Promise.resolve({ + crate: { + name: 'serde', + }, + versions: [ + { num: '1.0.220', yanked: true }, + { num: '1.0.200', yanked: true }, + ], + }), + }) + + const result = await resolveCrateDocsWithAttempts('serde') + + expect(result.package).toBeNull() + expect(result.attempts).toContainEqual({ + source: 'crates', + url: 'https://crates.io/api/v1/crates/serde', + status: 'error', + message: 'No usable crate versions found', + }) + }) + + it('enriches metadata from GitHub when repository points to GitHub', async () => { + const { resolveGitHubRepo } = await import('../../src/sources/github') + const { fetchLlmsUrl } = await import('../../src/sources/llms') + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: () => Promise.resolve({ + crate: { + name: 'serde', + max_version: '1.0.220', + repository: 'git+https://github.com/serde-rs/serde.git', + description: 'serde description', + }, + versions: [ + { num: '1.0.220', yanked: false, created_at: '2025-01-10T00:00:00Z' }, + ], + }), + }) + + vi.mocked(resolveGitHubRepo).mockResolvedValue({ + name: 'serde', + version: '1.0.220', + description: 'github description', + docsUrl: 'https://serde.rs', + readmeUrl: 'ungh://serde-rs/serde', + repoUrl: 'https://github.com/serde-rs/serde', + releasedAt: '2025-01-12T00:00:00Z', + }) + vi.mocked(fetchLlmsUrl).mockResolvedValue('https://serde.rs/llms.txt') + + const result = await resolveCrateDocsWithAttempts('serde') + + expect(result.package).toMatchObject({ + name: 'serde', + version: '1.0.220', + description: 'serde description', + docsUrl: 'https://docs.rs/serde/1.0.220', + readmeUrl: 'ungh://serde-rs/serde', + repoUrl: 'https://github.com/serde-rs/serde', + llmsUrl: 'https://serde.rs/llms.txt', + }) + expect(result.attempts).toContainEqual({ + source: 'github-meta', + url: 'https://github.com/serde-rs/serde', + status: 'success', + message: 'Enriched via GitHub repo metadata', + }) + expect(result.attempts).toContainEqual({ + source: 'llms.txt', + url: 'https://serde.rs/llms.txt', + status: 'success', + }) + }) + + it('records github-meta not-found attempt when enrichment fails', async () => { + const { resolveGitHubRepo } = await import('../../src/sources/github') + const { fetchLlmsUrl } = await import('../../src/sources/llms') + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: () => Promise.resolve({ + crate: { + name: 'serde', + max_version: '1.0.220', + repository: 'https://github.com/serde-rs/serde', + }, + versions: [ + { num: '1.0.220', yanked: false }, + ], + }), + }) + + vi.mocked(resolveGitHubRepo).mockResolvedValue(null) + vi.mocked(fetchLlmsUrl).mockResolvedValue(null) + + const result = await resolveCrateDocsWithAttempts('serde') + + expect(result.attempts).toContainEqual({ + source: 'github-meta', + url: 'https://github.com/serde-rs/serde', + status: 'not-found', + message: 'GitHub enrichment failed, using crates.io metadata', + }) + }) +}) diff --git a/test/unit/sync-crate-routing.test.ts b/test/unit/sync-crate-routing.test.ts new file mode 100644 index 00000000..ce564165 --- /dev/null +++ b/test/unit/sync-crate-routing.test.ts @@ -0,0 +1,127 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('../../src/commands/sync-parallel', () => ({ + syncPackagesParallel: vi.fn(), +})) + +vi.mock('../../src/sources/index.ts', async importOriginal => { + const actual = await importOriginal() + return { + ...actual, + resolveCrateDocsWithAttempts: vi.fn().mockResolvedValue({ + package: null, + attempts: [ + { + source: 'crates', + status: 'not-found', + message: 'Crate not found on crates.io', + }, + ], + }), + } +}) + +const { syncCommand, isCrateSpec } = await import('../../src/commands/sync') + +describe('commands/sync crate routing', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + it('detects crate spec prefix', () => { + expect(isCrateSpec('crate:serde')).toBe(true) + expect(isCrateSpec('serde')).toBe(false) + expect(isCrateSpec('crate')).toBe(false) + }) + + it('keeps npm batch on parallel path when mixed with crate specs', async () => { + const { syncPackagesParallel } = await import('../../src/commands/sync-parallel') + + await syncCommand( + { + skills: [], + deps: new Map(), + missing: [], + outdated: [], + synced: [], + unmatched: [], + }, + { + packages: ['vue', 'nuxt', 'crate:'], + global: false, + agent: 'claude-code', + yes: true, + }, + ) + + expect(vi.mocked(syncPackagesParallel)).toHaveBeenCalledTimes(1) + expect(vi.mocked(syncPackagesParallel)).toHaveBeenCalledWith( + expect.objectContaining({ + packages: ['vue', 'nuxt'], + }), + ) + }) + + it('routes valid crate spec through single-package path while keeping npm parallel', async () => { + const { syncPackagesParallel } = await import('../../src/commands/sync-parallel') + const { log } = await import('@clack/prompts') + const errorSpy = vi.spyOn(log, 'error').mockImplementation(() => {}) + + await syncCommand( + { + skills: [], + deps: new Map(), + missing: [], + outdated: [], + synced: [], + unmatched: [], + }, + { + packages: ['vue', 'nuxt', 'crate:serde'], + global: false, + agent: 'claude-code', + yes: true, + }, + ) + + expect(vi.mocked(syncPackagesParallel)).toHaveBeenCalledTimes(1) + expect(vi.mocked(syncPackagesParallel)).toHaveBeenCalledWith( + expect.objectContaining({ + packages: ['vue', 'nuxt'], + }), + ) + expect(errorSpy).not.toHaveBeenCalled() + }) + + it('does not invoke npm parallel sync when only crate specs are provided', async () => { + const { syncPackagesParallel } = await import('../../src/commands/sync-parallel') + const { log } = await import('@clack/prompts') + const errorSpy = vi.spyOn(log, 'error').mockImplementation(() => {}) + + await syncCommand( + { + skills: [], + deps: new Map(), + missing: [], + outdated: [], + synced: [], + unmatched: [], + }, + { + packages: ['crate:', 'crate: '], + global: false, + agent: 'claude-code', + yes: true, + }, + ) + + expect(vi.mocked(syncPackagesParallel)).not.toHaveBeenCalled() + expect(errorSpy).toHaveBeenCalledTimes(2) + expect(errorSpy).toHaveBeenNthCalledWith(1, 'Invalid crate spec. Use format: crate:') + expect(errorSpy).toHaveBeenNthCalledWith(2, 'Invalid crate spec. Use format: crate:') + }) +}) diff --git a/test/unit/sync-crate-version.test.ts b/test/unit/sync-crate-version.test.ts new file mode 100644 index 00000000..cda9ceff --- /dev/null +++ b/test/unit/sync-crate-version.test.ts @@ -0,0 +1,97 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const stopAfterVersion = new Error('stop-after-version') +const isCachedMock = vi.fn((_: string, __: string) => { + throw stopAfterVersion +}) + +vi.mock('../../src/cache/index.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + isCached: isCachedMock, + } +}) + +vi.mock('../../src/sources/index.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + resolvePackageDocsWithAttempts: vi.fn().mockResolvedValue({ + package: { + name: 'tokio', + version: '2.0.0', + docsUrl: 'https://tokio.dev', + }, + attempts: [ + { + source: 'npm', + status: 'success', + }, + ], + }), + resolveCrateDocsWithAttempts: vi.fn().mockResolvedValue({ + package: { + name: 'serde', + version: '1.0.220', + docsUrl: 'https://docs.rs/serde/1.0.220', + }, + attempts: [ + { + source: 'crates', + status: 'success', + }, + ], + }), + } +}) + +const { syncCommand } = await import('../../src/commands/sync') + +describe('commands/sync crate version selection', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('uses resolved crate version for cache checks when requested version falls back', async () => { + await expect(syncCommand( + { + skills: [], + deps: new Map(), + missing: [], + outdated: [], + synced: [], + unmatched: [], + }, + { + packages: ['crate:serde@1.0.200'], + global: false, + agent: 'claude-code', + yes: true, + }, + )).rejects.toThrow(stopAfterVersion) + + expect(isCachedMock).toHaveBeenCalledWith('@skilld-crate/serde', '1.0.220') + }) + + it('keeps npm package cache key unnamespaced for same-name package', async () => { + await expect(syncCommand( + { + skills: [], + deps: new Map(), + missing: [], + outdated: [], + synced: [], + unmatched: [], + }, + { + packages: ['tokio'], + global: false, + agent: 'claude-code', + yes: true, + }, + )).rejects.toThrow(stopAfterVersion) + + expect(isCachedMock).toHaveBeenCalledWith('tokio', '2.0.0') + }) +})