diff --git a/install.ps1 b/install.ps1 index e3d008e..e9f3fb6 100644 --- a/install.ps1 +++ b/install.ps1 @@ -1,6 +1,9 @@ -# Install ndx from GitHub Releases. +# Install ndx. +# The script is published on GitHub Releases. The binary comes from the +# nuget.org RID package; the blob feed is used when nuget.org is unreachable. # irm https://github.com/devlooped/ndx/releases/latest/download/install.ps1 | iex # Env / flags: NDX_VERSION, NDX_PREFIX, NDX_ARCHIVE, NDX_RID, NDX_REPO, NDX_SKIP_PATH +# NDX_NUGET_FLAT, NDX_NUGET_REG, NDX_BLOB_FLAT # Also accepts --version --prefix --archive --rid --repo --skip-path $ErrorActionPreference = 'Stop' @@ -11,6 +14,9 @@ $Prefix = $env:NDX_PREFIX $Archive = $env:NDX_ARCHIVE $Rid = $env:NDX_RID $SkipPath = $env:NDX_SKIP_PATH -eq '1' +$NugetFlat = if ($env:NDX_NUGET_FLAT) { $env:NDX_NUGET_FLAT } else { 'https://api.nuget.org/v3-flatcontainer' } +$NugetReg = if ($env:NDX_NUGET_REG) { $env:NDX_NUGET_REG } else { 'https://api.nuget.org/v3/registration5-gz-semver2' } +$BlobFlat = if ($env:NDX_BLOB_FLAT) { $env:NDX_BLOB_FLAT } else { 'https://kzu.blob.core.windows.net/nuget/flatcontainer' } function Get-NdxRuntimeIdentifier { $arch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture @@ -68,6 +74,150 @@ public static extern IntPtr SendMessageTimeout( [ref]$result) } +function Get-NdxHttp { + if (-not $script:NdxClient) { + try { + [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + } catch {} + Add-Type -AssemblyName System.Net.Http + $handler = [System.Net.Http.HttpClientHandler]::new() + $handler.AutomaticDecompression = [System.Net.DecompressionMethods]::GZip -bor [System.Net.DecompressionMethods]::Deflate + $script:NdxClient = [System.Net.Http.HttpClient]::new($handler) + $script:NdxClient.Timeout = [TimeSpan]::FromMinutes(5) + $script:NdxClient.DefaultRequestHeaders.UserAgent.ParseAdd('ndx') + } + return $script:NdxClient +} + +function Save-NdxUrl([string]$url, [string]$dest) { + $client = Get-NdxHttp + try { + $resp = $client.GetAsync($url).GetAwaiter().GetResult() + } catch { + return $false + } + try { + if (-not $resp.IsSuccessStatusCode) { return $false } + $fs = [IO.File]::Create($dest) + try { + $stream = $resp.Content.ReadAsStreamAsync().GetAwaiter().GetResult() + try { $stream.CopyTo($fs) } finally { $stream.Dispose() } + } finally { $fs.Dispose() } + return $true + } finally { + $resp.Dispose() + } +} + +function Get-NdxText([string]$url) { + $client = Get-NdxHttp + try { + $resp = $client.GetAsync($url).GetAwaiter().GetResult() + } catch { + return $null + } + try { + if (-not $resp.IsSuccessStatusCode) { return $null } + return $resp.Content.ReadAsStringAsync().GetAwaiter().GetResult() + } finally { + $resp.Dispose() + } +} + +function Get-NdxLatestStable([string]$flat, [string]$id) { + $text = Get-NdxText ("{0}/{1}/index.json" -f $flat.TrimEnd('/'), $id.ToLowerInvariant()) + if (-not $text) { return $null } + try { $obj = $text | ConvertFrom-Json } catch { return $null } + $best = $null + $bestText = $null + foreach ($v in @($obj.versions)) { + if (-not $v -or "$v".Contains('-')) { continue } + try { $parsed = [version]$v } catch { continue } + if ($null -eq $best -or $parsed -gt $best) { + $best = $parsed + $bestText = [string]$v + } + } + return $bestText +} + +function Get-NdxCatalogHash([string]$reg, [string]$id, [string]$ver) { + $leafText = Get-NdxText ("{0}/{1}/{2}.json" -f $reg.TrimEnd('/'), $id, $ver) + if (-not $leafText) { return $null } + try { $leaf = $leafText | ConvertFrom-Json } catch { return $null } + $catalog = $leaf.catalogEntry + if ($catalog -isnot [string]) { + if ($null -eq $catalog) { return $null } + $catalog = $catalog.'@id' + } + if (-not $catalog) { return $null } + $entryText = Get-NdxText ([string]$catalog) + if (-not $entryText) { return $null } + try { $entry = $entryText | ConvertFrom-Json } catch { return $null } + $algo = [string]$entry.packageHashAlgorithm + if ($algo -and $algo -ne 'SHA512') { return $null } + $hash = [string]$entry.packageHash + if (-not $hash) { return $null } + return $hash.Trim() +} + +function Get-NdxSha512([string]$path) { + $sha = [Security.Cryptography.SHA512]::Create() + try { + $fs = [IO.File]::OpenRead($path) + try { return [Convert]::ToBase64String($sha.ComputeHash($fs)) } + finally { $fs.Dispose() } + } finally { + $sha.Dispose() + } +} + +function Save-NdxRidPackage([string]$id, [string]$ver, [string]$dest) { + $idL = $id.ToLowerInvariant() + $verL = $ver.ToLowerInvariant() + $rel = "$idL/$verL/$idL.$verL.nupkg" + if (Save-NdxUrl ("{0}/{1}" -f $NugetFlat.TrimEnd('/'), $rel) $dest) { + $expected = Get-NdxCatalogHash $NugetReg $idL $verL + if (-not $expected) { + Remove-Item -LiteralPath $dest -Force -ErrorAction SilentlyContinue + } else { + $actual = Get-NdxSha512 $dest + if ($actual -ne $expected) { + throw "ndx: SHA512 mismatch for $idL.$verL.nupkg`n expected: $expected`n actual: $actual" + } + return $true + } + } + return (Save-NdxUrl ("{0}/{1}" -f $BlobFlat.TrimEnd('/'), $rel) $dest) +} + +function Expand-NdxPackage([string]$nupkg, [string]$destFile, [string]$entryName) { + if (-not ('System.IO.Compression.ZipFile' -as [type])) { + Add-Type -AssemblyName System.IO.Compression.FileSystem -ErrorAction SilentlyContinue + } + if (-not ('System.IO.Compression.ZipFile' -as [type])) { + Add-Type -AssemblyName System.IO.Compression + } + $zip = [IO.Compression.ZipFile]::OpenRead($nupkg) + try { + $entry = $null + foreach ($item in $zip.Entries) { + if ($item.FullName.Replace('\', '/') -eq $entryName) { + $entry = $item + break + } + } + if (-not $entry) { throw "ndx: package did not contain $entryName" } + $out = [IO.File]::Create($destFile) + try { + $input = $entry.Open() + try { $input.CopyTo($out) } finally { $input.Dispose() } + } finally { $out.Dispose() } + } finally { + $zip.Dispose() + } +} + function Add-NdxToUserPath([string]$dir) { $parts = [Environment]::GetEnvironmentVariable('Path', 'User') if (-not $parts) { $parts = '' } @@ -98,6 +248,7 @@ for ($i = 0; $i -lt $args.Count; $i++) { if (-not $Rid) { $Rid = Get-NdxRuntimeIdentifier } +$Rid = $Rid.ToLowerInvariant() $windows = $Rid.StartsWith('win', [StringComparison]::OrdinalIgnoreCase) $binary = if ($windows) { 'ndx.exe' } else { 'ndx' } @@ -114,47 +265,56 @@ if (-not $Prefix) { $tmp = Join-Path ([IO.Path]::GetTempPath()) ("ndx-install-" + [guid]::NewGuid().ToString('n')) New-Item -ItemType Directory -Path $tmp | Out-Null try { + $extract = Join-Path $tmp 'extract' + New-Item -ItemType Directory -Path $extract | Out-Null + $fromPackage = $false if (-not $Archive) { - if ($Version) { - if ($Version -eq 'ci') { - $tag = 'ci' - $resolved = 'ci' - } elseif ($Version.StartsWith('v')) { - $tag = $Version - $resolved = $tag.TrimStart('v') - } else { - $tag = "v$Version" + $pkg = "ndx.$Rid" + if ($Version -and $Version.ToLowerInvariant() -eq 'ci') { + $tag = 'ci' + $resolved = 'ci' + $name = "ndx-$resolved-$Rid.$ext" + $base = "https://github.com/$Repo/releases/download/$tag" + $Archive = Join-Path $tmp $name + Invoke-WebRequest -Uri "$base/$name" -OutFile $Archive + Invoke-WebRequest -Uri "$base/$name.sha256" -OutFile "$Archive.sha256" + } else { + # GitHub's unauthenticated releases API returns 403 once the hourly + # quota is spent. The RID package on nuget.org is the same binary. + if ($Version) { $resolved = $Version + if ($resolved.StartsWith('v') -or $resolved.StartsWith('V')) { + $resolved = $resolved.Substring(1) + } + } else { + $resolved = Get-NdxLatestStable $NugetFlat $pkg + if (-not $resolved) { $resolved = Get-NdxLatestStable $BlobFlat $pkg } + if (-not $resolved) { throw "ndx: could not resolve the latest stable version of $pkg" } } - } else { - $release = Invoke-RestMethod -Headers @{ Accept = 'application/vnd.github+json' } ` - -Uri "https://api.github.com/repos/$Repo/releases/latest" - $tag = $release.tag_name - if (-not $tag) { throw "ndx: could not resolve latest release of $Repo" } - $resolved = $tag.TrimStart('v') - } - $name = "ndx-$resolved-$Rid.$ext" - $base = "https://github.com/$Repo/releases/download/$tag" - $Archive = Join-Path $tmp $name - Invoke-WebRequest -Uri "$base/$name" -OutFile $Archive - Invoke-WebRequest -Uri "$base/$name.sha256" -OutFile "$Archive.sha256" + $nupkg = Join-Path $tmp "$pkg.$resolved.nupkg" + if (-not (Save-NdxRidPackage $pkg $resolved $nupkg)) { + throw "ndx: could not download $pkg $resolved" + } + Expand-NdxPackage $nupkg (Join-Path $extract $binary) "tools/any/$Rid/$binary" + $fromPackage = $true + } } - if (Test-Path "$Archive.sha256") { - $expected = ((Get-Content -Raw "$Archive.sha256").Trim() -split '\s+')[0].ToLowerInvariant() - $actual = (Get-FileHash -Algorithm SHA256 -Path $Archive).Hash.ToLowerInvariant() - if ($actual -ne $expected) { - throw "ndx: SHA256 mismatch for $(Split-Path $Archive -Leaf)`n expected: $expected`n actual: $actual" + if (-not $fromPackage) { + if (Test-Path "$Archive.sha256") { + $expected = ((Get-Content -Raw "$Archive.sha256").Trim() -split '\s+')[0].ToLowerInvariant() + $actual = (Get-FileHash -Algorithm SHA256 -Path $Archive).Hash.ToLowerInvariant() + if ($actual -ne $expected) { + throw "ndx: SHA256 mismatch for $(Split-Path $Archive -Leaf)`n expected: $expected`n actual: $actual" + } } - } - $extract = Join-Path $tmp 'extract' - New-Item -ItemType Directory -Path $extract | Out-Null - if ($windows) { - Expand-Archive -Path $Archive -DestinationPath $extract -Force - } else { - tar -xzf $Archive -C $extract + if ($windows) { + Expand-Archive -Path $Archive -DestinationPath $extract -Force + } else { + tar -xzf $Archive -C $extract + } } $source = Join-Path $extract $binary @@ -172,5 +332,6 @@ try { } } finally { + if ($script:NdxClient) { $script:NdxClient.Dispose() } Remove-Item -Recurse -Force $tmp -ErrorAction SilentlyContinue } diff --git a/install.sh b/install.sh index 60c9cd3..bd60e84 100755 --- a/install.sh +++ b/install.sh @@ -1,6 +1,10 @@ #!/bin/sh -# Install ndx from GitHub Releases. +# Install ndx. +# The script is published on GitHub Releases. The binary comes from the +# nuget.org RID package; the blob feed is used when nuget.org is unreachable. # curl -fsSL https://github.com/devlooped/ndx/releases/latest/download/install.sh | sh +# Env: NDX_VERSION NDX_PREFIX NDX_ARCHIVE NDX_RID NDX_REPO NDX_SKIP_PATH +# NDX_NUGET_FLAT NDX_NUGET_REG NDX_BLOB_FLAT set -eu REPO="${NDX_REPO:-devlooped/ndx}" @@ -9,6 +13,9 @@ PREFIX="${NDX_PREFIX:-${HOME}/.local/bin}" ARCHIVE="${NDX_ARCHIVE:-}" RID="${NDX_RID:-}" SKIP_PATH="${NDX_SKIP_PATH:-0}" +NUGET_FLAT="${NDX_NUGET_FLAT:-https://api.nuget.org/v3-flatcontainer}" +NUGET_REG="${NDX_NUGET_REG:-https://api.nuget.org/v3/registration5-gz-semver2}" +BLOB_FLAT="${NDX_BLOB_FLAT:-https://kzu.blob.core.windows.net/nuget/flatcontainer}" is_musl() { # Alpine and other musl hosts. gcompat may also add a glibc loader; the musl @@ -57,26 +64,182 @@ detect_rid() { esac } -github_json() { +download() { url=$1 + dest=$2 if command -v curl >/dev/null 2>&1; then - curl -fsSL -H "Accept: application/vnd.github+json" "$url" - elif command -v wget >/dev/null 2>&1; then - wget -qO- --header="Accept: application/vnd.github+json" "$url" + curl -fsSL "$url" -o "$dest" else - echo "ndx: need curl or wget" >&2 - exit 1 + wget -qO "$dest" "$url" fi } -download() { +# JSON from nuget.org registration and the blob feed is gzip content-encoded. +# curl --compressed unwraps it. A still-gzipped body (wget) is inflated here. +# Do not use this for release archives: those files are themselves gzip. +fetch() { url=$1 dest=$2 if command -v curl >/dev/null 2>&1; then - curl -fsSL "$url" -o "$dest" + curl -fsSL --compressed "$url" -o "$dest" 2>/dev/null || return 1 + elif command -v wget >/dev/null 2>&1; then + wget -qO "$dest" "$url" || return 1 else - wget -qO "$dest" "$url" + echo "ndx: need curl or wget" >&2 + exit 1 + fi + if command -v gzip >/dev/null 2>&1 && gzip -t "$dest" 2>/dev/null; then + gzip -dc "$dest" > "${dest}.raw" || return 1 + mv "${dest}.raw" "$dest" + fi + return 0 +} + +version_gt() { + _lhs=$1 + _rhs=$2 + _oa1=0; _oa2=0; _oa3=0; _oa4=0 + _ob1=0; _ob2=0; _ob3=0; _ob4=0 + IFS=. read -r _oa1 _oa2 _oa3 _oa4 </dev/null 2>&1; then + digest=$(openssl dgst -sha512 -binary "$file" | openssl base64 | tr -d '\n\r ') + if [ -n "$digest" ]; then + printf '%s' "$digest" + return 0 + fi + fi + if command -v python3 >/dev/null 2>&1; then + python3 -c 'import hashlib,base64,sys; sys.stdout.write(base64.b64encode(hashlib.sha512(open(sys.argv[1],"rb").read()).digest()).decode())' "$file" + return 0 + fi + return 1 +} + +download_package() { + id=$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]') + ver=$(printf '%s' "$2" | tr '[:upper:]' '[:lower:]') + dest=$3 + rel="${id}/${ver}/${id}.${ver}.nupkg" + + if fetch "${NUGET_FLAT%/}/${rel}" "$dest"; then + if expected=$(catalog_hash "$NUGET_REG" "$id" "$ver"); then + actual=$(sha512_b64 "$dest") || { + echo "ndx: no sha512 tool found (openssl or python3)" >&2 + exit 1 + } + if [ "$actual" != "$expected" ]; then + echo "ndx: SHA512 mismatch for ${id}.${ver}.nupkg" >&2 + echo " expected: $expected" >&2 + echo " actual: $actual" >&2 + exit 1 + fi + return 0 + fi + rm -f "$dest" + fi + + fetch "${BLOB_FLAT%/}/${rel}" "$dest" +} + +extract_nupkg_binary() { + nupkg=$1 + dest=$2 + entry="tools/any/${RID}/${binary}" + if command -v unzip >/dev/null 2>&1; then + if unzip -p "$nupkg" "$entry" > "$dest" 2>/dev/null && [ -s "$dest" ]; then + return 0 + fi + rm -f "$dest" fi + if command -v python3 >/dev/null 2>&1; then + if python3 -c 'import sys,zipfile +z=zipfile.ZipFile(sys.argv[1]) +with z.open(sys.argv[2]) as src, open(sys.argv[3],"wb") as dst: + dst.write(src.read())' "$nupkg" "$entry" "$dest" && [ -s "$dest" ]; then + return 0 + fi + rm -f "$dest" + fi + echo "ndx: package did not contain ${entry}" >&2 + exit 1 } json_string() { @@ -111,6 +274,7 @@ verify_sha256() { if [ -z "$RID" ]; then RID=$(detect_rid) fi +RID=$(printf '%s' "$RID" | tr '[:upper:]' '[:lower:]') case "$RID" in win-*) binary=ndx.exe; ext=zip ;; @@ -124,39 +288,49 @@ esac tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT INT TERM +from_package=0 if [ -z "$ARCHIVE" ]; then + pkg="ndx.${RID}" if [ -n "$VERSION" ]; then case "$(printf '%s' "$VERSION" | tr '[:upper:]' '[:lower:]')" in ci) tag=ci version=ci ;; - v*) - tag=$VERSION - version=${tag#v} - ;; *) - tag="v${VERSION}" - version=$VERSION + version=$(printf '%s' "$VERSION" | sed 's/^[vV]//') + from_package=1 ;; esac else - json=$(github_json "https://api.github.com/repos/${REPO}/releases/latest") - tag=$(printf '%s' "$json" | json_string tag_name) - if [ -z "$tag" ]; then - echo "ndx: could not resolve latest release of ${REPO}" >&2 + # GitHub's unauthenticated releases API returns 403 once the hourly + # quota is spent. The RID package on nuget.org is the same binary. + version=$(latest_stable "$NUGET_FLAT" "$pkg" || true) + if [ -z "$version" ]; then + version=$(latest_stable "$BLOB_FLAT" "$pkg" || true) + fi + if [ -z "$version" ]; then + echo "ndx: could not resolve the latest stable version of ${pkg}" >&2 exit 1 fi - version=${tag#v} + from_package=1 fi - name="ndx-${version}-${RID}.${ext}" - base="https://github.com/${REPO}/releases/download/${tag}" - ARCHIVE="${tmp}/${name}" - download "${base}/${name}" "$ARCHIVE" - download "${base}/${name}.sha256" "${ARCHIVE}.sha256" - expected=$(awk '{print $1}' "${ARCHIVE}.sha256") - verify_sha256 "$ARCHIVE" "$expected" + if [ "$from_package" = 1 ]; then + nupkg="${tmp}/package.nupkg" + if ! download_package "$pkg" "$version" "$nupkg"; then + echo "ndx: could not download ${pkg} ${version}" >&2 + exit 1 + fi + else + name="ndx-${version}-${RID}.${ext}" + base="https://github.com/${REPO}/releases/download/${tag}" + ARCHIVE="${tmp}/${name}" + download "${base}/${name}" "$ARCHIVE" + download "${base}/${name}.sha256" "${ARCHIVE}.sha256" + expected=$(awk '{print $1}' "${ARCHIVE}.sha256") + verify_sha256 "$ARCHIVE" "$expected" + fi else if [ -f "${ARCHIVE}.sha256" ]; then expected=$(awk '{print $1}' "${ARCHIVE}.sha256") @@ -166,19 +340,23 @@ fi extract="${tmp}/extract" mkdir -p "$extract" -case "$ext" in - zip) - if command -v unzip >/dev/null 2>&1; then - unzip -o -q "$ARCHIVE" -d "$extract" - else - echo "ndx: unzip is required to extract Windows archives" >&2 - exit 1 - fi - ;; - tar.gz) - tar -xzf "$ARCHIVE" -C "$extract" - ;; -esac +if [ "$from_package" = 1 ]; then + extract_nupkg_binary "$nupkg" "${extract}/${binary}" +else + case "$ext" in + zip) + if command -v unzip >/dev/null 2>&1; then + unzip -o -q "$ARCHIVE" -d "$extract" + else + echo "ndx: unzip is required to extract Windows archives" >&2 + exit 1 + fi + ;; + tar.gz) + tar -xzf "$ARCHIVE" -C "$extract" + ;; + esac +fi if [ ! -f "${extract}/${binary}" ]; then echo "ndx: archive did not contain ${binary}" >&2 diff --git a/readme.md b/readme.md index 78bd73a..938c9dd 100644 --- a/readme.md +++ b/readme.md @@ -38,16 +38,22 @@ irm https://github.com/devlooped/ndx/releases/latest/download/install.ps1 | iex > Alternatively (perhaps of debatable utility), you can install using the .NET SDK too: > `dotnet tool install -g ndx` -The Linux archives are glibc builds. Each is also published as -`ndx--linux--gnu.tar.gz` so an installer can see that it needs -GNU libc. Alpine and other musl systems get `linux-musl-x64` / `linux-musl-arm64`, -both as release archives (`install.sh` selects them) and as NuGet RID packages, -so `dotnet tool install -g ndx` does too. +The installer reads the `ndx.` flat-container index on nuget.org, downloads +that RID package, checks the catalog SHA512, and copies `tools/any//ndx` +into place. If nuget.org can't be reached, it uses the blob feed at +`https://kzu.blob.core.windows.net/nuget`. `NDX_VERSION=ci` still installs the +rolling GitHub Release. + +glibc Linux builds are `linux-x64` / `linux-arm64`. Alpine and other musl +systems get `linux-musl-x64` / `linux-musl-arm64`, so `dotnet tool install -g ndx` +does too. The same bits are attached to the GitHub Release, including a +`ndx--linux--gnu.tar.gz` alias so an installer can see that the +glibc build needs GNU libc. ## Update -Self-update the installed binary (optional version, including downgrades): +Self-update the installed binary from the same NuGet RID package (optional version, including downgrades). `ndx --update ci` still tracks the rolling GitHub Release. ```bash ndx --update diff --git a/src/Tests/InstallScriptTests.cs b/src/Tests/InstallScriptTests.cs index da5bab8..7527630 100644 --- a/src/Tests/InstallScriptTests.cs +++ b/src/Tests/InstallScriptTests.cs @@ -1,4 +1,5 @@ using System.Diagnostics; +using System.Net; using ndx; namespace Tests; @@ -276,9 +277,96 @@ public void Install_scripts_resolve_ci_channel_without_a_v_prefix() Assert.Contains("ci)", sh); Assert.Contains("tag=ci", sh); Assert.Contains("version=ci", sh); - Assert.Contains("$Version -eq 'ci'", ps); + Assert.Contains("$Version.ToLowerInvariant() -eq 'ci'", ps); Assert.Contains("$tag = 'ci'", ps); Assert.Contains("$resolved = 'ci'", ps); + Assert.DoesNotContain("api.github.com", sh); + Assert.DoesNotContain("api.github.com", ps); + Assert.Contains("https://api.nuget.org/v3-flatcontainer", sh); + Assert.Contains("https://api.nuget.org/v3/registration5-gz-semver2", sh); + Assert.Contains("https://kzu.blob.core.windows.net/nuget/flatcontainer", sh); + Assert.Contains("tools/any/", sh); + Assert.Contains("https://api.nuget.org/v3-flatcontainer", ps); + Assert.Contains("https://api.nuget.org/v3/registration5-gz-semver2", ps); + Assert.Contains("https://kzu.blob.core.windows.net/nuget/flatcontainer", ps); + Assert.Contains("tools/any/", ps); + } + + [Fact] + public void Shell_installer_unpacks_the_highest_stable_rid_package_from_nuget() + { + var sh = FindBash() is not null ? "/bin/sh" : null; + Assert.True(sh is not null && File.Exists(sh), "sh is required to verify install.sh"); + + using var feed = new ScriptFeed(); + using var dir = new TempDir(); + var payload = "from-nuget"u8.ToArray(); + var nupkg = File.ReadAllBytes(RidNupkg.Write(dir.Publish, "linux-x64", payload)); + var hash = Convert.ToBase64String(System.Security.Cryptography.SHA512.HashData(nupkg)); + const string version = "1.0.10"; + var catalog = feed.Url($"/catalog/ndx.linux-x64.{version}.json"); + feed.Map($"/nuget/flat/ndx.linux-x64/index.json", """{"versions":["1.0.9","1.0.10","2.0.0-preview"]}"""); + feed.Map($"/nuget/flat/ndx.linux-x64/{version}/ndx.linux-x64.{version}.nupkg", nupkg, "application/octet-stream"); + feed.Map($"/nuget/reg/ndx.linux-x64/{version}.json", + "{\"catalogEntry\":\"" + catalog + "\",\"@context\":{\"catalogEntry\":{\"@type\":\"@id\"}}}"); + feed.Map($"/catalog/ndx.linux-x64.{version}.json", + $$""" + { + "packageHashAlgorithm": "SHA512", + "packageHash": "{{hash}}" + } + """); + + RunFeedInstall("/bin/sh", FindRepoRoot(), dir, feed, "linux-x64"); + + Assert.Equal(payload, File.ReadAllBytes(Path.Combine(dir.Prefix, "ndx"))); + Assert.DoesNotContain(feed.Hits, hit => hit.Contains("/blob/", StringComparison.Ordinal)); + Assert.Contains(feed.Hits, hit => hit.Contains($"/ndx.linux-x64/{version}/", StringComparison.Ordinal)); + } + + [Fact] + public void Shell_installer_uses_the_blob_feed_when_nuget_org_is_unavailable() + { + Assert.True(File.Exists("/bin/sh"), "sh is required to verify install.sh"); + + using var feed = new ScriptFeed(); + using var dir = new TempDir(); + var payload = "from-blob"u8.ToArray(); + var nupkg = File.ReadAllBytes(RidNupkg.Write(dir.Output, "linux-x64", payload)); + const string version = "3.1.0"; + feed.Map("/nuget/flat/ndx.linux-x64/index.json", "unavailable", status: 503); + feed.Map($"/blob/flat/ndx.linux-x64/index.json", $$"""{"versions":["{{version}}"]}"""); + feed.Map($"/blob/flat/ndx.linux-x64/{version}/ndx.linux-x64.{version}.nupkg", nupkg, "application/octet-stream"); + + RunFeedInstall("/bin/sh", FindRepoRoot(), dir, feed, "linux-x64"); + + Assert.Equal(payload, File.ReadAllBytes(Path.Combine(dir.Prefix, "ndx"))); + Assert.Contains(feed.Hits, hit => hit.Contains("/blob/flat/", StringComparison.Ordinal)); + } + + [Fact] + public void Shell_installer_rejects_a_catalog_hash_mismatch() + { + Assert.True(File.Exists("/bin/sh"), "sh is required to verify install.sh"); + + using var feed = new ScriptFeed(); + using var dir = new TempDir(); + var nupkg = File.ReadAllBytes(RidNupkg.Write(dir.Publish, "linux-x64", "bad"u8.ToArray())); + const string version = "1.2.0"; + var catalog = feed.Url($"/catalog/ndx.linux-x64.{version}.json"); + feed.Map($"/nuget/flat/ndx.linux-x64/index.json", $$"""{"versions":["{{version}}"]}"""); + feed.Map($"/nuget/flat/ndx.linux-x64/{version}/ndx.linux-x64.{version}.nupkg", nupkg, "application/octet-stream"); + feed.Map($"/nuget/reg/ndx.linux-x64/{version}.json", + $$"""{"catalogEntry":"{{catalog}}"}"""); + feed.Map($"/catalog/ndx.linux-x64.{version}.json", + """{"packageHash":"AA==","packageHashAlgorithm":"SHA512"}"""); + + var (exit, stdout, stderr) = RunFeedInstall("/bin/sh", FindRepoRoot(), dir, feed, "linux-x64", expectSuccess: false); + + Assert.NotEqual(0, exit); + Assert.Contains("SHA512", stderr); + Assert.False(File.Exists(Path.Combine(dir.Prefix, "ndx")), stdout); + Assert.DoesNotContain(feed.Hits, hit => hit.Contains("/blob/", StringComparison.Ordinal)); } [Fact] @@ -306,6 +394,46 @@ public void Workflow_publishes_nuget_and_sleet_from_release_and_ci_build() Assert.Contains("uninstall.ps1", ci); } + static (int ExitCode, string Stdout, string Stderr) RunFeedInstall( + string shell, + string repoRoot, + TempDir dir, + ScriptFeed feed, + string rid, + bool expectSuccess = true) + { + var start = new ProcessStartInfo + { + FileName = shell, + WorkingDirectory = repoRoot, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + start.ArgumentList.Add(Path.Combine(repoRoot, "install.sh")); + start.Environment["HOME"] = dir.Home; + start.Environment["NDX_PREFIX"] = dir.Prefix; + start.Environment["NDX_RID"] = rid; + start.Environment["NDX_SKIP_PATH"] = "1"; + start.Environment["NDX_VERSION"] = ""; + start.Environment["NDX_ARCHIVE"] = ""; + start.Environment["NDX_NUGET_FLAT"] = feed.Url("/nuget/flat"); + start.Environment["NDX_NUGET_REG"] = feed.Url("/nuget/reg"); + start.Environment["NDX_BLOB_FLAT"] = feed.Url("/blob/flat"); + + using var process = Process.Start(start) ?? throw new InvalidOperationException("failed to start sh"); + var stdout = process.StandardOutput.ReadToEnd(); + var stderr = process.StandardError.ReadToEnd(); + process.WaitForExit(); + if (expectSuccess) + { + Assert.True(process.ExitCode == 0, $"install.sh failed ({process.ExitCode}).{Environment.NewLine}{stdout}{Environment.NewLine}{stderr}"); + Assert.Contains("installed", stdout); + } + + return (process.ExitCode, stdout, stderr); + } + static (int ExitCode, string Stdout, string Stderr) RunPowershell(string script, TempDir dir, string? archive, bool skipPath) { var start = new ProcessStartInfo @@ -459,6 +587,106 @@ static string FindRepoRoot() throw new InvalidOperationException("Could not locate repo root."); } + sealed class ScriptFeed : IDisposable + { + readonly HttpListener listener = new(); + readonly object gate = new(); + readonly Dictionary map = new(StringComparer.Ordinal); + readonly Task loop; + + public List Hits { get; } = []; + public string BaseUrl { get; } + + public ScriptFeed() + { + var port = FreePort(); + BaseUrl = $"http://127.0.0.1:{port}"; + listener.Prefixes.Add(BaseUrl + "/"); + listener.Start(); + loop = Task.Run(Serve); + } + + public string Url(string path) => BaseUrl + path; + + public void Map(string path, string body, string contentType = "application/json", int status = 200) + => Map(path, System.Text.Encoding.UTF8.GetBytes(body), contentType, status); + + public void Map(string path, byte[] body, string contentType = "application/json", int status = 200) + { + lock (gate) + map[path] = (status, body, contentType); + } + + async Task Serve() + { + while (listener.IsListening) + { + HttpListenerContext ctx; + try + { + ctx = await listener.GetContextAsync().ConfigureAwait(false); + } + catch (Exception) when (!listener.IsListening) + { + break; + } + catch (HttpListenerException) + { + break; + } + catch (ObjectDisposedException) + { + break; + } + + var path = ctx.Request.Url?.AbsolutePath ?? ""; + (int Status, byte[]? Body, string? ContentType) mapped; + lock (gate) + { + Hits.Add(path); + map.TryGetValue(path, out var found); + mapped = (found.Status, found.Body, found.ContentType); + } + var status = mapped.Body is null ? 404 : mapped.Status; + var body = mapped.Body ?? "not found"u8.ToArray(); + var type = mapped.ContentType ?? "text/plain"; + ctx.Response.StatusCode = status; + ctx.Response.ContentType = type; + ctx.Response.ContentLength64 = body.Length; + ctx.Response.KeepAlive = false; + try + { + ctx.Response.OutputStream.Write(body, 0, body.Length); + ctx.Response.OutputStream.Close(); + } + catch (HttpListenerException) + { + } + finally + { + ctx.Response.Close(); + } + } + } + + static int FreePort() + { + var tcp = new System.Net.Sockets.TcpListener(System.Net.IPAddress.Loopback, 0); + tcp.Start(); + var port = ((System.Net.IPEndPoint)tcp.LocalEndpoint).Port; + tcp.Stop(); + return port; + } + + public void Dispose() + { + listener.Stop(); + listener.Close(); + try { loop.Wait(TimeSpan.FromSeconds(2)); } + catch (AggregateException) { } + } + } + sealed class TempDir : IDisposable { public string Root { get; } = Path.Combine(Path.GetTempPath(), "ndx-install-tests", Guid.NewGuid().ToString("n")); diff --git a/src/Tests/SelfUpdateTests.cs b/src/Tests/SelfUpdateTests.cs index bac2996..6bd135f 100644 --- a/src/Tests/SelfUpdateTests.cs +++ b/src/Tests/SelfUpdateTests.cs @@ -1,4 +1,5 @@ using System.Net; +using System.Security.Cryptography; using System.Text; using ndx; @@ -79,8 +80,9 @@ public async Task Update_skips_download_when_already_on_latest() Assert.Equal(0, code); Assert.Equal(payload, File.ReadAllBytes(current)); Assert.Contains("already 0.2.0", host.Out.ToString()); - Assert.Equal(1, handler.Hits.Count(u => u.Contains("/releases/latest", StringComparison.Ordinal))); - Assert.DoesNotContain(handler.Hits, u => u.Contains("/releases/download/", StringComparison.Ordinal)); + Assert.Equal(1, handler.Hits.Count(u => u.Contains("/index.json", StringComparison.Ordinal))); + Assert.DoesNotContain(handler.Hits, u => u.EndsWith(".nupkg", StringComparison.Ordinal)); + Assert.DoesNotContain(handler.Hits, u => u.Contains("/releases/latest", StringComparison.Ordinal)); } [Fact] @@ -159,7 +161,7 @@ public async Task Update_to_a_missing_version_fails() } [Fact] - public async Task Update_extracts_a_unix_targz_archive() + public async Task Update_extracts_the_rid_binary_from_the_nupkg() { using var dir = new TempDir(); var current = Path.Combine(dir.Prefix, "ndx"); @@ -167,16 +169,7 @@ public async Task Update_extracts_a_unix_targz_archive() const string unixRid = "linux-x64"; using var handler = new MapHandler(); - var packed = NativePacker.Pack( - RidNupkg.Write(Path.Combine(dir.Root, "nupkg-unix"), unixRid, "new-unix"u8.ToArray()), - unixRid, - Path.Combine(dir.Root, "out-unix"), - "0.3.0"); - var name = Path.GetFileName(packed.ArchivePath); - handler.Map[SelfUpdate.AssetUrl(Repo, "v0.3.0", name)] = - (HttpStatusCode.OK, File.ReadAllBytes(packed.ArchivePath), "application/octet-stream"); - handler.Map[SelfUpdate.AssetUrl(Repo, "v0.3.0", name) + ".sha256"] = - (HttpStatusCode.OK, File.ReadAllBytes(packed.Sha256Path), "text/plain"); + AddPackage(handler, dir, "0.3.0", "new-unix"u8.ToArray(), SelfUpdate.NugetFlatContainer, SelfUpdate.NugetRegistration, unixRid); var host = new NdxHost { @@ -216,23 +209,53 @@ public async Task Update_does_not_launch_a_child() } [Fact] - public async Task Sha256_mismatch_leaves_the_current_binary() + public async Task Sha512_mismatch_leaves_the_current_binary() { using var dir = new TempDir(); var current = Path.Combine(dir.Prefix, "ndx.exe"); File.WriteAllBytes(current, "old-binary"u8.ToArray()); using var handler = Feed(dir, latest: "0.2.0", payload: "new-binary"u8.ToArray()); - var archive = SelfUpdate.ArchiveFileName(Rid, "0.2.0"); - handler.Map[SelfUpdate.AssetUrl(Repo, "v0.2.0", archive) + ".sha256"] = - (HttpStatusCode.OK, "0"u8.ToArray(), "text/plain"); + handler.Map[CatalogUrl(SelfUpdate.RidPackageId(Rid), "0.2.0")] = + (HttpStatusCode.OK, """{"packageHash":"AA==","packageHashAlgorithm":"SHA512"}"""u8.ToArray(), "application/json"); var host = NewHost(dir, current, "0.1.0", handler); var code = await App.RunAsync(["--update"], host); Assert.Equal(1, code); Assert.Equal("old-binary"u8.ToArray(), File.ReadAllBytes(current)); - Assert.Contains("SHA256", host.Error.ToString()); + Assert.Contains("SHA512", host.Error.ToString()); + Assert.DoesNotContain(handler.Hits, u => u.Contains("blob.core.windows.net", StringComparison.Ordinal)); + } + + [Fact] + public async Task Update_falls_back_to_the_blob_feed_when_nuget_org_is_unavailable() + { + using var dir = new TempDir(); + var current = Path.Combine(dir.Prefix, "ndx.exe"); + File.WriteAllBytes(current, "old-binary"u8.ToArray()); + + using var handler = new MapHandler(); + var id = SelfUpdate.RidPackageId(Rid); + handler.Map[SelfUpdate.FlatIndexUrl(SelfUpdate.NugetFlatContainer, id)] = + (HttpStatusCode.ServiceUnavailable, "nope"u8.ToArray(), "text/plain"); + AddIndex(handler, SelfUpdate.BlobFlatContainer, "0.4.0"); + AddPackage(handler, dir, "0.4.0", "blob-binary"u8.ToArray(), SelfUpdate.BlobFlatContainer, registration: null); + + var host = NewHost(dir, current, "0.1.0", handler); + var code = await App.RunAsync(["--update"], host); + + Assert.Equal(0, code); + Assert.Equal("blob-binary"u8.ToArray(), File.ReadAllBytes(current)); + Assert.Contains("Updating to 0.4.0", host.Out.ToString()); + } + + [Fact] + public void SelectLatestStable_skips_prereleases_and_picks_the_highest_version() + { + var latest = SelfUpdate.SelectLatestStable(["1.0.9", "1.0.10", "2.0.0-preview", "1.2.0"]); + Assert.Equal("1.2.0", latest); + Assert.Null(SelfUpdate.SelectLatestStable(["1.0.0-ci", "9.9.9-preview"])); } static NdxHost NewHost(TempDir dir, string executable, string currentVersion, MapHandler handler, IProcessRunner? runner = null) @@ -257,16 +280,54 @@ static MapHandler Feed( string? extraVersion = null) { var handler = new MapHandler(); - handler.Map[SelfUpdate.LatestReleaseUrl(Repo)] = - (HttpStatusCode.OK, Encoding.UTF8.GetBytes($$"""{"tag_name":"v{{latest}}"}"""), "application/json"); - - AddRelease(handler, dir, latest, payload); + var versions = extraVersion is null + ? new[] { "0.0.1", latest, "9.9.9-preview" } + : new[] { extraVersion, latest, "9.9.9-preview" }; + AddIndex(handler, SelfUpdate.NugetFlatContainer, versions); + AddPackage(handler, dir, latest, payload, SelfUpdate.NugetFlatContainer, SelfUpdate.NugetRegistration); if (extraVersion is not null) - AddRelease(handler, dir, extraVersion, payload); + AddPackage(handler, dir, extraVersion, payload, SelfUpdate.NugetFlatContainer, SelfUpdate.NugetRegistration); return handler; } + static void AddIndex(MapHandler handler, string flat, params string[] versions) + { + var id = SelfUpdate.RidPackageId(Rid); + var json = "{\"versions\":[" + string.Join(',', versions.Select(v => "\"" + v + "\"")) + "]}"; + handler.Map[SelfUpdate.FlatIndexUrl(flat, id)] = + (HttpStatusCode.OK, Encoding.UTF8.GetBytes(json), "application/json"); + } + + static void AddPackage( + MapHandler handler, + TempDir dir, + string version, + byte[] payload, + string flat, + string? registration, + string? rid = null) + { + rid ??= Rid; + var id = SelfUpdate.RidPackageId(rid); + var nupkgPath = RidNupkg.Write(Path.Combine(dir.Root, "nupkg-" + Guid.NewGuid().ToString("n")), rid, payload); + var nupkg = File.ReadAllBytes(nupkgPath); + handler.Map[SelfUpdate.NupkgUrl(flat, id, version)] = + (HttpStatusCode.OK, nupkg, "application/octet-stream"); + if (registration is null) + return; + + var catalog = CatalogUrl(id, version); + handler.Map[SelfUpdate.RegistrationLeafUrl(registration, id, version)] = + (HttpStatusCode.OK, Encoding.UTF8.GetBytes("{\"catalogEntry\":\"" + catalog + "\"}"), "application/json"); + var hash = Convert.ToBase64String(SHA512.HashData(nupkg)); + handler.Map[catalog] = + (HttpStatusCode.OK, Encoding.UTF8.GetBytes($$"""{"packageHash":"{{hash}}","packageHashAlgorithm":"SHA512"}"""), "application/json"); + } + + static string CatalogUrl(string packageId, string version) + => $"https://api.nuget.org/v3/catalog0/data/test/{packageId}.{version}.json"; + static void AddRelease(MapHandler handler, TempDir dir, string version, byte[] payload, string? tag = null) { var packed = NativePacker.Pack( diff --git a/src/ndx/NuGetJson.cs b/src/ndx/NuGetJson.cs index fe16e5f..d4ae62c 100644 --- a/src/ndx/NuGetJson.cs +++ b/src/ndx/NuGetJson.cs @@ -31,6 +31,8 @@ sealed class RegistrationLeaf sealed class CatalogLeaf { public long? PackageSize { get; set; } + public string? PackageHash { get; set; } + public string? PackageHashAlgorithm { get; set; } } sealed class RuntimeGraphFile @@ -44,12 +46,6 @@ sealed class RuntimeGraphNode public string[]? Import { get; set; } } -sealed class GitHubRelease -{ - [JsonPropertyName("tag_name")] - public string? TagName { get; set; } -} - sealed class RuntimeConfigFile { public RuntimeConfigOptions? RuntimeOptions { get; set; } @@ -78,6 +74,5 @@ sealed class RuntimeConfigFramework [JsonSerializable(typeof(RegistrationLeaf))] [JsonSerializable(typeof(CatalogLeaf))] [JsonSerializable(typeof(RuntimeGraphFile))] -[JsonSerializable(typeof(GitHubRelease))] [JsonSerializable(typeof(RuntimeConfigFile))] sealed partial class NuGetJsonContext : JsonSerializerContext; diff --git a/src/ndx/SelfUpdate.cs b/src/ndx/SelfUpdate.cs index 8e9f6ce..59767aa 100644 --- a/src/ndx/SelfUpdate.cs +++ b/src/ndx/SelfUpdate.cs @@ -1,21 +1,33 @@ using System.Formats.Tar; using System.IO.Compression; -using System.Net.Http.Headers; using System.Net.Http.Json; using System.Reflection; using System.Runtime.InteropServices; using System.Security.Cryptography; +using System.Text.Json; namespace ndx; /// -/// Replaces the running ndx binary with a GitHub Release asset. +/// Replaces the running ndx binary with the one inside the NuGet RID package. +/// The rolling ci channel still comes from the GitHub Release tag ci. /// public static class SelfUpdate { public const string DefaultRepository = "devlooped/ndx"; public const string CiChannel = "ci"; + /// nuget.org flat container. Version lists and nupkgs. No trailing lookup of the service index. + public const string NugetFlatContainer = "https://api.nuget.org/v3-flatcontainer/"; + + /// nuget.org registration base. The leaf points at the catalog entry that carries the SHA512. + public const string NugetRegistration = "https://api.nuget.org/v3/registration5-gz-semver2/"; + + /// + /// Sleet feed used when nuget.org can't be reached. Same RID packages, no catalog hash. + /// + public const string BlobFlatContainer = "https://kzu.blob.core.windows.net/nuget/flatcontainer/"; + public static bool IsCiChannel(string? value) => value is not null && value.Equals(CiChannel, StringComparison.OrdinalIgnoreCase); @@ -36,62 +48,54 @@ public static async Task RunAsync( var executable = host.ExecutablePath ?? ResolveExecutablePath(); var log = IsDetailed(invocation.Verbosity) ? host.Out : null; - EnsureGitHubHeaders(http); + EnsureUserAgent(http); - string targetLabel; - string tag; - if (IsCiChannel(invocation.Version)) + if (!File.Exists(executable)) { - // Rolling prerelease: assets under tag `ci` change in place, so always download. - targetLabel = CiChannel; - tag = CiChannel; + throw new InvalidOperationException( + $"Cannot self-update: '{executable}' was not found."); } - else + + // Rolling prerelease: assets under tag `ci` are not on nuget.org. + if (IsCiChannel(invocation.Version)) { - var target = invocation.Version is { } specified - ? NormalizeVersion(specified) - : await ResolveLatestVersionAsync(http, repo, log, cancellationToken).ConfigureAwait(false); + return await UpdateFromReleaseArchiveAsync( + http, host, repo, rid, executable, CiChannel, CiChannel, log, cancellationToken) + .ConfigureAwait(false); + } - if (!PackageVersion.TryParse(target, out var targetVersion)) + var packageId = RidPackageId(rid); + string target; + if (invocation.Version is { } specified) + { + target = NormalizeVersion(specified); + if (!PackageVersion.TryParse(target, out _)) throw new InvalidOperationException($"Invalid version '{target}'."); - - if (PackageVersion.TryParse(currentVersion, out var current) && current.Equals(targetVersion)) - { - host.Out.WriteLine($"ndx is already {targetVersion}"); - return 0; - } - - targetLabel = targetVersion.ToString(); - tag = ReleaseTag(targetLabel); } - - if (!File.Exists(executable)) + else { - throw new InvalidOperationException( - $"Cannot self-update: '{executable}' was not found."); + target = await ResolveLatestStableAsync(http, packageId, log, cancellationToken).ConfigureAwait(false); } - host.Out.WriteLine($"Updating to {targetLabel}"); + if (PackageVersion.TryParse(target, out var targetVersion) && + PackageVersion.TryParse(currentVersion, out var current) && + current.Equals(targetVersion)) + { + host.Out.WriteLine($"ndx is already {targetVersion}"); + return 0; + } - var archiveName = ArchiveFileName(rid, targetLabel); - var binaryName = BinaryFileName(rid); - var archiveUrl = AssetUrl(repo, tag, archiveName); - var shaUrl = archiveUrl + ".sha256"; + host.Out.WriteLine($"Updating to {target}"); var tmp = Path.Combine(Path.GetTempPath(), "ndx-update-" + Guid.NewGuid().ToString("n")); Directory.CreateDirectory(tmp); try { - var archivePath = Path.Combine(tmp, archiveName); - log?.WriteLine($"Downloading {archiveUrl}"); - await DownloadAsync(http, archiveUrl, archivePath, cancellationToken).ConfigureAwait(false); - - log?.WriteLine($"Downloading {shaUrl}"); - var expected = await DownloadStringAsync(http, shaUrl, cancellationToken).ConfigureAwait(false); - VerifySha256(archivePath, expected); - + var nupkg = await DownloadRidPackageAsync(http, packageId, target, tmp, log, cancellationToken) + .ConfigureAwait(false); + var binaryName = BinaryFileName(rid); var extracted = Path.Combine(tmp, binaryName); - ExtractBinary(archivePath, rid, binaryName, extracted); + ExtractNupkgBinary(nupkg, rid, binaryName, extracted); ReplaceExecutable(executable, extracted); host.Out.WriteLine($"updated {executable}"); return 0; @@ -103,6 +107,44 @@ public static async Task RunAsync( } } + public static string RidPackageId(string rid) + => "ndx." + rid.ToLowerInvariant(); + + public static string FlatIndexUrl(string flatBase, string packageId) + => Combine(flatBase, packageId.ToLowerInvariant() + "/index.json"); + + public static string NupkgUrl(string flatBase, string packageId, string version) + { + var id = packageId.ToLowerInvariant(); + var ver = version.ToLowerInvariant(); + return Combine(flatBase, $"{id}/{ver}/{id}.{ver}.nupkg"); + } + + public static string RegistrationLeafUrl(string registrationBase, string packageId, string version) + => Combine(registrationBase, $"{packageId.ToLowerInvariant()}/{version.ToLowerInvariant()}.json"); + + /// Highest stable version in a flat-container versions array. Prereleases are skipped. + public static string? SelectLatestStable(IEnumerable? versions) + { + if (versions is null) + return null; + + PackageVersion? best = null; + string? bestText = null; + foreach (var text in versions) + { + if (!PackageVersion.TryParse(text, out var version) || version.IsPrerelease) + continue; + if (best is null || version.CompareTo(best.Value) > 0) + { + best = version; + bestText = text.Trim(); + } + } + + return bestText; + } + public static string ReadCurrentVersion() => NormalizeVersion(ReadInformationalVersion()); @@ -188,9 +230,6 @@ public static bool HasMuslLoader(string root = "/") return false; } - public static string LatestReleaseUrl(string repo) - => $"https://api.github.com/repos/{repo}/releases/latest"; - public static string AssetUrl(string repo, string tag, string fileName) => $"https://github.com/{repo}/releases/download/{tag}/{fileName}"; @@ -216,31 +255,244 @@ static string NormalizeVersion(string value) return text; } - static async Task ResolveLatestVersionAsync( + static async Task UpdateFromReleaseArchiveAsync( HttpClient http, + NdxHost host, string repo, + string rid, + string executable, + string targetLabel, + string tag, TextWriter? log, CancellationToken cancellationToken) { - var url = LatestReleaseUrl(repo); - log?.WriteLine($"GET {url}"); - using var response = await http.GetAsync(url, cancellationToken).ConfigureAwait(false); - if (!response.IsSuccessStatusCode) + host.Out.WriteLine($"Updating to {targetLabel}"); + + var archiveName = ArchiveFileName(rid, targetLabel); + var binaryName = BinaryFileName(rid); + var archiveUrl = AssetUrl(repo, tag, archiveName); + var shaUrl = archiveUrl + ".sha256"; + + var tmp = Path.Combine(Path.GetTempPath(), "ndx-update-" + Guid.NewGuid().ToString("n")); + Directory.CreateDirectory(tmp); + try + { + var archivePath = Path.Combine(tmp, archiveName); + log?.WriteLine($"Downloading {archiveUrl}"); + await DownloadAsync(http, archiveUrl, archivePath, cancellationToken).ConfigureAwait(false); + + log?.WriteLine($"Downloading {shaUrl}"); + var expected = await DownloadStringAsync(http, shaUrl, cancellationToken).ConfigureAwait(false); + VerifySha256(archivePath, expected); + + var extracted = Path.Combine(tmp, binaryName); + ExtractBinary(archivePath, rid, binaryName, extracted); + ReplaceExecutable(executable, extracted); + host.Out.WriteLine($"updated {executable}"); + return 0; + } + finally + { + try { Directory.Delete(tmp, recursive: true); } + catch (IOException) { } + } + } + + static async Task ResolveLatestStableAsync( + HttpClient http, + string packageId, + TextWriter? log, + CancellationToken cancellationToken) + { + Exception? last = null; + foreach (var flat in new[] { NugetFlatContainer, BlobFlatContainer }) + { + var url = FlatIndexUrl(flat, packageId); + try + { + log?.WriteLine($"GET {url}"); + using var response = await http.GetAsync(url, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + last = new InvalidOperationException( + $"Could not resolve latest {packageId} ({(int)response.StatusCode})."); + continue; + } + + var index = await response.Content + .ReadFromJsonAsync(NuGetJsonContext.Default.FlatContainerIndex, cancellationToken) + .ConfigureAwait(false); + var latest = SelectLatestStable(index?.Versions); + if (latest is null) + { + last = new InvalidOperationException( + $"Could not resolve latest stable version of {packageId}."); + continue; + } + + return latest; + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + last = ex; + } + } + + throw last ?? new InvalidOperationException($"Could not resolve latest stable version of {packageId}."); + } + + static async Task DownloadRidPackageAsync( + HttpClient http, + string packageId, + string version, + string directory, + TextWriter? log, + CancellationToken cancellationToken) + { + Exception? last = null; + foreach (var (flat, registration) in new (string Flat, string? Registration)[] + { + (NugetFlatContainer, NugetRegistration), + (BlobFlatContainer, null), + }) + { + var url = NupkgUrl(flat, packageId, version); + var destination = Path.Combine(directory, Path.GetFileName(url)); + try + { + log?.WriteLine($"Downloading {url}"); + using var response = await http.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken) + .ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + last = new InvalidOperationException( + $"Failed to download {url} ({(int)response.StatusCode})."); + continue; + } + + await using (var input = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false)) + await using (var output = File.Create(destination)) + { + await input.CopyToAsync(output, cancellationToken).ConfigureAwait(false); + } + + if (registration is not null) + await VerifyCatalogHashAsync(http, registration, packageId, version, destination, log, cancellationToken) + .ConfigureAwait(false); + + return destination; + } + catch (PackageHashMismatchException) + { + throw; + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + last = ex; + TryDelete(destination); + } + } + + throw last ?? new InvalidOperationException($"Failed to download {packageId} {version}."); + } + + static async Task VerifyCatalogHashAsync( + HttpClient http, + string registrationBase, + string packageId, + string version, + string nupkgPath, + TextWriter? log, + CancellationToken cancellationToken) + { + var leafUrl = RegistrationLeafUrl(registrationBase, packageId, version); + log?.WriteLine($"GET {leafUrl}"); + using var leafResponse = await http.GetAsync(leafUrl, cancellationToken).ConfigureAwait(false); + if (!leafResponse.IsSuccessStatusCode) + { + throw new InvalidOperationException( + $"Failed to read registration for {packageId} {version} ({(int)leafResponse.StatusCode})."); + } + + var leaf = await leafResponse.Content + .ReadFromJsonAsync(NuGetJsonContext.Default.RegistrationLeaf, cancellationToken) + .ConfigureAwait(false); + var catalogUrl = CatalogEntryUrl(leaf?.CatalogEntry ?? default); + if (string.IsNullOrWhiteSpace(catalogUrl)) + throw new InvalidOperationException($"Registration for {packageId} {version} has no catalog entry."); + + log?.WriteLine($"GET {catalogUrl}"); + using var catalogResponse = await http.GetAsync(catalogUrl, cancellationToken).ConfigureAwait(false); + if (!catalogResponse.IsSuccessStatusCode) { throw new InvalidOperationException( - $"Could not resolve latest release of {repo} ({(int)response.StatusCode})."); + $"Failed to read catalog entry for {packageId} {version} ({(int)catalogResponse.StatusCode})."); } - var release = await response.Content - .ReadFromJsonAsync(NuGetJsonContext.Default.GitHubRelease, cancellationToken) + var catalog = await catalogResponse.Content + .ReadFromJsonAsync(NuGetJsonContext.Default.CatalogLeaf, cancellationToken) .ConfigureAwait(false); - var tag = release?.TagName; - if (string.IsNullOrWhiteSpace(tag)) - throw new InvalidOperationException($"Could not resolve latest release of {repo}."); + var expected = catalog?.PackageHash?.Trim(); + if (string.IsNullOrEmpty(expected) || + !string.Equals(catalog?.PackageHashAlgorithm, "SHA512", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException( + $"Catalog entry for {packageId} {version} has no SHA512 package hash."); + } - return NormalizeVersion(tag); + var actual = Convert.ToBase64String(SHA512.HashData(File.ReadAllBytes(nupkgPath))); + if (!string.Equals(expected, actual, StringComparison.Ordinal)) + { + throw new PackageHashMismatchException( + $"SHA512 mismatch for {Path.GetFileName(nupkgPath)}{Environment.NewLine} expected: {expected}{Environment.NewLine} actual: {actual}"); + } } + static string? CatalogEntryUrl(JsonElement catalogEntry) + { + switch (catalogEntry.ValueKind) + { + case JsonValueKind.String: + return catalogEntry.GetString(); + case JsonValueKind.Object: + if (catalogEntry.TryGetProperty("@id", out var id) || + catalogEntry.TryGetProperty("@Id", out id)) + return id.GetString(); + return null; + default: + return null; + } + } + + static void ExtractNupkgBinary(string nupkgPath, string rid, string binaryName, string destination) + { + var wanted = "tools/any/" + rid.ToLowerInvariant() + "/" + binaryName; + using var zip = ZipFile.OpenRead(nupkgPath); + var entry = zip.Entries.FirstOrDefault(e => + e.FullName.Replace('\\', '/').Equals(wanted, StringComparison.OrdinalIgnoreCase)); + if (entry is null) + throw new InvalidOperationException($"Package did not contain {wanted}."); + + entry.ExtractToFile(destination, overwrite: true); + } + + static string Combine(string baseUrl, string relative) + { + if (!baseUrl.EndsWith('/')) + baseUrl += "/"; + return baseUrl + relative.TrimStart('/'); + } + + sealed class PackageHashMismatchException(string message) : InvalidOperationException(message); + static async Task DownloadAsync(HttpClient http, string url, string destination, CancellationToken cancellationToken) { using var response = await http.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken) @@ -360,12 +612,10 @@ static void TryDelete(string path) catch (UnauthorizedAccessException) { } } - static void EnsureGitHubHeaders(HttpClient http) + static void EnsureUserAgent(HttpClient http) { if (http.DefaultRequestHeaders.UserAgent.Count == 0) http.DefaultRequestHeaders.UserAgent.ParseAdd("ndx"); - if (http.DefaultRequestHeaders.Accept.Count == 0) - http.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json")); } static bool IsDetailed(string? verbosity)