diff --git a/internal/lfs/batch.go b/internal/lfs/batch.go
index ba2de10..a08f62e 100644
--- a/internal/lfs/batch.go
+++ b/internal/lfs/batch.go
@@ -6,12 +6,14 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
+ "errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
+ "strings"
"time"
)
@@ -30,9 +32,25 @@ type batchRequest struct {
// batchAction is one server-provided action, here the basic-transfer download
// instruction with its (typically pre-signed) URL.
+//
+// The expiry fields matter to a long run: a pre-signed URL stops working once it
+// lapses, and the server is the only party that can issue a fresh one.
type batchAction struct {
- Href string `json:"href"`
- Header map[string]string `json:"header"`
+ Href string `json:"href"`
+ Header map[string]string `json:"header"`
+ ExpiresIn int64 `json:"expires_in"`
+ ExpiresAt string `json:"expires_at"`
+}
+
+// expired reports whether the action's URL has already lapsed. A past ExpiresAt
+// is authoritative; otherwise the relative expiry is measured from now.
+func (a *batchAction) expired() bool {
+ if a.ExpiresAt != "" {
+ if at, err := time.Parse(time.RFC3339, a.ExpiresAt); err == nil {
+ return !at.After(time.Now())
+ }
+ }
+ return a.ExpiresIn < 0
}
// batchResponseObject is the server's verdict for one object; exactly one of
@@ -53,6 +71,30 @@ type batchResponse struct {
Objects []batchResponseObject `json:"objects"`
}
+// errBatchRejected marks a batch request the endpoint answered with an
+// unexpected status instead of scheduling the objects. Callers narrow such a
+// request down to individual objects, because a rejection is usually one
+// object's fault (the endpoint's object limit, a stale pointer) rather than the
+// whole batch's.
+var errBatchRejected = errors.New("batch request rejected by the endpoint")
+
+// errObjectUnavailable reports that the endpoint answered the batch request but
+// will not serve one of the objects it named, so the object cannot be mirrored.
+var errObjectUnavailable = errors.New("lfs object unavailable on the remote")
+
+// batchObjectError is one object the endpoint refused: a pointer kept in the
+// repository after its object was removed or garbage-collected on the server.
+type batchObjectError struct {
+ oid string
+ message string
+}
+
+func (e *batchObjectError) Error() string {
+ return fmt.Sprintf("LFS object %s unavailable: %s", shortOID(e.oid), e.message)
+}
+
+func (e *batchObjectError) Unwrap() error { return errObjectUnavailable }
+
// batchClient performs the LFS batch protocol over HTTP.
type batchClient struct {
client *http.Client
@@ -67,10 +109,140 @@ func newBatchClient(client *http.Client) *batchClient {
const lfsMediaType = "application/vnd.git-lfs+json"
-// batch submits every pointer for download scheduling. A 403 or 404 from the
-// endpoint means the remote has Git LFS switched off, which callers treat as
-// an expected skip rather than a failure.
-func (c *batchClient) batch(ctx context.Context, endpoint, username, password string, pointers []pointer) ([]batchResponseObject, error) {
+// authorize offers the credentials to a request when there are any to offer.
+// Every batch request goes out authenticated rather than waiting for a 401,
+// because a forge is free to answer 403 to an anonymous request, and a 403 is
+// also how a repository with LFS switched off answers.
+func (c credentials) authorize(request *http.Request) {
+ if !c.available() {
+ return
+ }
+ request.SetBasicAuth(c.username, c.password)
+}
+
+// probe asks an endpoint for one object to find out whether it serves this
+// repository's LFS API at all. Discovery happens once, before any object is
+// fetched, so the fetch itself runs against a single known-good endpoint: the
+// answer separates "the API answered for this object" — including that the
+// server does not have it — from "LFS is switched off here" and from "nothing is
+// mounted at this path".
+//
+// A single object keeps the probe from tripping a server's object limit, so a
+// refused probe is about the path rather than the request's shape. A nil return
+// therefore means the API root is right, and the object's own fate is the
+// fetch's business.
+//
+// The credentials are offered on the first request when there are any. Waiting
+// for a 401 to offer them would read a forge that answers 403 to anonymous
+// requests as a repository with LFS switched off, which backs it up without its
+// LFS content instead of reporting that the credential was refused.
+func (c *batchClient) probe(ctx context.Context, endpoint string, creds credentials, object pointer) error {
+ return c.doProbe(ctx, endpoint, creds, object)
+}
+
+// doProbe performs one probe request, offering the credentials when there are
+// any to offer.
+func (c *batchClient) doProbe(ctx context.Context, endpoint string, creds credentials, object pointer) error {
+ body, err := json.Marshal(batchRequest{
+ Operation: "download",
+ Transfers: []string{"basic"},
+ Objects: toBatchObjects([]pointer{object}),
+ })
+ if err != nil {
+ return err
+ }
+
+ request, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint+"/objects/batch", bytes.NewReader(body))
+ if err != nil {
+ return err
+ }
+ request.Header.Set("Content-Type", lfsMediaType)
+ request.Header.Set("Accept", lfsMediaType)
+ creds.authorize(request)
+
+ response, err := c.client.Do(request)
+ if err != nil {
+ return fmt.Errorf("batch request: %w", err)
+ }
+ defer drainAndClose(response)
+
+ switch response.StatusCode {
+ case http.StatusOK:
+ // The API answered for the object, whether by scheduling it or by
+ // reporting that the server does not have it.
+ return nil
+ case http.StatusUnauthorized:
+ // The endpoint refused what it was sent, and the credentials were
+ // already offered, so this is reported rather than retried: repeating
+ // the request would send the same credentials again. Reading it as a
+ // path problem would be worse still, because the path answered.
+ return fmt.Errorf("batch request failed with status %d", response.StatusCode)
+ case http.StatusForbidden:
+ // Two different things answer 403: a forge saying LFS is switched off
+ // for this repository, and a host page refusing a path it does not
+ // serve. The LFS API explains itself — in JSON, and sometimes with no
+ // body at all, which is still a verdict about the repository — while an
+ // edge page does not.
+ return classifyRefusal(response, ErrDisabled)
+ case http.StatusNotFound:
+ // Nothing is mounted at this path: a host that routes by path answers
+ // 404.
+ return ErrNoEndpoint
+ case http.StatusUnprocessableEntity:
+ // Unprocessable is how an edge refuses a path it does not route as much
+ // as it is how the API refuses a request's shape. A single object cannot
+ // trip an object limit, so a page here is a wrong path and anything else
+ // is the API refusing the probe.
+ return classifyRefusal(response, fmt.Errorf("batch request failed with status %d", response.StatusCode))
+ default:
+ // Every other status describes the endpoint or the credential rather
+ // than the path — a rate limit, an authentication failure, the server's
+ // own error — so it stays a reported failure. Reading an edge's error
+ // page as "no endpoint here" would let the mirror layer record a broken
+ // or unauthorized service as a repository with LFS switched off.
+ return fmt.Errorf("batch request failed with status %d", response.StatusCode)
+ }
+}
+
+// classifyRefusal reads a refusal the API did not phrase as a verdict about one
+// object. Markup means a host page answered — the path is not the API — and
+// anything else is the API speaking, which the caller reads as verdict.
+func classifyRefusal(response *http.Response, verdict error) error {
+ body, err := io.ReadAll(io.LimitReader(response.Body, 4096))
+ if err == nil && looksLikeHTML(body) {
+ return ErrNoEndpoint
+ }
+ return verdict
+}
+
+// credentials are the HTTP basic-auth pair the remote's own API accepts. They
+// travel as a value rather than as raw strings because a client may need to
+// offer them more than once: the batch API answers 401 when credentials are
+// needed but were not sent, and the request is then retried once with them.
+type credentials struct {
+ username string
+ password string
+}
+
+// available reports whether there is anything to offer. A request with no
+// credentials to offer goes out anonymous, because there is nothing to add.
+func (c credentials) available() bool {
+ return c.username != "" || c.password != ""
+}
+
+// batch submits every pointer for download scheduling.
+//
+// A status describing the request's shape is reported as errBatchRejected so
+// callers can retry the pointers in smaller batches; every other status
+// describes the endpoint rather than what was asked of it, so splitting the
+// chunk would only repeat the same answer more slowly. Discovery already
+// established which endpoint serves the API, so a 403 or 404 here describes a
+// service that has stopped answering rather than a path worth retrying.
+//
+// A 401 means the endpoint would not accept what was sent, so it is reported
+// rather than retried: the credentials were already offered, and repeating the
+// request would send the same ones.
+func (c *batchClient) batch(ctx context.Context, creds credentials, endpoint string, pointers []pointer) ([]batchResponseObject, error) {
body, err := json.Marshal(batchRequest{
Operation: "download",
Transfers: []string{"basic"},
@@ -86,9 +258,7 @@ func (c *batchClient) batch(ctx context.Context, endpoint, username, password st
}
request.Header.Set("Content-Type", lfsMediaType)
request.Header.Set("Accept", lfsMediaType)
- if username != "" || password != "" {
- request.SetBasicAuth(username, password)
- }
+ creds.authorize(request)
response, err := c.client.Do(request)
if err != nil {
@@ -96,11 +266,11 @@ func (c *batchClient) batch(ctx context.Context, endpoint, username, password st
}
defer drainAndClose(response)
- switch response.StatusCode {
- case http.StatusOK:
+ switch {
+ case response.StatusCode == http.StatusOK:
// Handled below.
- case http.StatusForbidden, http.StatusNotFound:
- return nil, ErrDisabled
+ case describesRequestShape(response.StatusCode):
+ return nil, fmt.Errorf("%w: status %d", errBatchRejected, response.StatusCode)
default:
return nil, fmt.Errorf("batch request failed with status %d", response.StatusCode)
}
@@ -115,49 +285,141 @@ func (c *batchClient) batch(ctx context.Context, endpoint, username, password st
return decoded.Objects, nil
}
+// describesRequestShape reports whether a status is how servers answer a request
+// they will not process as sent, which a smaller batch may avoid: an unprocessable
+// entity, a payload the server considers too large, or a body it cannot accept.
+// Credentials, rate limiting, and the endpoint's own failures are not among them.
+func describesRequestShape(status int) bool {
+ switch status {
+ case http.StatusBadRequest,
+ http.StatusRequestEntityTooLarge,
+ http.StatusUnprocessableEntity,
+ http.StatusNotAcceptable,
+ http.StatusUnsupportedMediaType:
+ return true
+ default:
+ return false
+ }
+}
+
+// looksLikeHTML reports whether a body is a markup document, which is what an
+// edge or proxy answers with and what the LFS API never does. A JSON document or
+// an empty body is therefore read as the API speaking; markup is read as the path
+// being refused by something that is not the API.
+func looksLikeHTML(body []byte) bool {
+ trimmed := bytes.TrimSpace(bytes.ToLower(body))
+ return bytes.HasPrefix(trimmed, []byte(" 0 {
+ size = wanted
+ }
+ if cached, err := os.Stat(filepath.Join(store, oid[0:2], oid[2:4], oid)); err == nil &&
+ cached.Mode().IsRegular() && (size == 0 || cached.Size() == size) {
+ // Already cached by a previous snapshot; git-lfs also skips it, and
+ // content already held needs no fresh URL, so this is settled before
+ // the expiry below can send it back for rescheduling. A file of a
+ // length the recorded size contradicts is not this object, so it
+ // falls through to the download, which verifies the digest as bytes
+ // arrive. No size anywhere leaves the length unstated rather than
+ // contradicted, and re-downloading content the store already holds
+ // would risk failing over it.
+ continue
}
-
action := object.Actions["download"]
- if action == nil || action.Href == "" {
- // The server knows the object but scheduled nothing; without a
- // href there is nothing this client can do beyond reporting it.
- return fmt.Errorf("LFS object %s has no download action", shortOID(object.OID))
+ if action.expired() {
+ // The URL lapsed before it was used — a long scan, a slow batch, a
+ // small expires_in — so the object needs a fresh one rather than a
+ // transfer that can only fail. The resolved size travels with it, so
+ // the refresh request and the pass over its answer keep the length
+ // the pointer records rather than falling back to an endpoint that
+ // stated none.
+ expired = append(expired, pointer{oid: oid, size: size})
+ continue
}
-
- if err := downloadObject(ctx, client.client, store, endpointHost, username, password, object.OID, action); err != nil {
- return err
+ if err := downloadObject(ctx, client.client, store, endpointHost, creds, oid, action); err != nil {
+ failed = append(failed, err)
}
}
- return nil
+ return expired, unavailable, failed
}
+// downloadObject streams one object into the LFS cache, verifying its SHA-256 as
+// bytes arrive.
func downloadObject(
ctx context.Context,
client *http.Client,
- store, endpointHost, username, password, oid string,
+ store, endpointHost string,
+ creds credentials,
+ oid string,
action *batchAction,
) error {
destination := filepath.Join(store, oid[0:2], oid[2:4], oid)
- if _, err := os.Stat(destination); err == nil {
- // Already cached by a previous snapshot; git-lfs also skips it.
- return nil
- }
request, err := http.NewRequestWithContext(ctx, http.MethodGet, action.Href, nil)
if err != nil {
@@ -169,8 +431,8 @@ func downloadObject(
// Pre-signed download URLs carry their own authorization; the basic
// credential only applies while the request stays on the LFS endpoint's
// host.
- if hostOf(action.Href) == endpointHost && (username != "" || password != "") {
- request.SetBasicAuth(username, password)
+ if hostOf(action.Href) == endpointHost && creds.available() {
+ request.SetBasicAuth(creds.username, creds.password)
}
response, err := client.Do(request)
diff --git a/internal/lfs/endpoint.go b/internal/lfs/endpoint.go
new file mode 100644
index 0000000..be57045
--- /dev/null
+++ b/internal/lfs/endpoint.go
@@ -0,0 +1,251 @@
+package lfs
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "net/url"
+ "strings"
+
+ "github.com/go-git/go-git/v5"
+ "github.com/neurekadev/git-backup/internal/paths"
+)
+
+// endpointCandidates lists, in the order they should be asked, the API roots a
+// repository's LFS objects may live under: an lfs.url override from the
+// repository's committed .lfsconfig when present, otherwise the remote URL's
+// standard /info/lfs root with and without the repository's .git suffix.
+//
+// Which of the derived roots answers is a property of the host, and no rule
+// decides it: GitHub and GitLab route only the suffixed path to their LFS
+// service and answer a suffix-less /info/lfs with 422, while a host may mount
+// the service exactly where its remote points. The distinction is therefore
+// settled by asking, once, before any object is fetched — see selectEndpoint.
+//
+// The batch request authenticates with the remote's credential, so an override
+// is honored only when it is an absolute http(s) URL on the remote's own host
+// and scheme: a hostile .lfsconfig must not redirect that credential elsewhere
+// or downgrade it to plaintext. Reading .lfsconfig is best-effort — any failure
+// falls back to the derived candidates, matching the common deployment.
+// endpointCandidate is one API root worth asking, and whether this package
+// derived it rather than taking it from the configured remote.
+//
+// The distinction decides how much an answer weighs. A path the remote actually
+// points at is the repository's own root, so its verdict speaks for the
+// repository; a derived path is a guess about how the host routes LFS, and a
+// guess answering "LFS is off" is weaker evidence than a real failure on the
+// path the remote configures.
+type endpointCandidate struct {
+ url string
+ // derived marks the suffixed root this package adds when the configured
+ // remote does not carry one.
+ derived bool
+}
+
+// endpointCandidates lists, in the order they should be asked, the API roots a
+// repository's LFS objects may live under: an lfs.url override from the
+// repository's committed .lfsconfig when present, otherwise the remote URL's
+// standard /info/lfs root with and without the repository's .git suffix.
+//
+// Which of the derived roots answers is a property of the host, and no rule
+// decides it: GitHub and GitLab route only the suffixed path to their LFS
+// service and answer a suffix-less /info/lfs with 422, while a host may mount
+// the service exactly where its remote points. The distinction is therefore
+// settled by asking, once, before any object is fetched — see selectEndpoint.
+//
+// The batch request authenticates with the remote's credential, so an override
+// is honored only when it is an absolute http(s) URL on the remote's own host
+// and scheme: a hostile .lfsconfig must not redirect that credential elsewhere
+// or downgrade it to plaintext. Reading .lfsconfig is best-effort — any failure
+// falls back to the derived candidates, matching the common deployment.
+func endpointCandidates(repository *git.Repository, parsed *url.URL) ([]endpointCandidate, error) {
+ config, err := readLFSConfig(repository)
+ if err == nil && config != nil {
+ if override := strings.TrimSpace(config.Raw.Section("lfs").Option("url")); override != "" {
+ overridden, ok := paths.ParseHTTPURL(override)
+ if !ok {
+ return nil, fmt.Errorf("unsupported lfs.url '%s': only absolute http and https URLs are allowed", redactedURL(override))
+ }
+ if !strings.EqualFold(overridden.Scheme, parsed.Scheme) || canonicalHost(overridden) != canonicalHost(parsed) {
+ return nil, fmt.Errorf("refusing lfs.url '%s': the override must stay on the remote host and scheme so the remote credential is not sent elsewhere", redactedURL(override))
+ }
+ // Drop a redundant scheme-default port so the endpoint is canonical:
+ // object-download host comparisons and logs then match hrefs rendered
+ // without the explicit port.
+ if overridden.Port() != "" && isSchemeDefaultPort(overridden) {
+ overridden.Host = strings.TrimSuffix(overridden.Host, ":"+overridden.Port())
+ }
+ // An override's own query, forced query, and fragment cannot ride
+ // along: the action path is appended to the endpoint, so anything
+ // after the path would land in the middle of the request line.
+ cleaned := endpointBase(overridden)
+ return []endpointCandidate{{url: strings.TrimSuffix(cleaned.String(), "/")}}, nil
+ }
+ }
+
+ // A remote that already carries the repository suffix, or has no path at
+ // all, reaches the API at one path, so there is nothing to choose between.
+ suffixed, plain := suffixedEndpoint(parsed), plainEndpoint(parsed)
+ if suffixed == plain {
+ return []endpointCandidate{{url: suffixed}}, nil
+ }
+ // The configured remote's own path is asked second only because a forge that
+ // requires the suffix is the more common deployment; it is not the weaker
+ // candidate.
+ return []endpointCandidate{{url: suffixed, derived: true}, {url: plain}}, nil
+}
+
+// selectEndpoint answers which candidate serves this repository's LFS API, by
+// asking each one for a single object before anything is downloaded.
+//
+// Asking first is what keeps the fetch itself simple: a probe separates the
+// answers a host can give — the API answered for the object, LFS is switched
+// off, nothing is mounted at that path, or the endpoint failed — so the fetch
+// then runs against one known-good endpoint with no fallback to reconcile.
+//
+// Every candidate is asked, and what decides the repository is the answer from
+// the path the remote itself configures. A derived guess never overrides that
+// path's answer in either direction: a guess reporting "LFS is off" must not mask
+// a credential, rate-limit, server, or transport failure on the configured path
+// — the mirror layer records disabled as a successful skip, so that would mirror
+// the repository with no LFS content and raise nothing — and a guess failing must
+// not turn a configured path's "LFS is off" into a failed backup. The guess only
+// decides when the configured path said nothing itself, which is the case it
+// exists for: a host that routes LFS solely under the suffixed path.
+func (f *Fetcher) selectEndpoint(
+ ctx context.Context,
+ candidates []endpointCandidate,
+ creds credentials,
+ first pointer,
+) (string, error) {
+ var configured, derived error
+ for _, candidate := range candidates {
+ switch err := f.client.probe(ctx, candidate.url, creds, first); {
+ case err == nil:
+ slog.Debug("Git LFS API answered.", "endpoint", redactedURL(candidate.url))
+ return candidate.url, nil
+ case errors.Is(err, ErrDisabled):
+ slog.Debug("Git LFS is switched off for this repository.", "endpoint", redactedURL(candidate.url))
+ if candidate.derived {
+ if derived == nil {
+ derived = err
+ }
+ } else if configured == nil {
+ configured = err
+ }
+ case errors.Is(err, ErrNoEndpoint):
+ slog.Debug("No Git LFS API is mounted at this endpoint.", "endpoint", redactedURL(candidate.url))
+ default:
+ // The endpoint and the error both describe URLs, and the error's own
+ // text repeats the one the request went to, so it is redacted too
+ // before it reaches the log.
+ slog.Debug("Git LFS API did not answer.",
+ "endpoint", redactedURL(candidate.url), "detail", redactedURL(err.Error()))
+ if candidate.derived {
+ if derived == nil {
+ derived = err
+ }
+ } else if configured == nil {
+ configured = err
+ }
+ }
+ }
+
+ if configured != nil {
+ if derived != nil {
+ // The guess's answer is kept for the debug trail rather than
+ // discarded — it is just not what decides the fetch.
+ slog.Debug("The derived path answered differently from the configured one.",
+ "detail", redactedURL(derived.Error()))
+ }
+ return "", configured
+ }
+ if derived != nil {
+ return "", derived
+ }
+ // No candidate serves the API. The mirror layer records that as LFS being
+ // switched off rather than failing the repository, which is the closest
+ // truthful reading available from a client.
+ return "", ErrDisabled
+}
+
+// suffixedEndpoint derives the remote's standard LFS API root, including the
+// repository's .git suffix.
+//
+// The parsed URL is copied and only its path rewritten, from its escaped form so
+// percent-encoding survives: a remote whose path holds a reserved byte — an
+// encoded slash in a repository name, say — must keep requesting that exact
+// path. Everything else the configured remote carries, userinfo for a deployment
+// that embeds credentials and a non-default port among it, still reaches the
+// endpoint. A remote with no path has no repository to suffix, but the API root
+// is still /info/lfs on that host.
+func suffixedEndpoint(remoteURL *url.URL) string {
+ endpoint := endpointBase(remoteURL)
+
+ escaped := strings.TrimSuffix(remoteURL.EscapedPath(), "/")
+ if escaped == "" {
+ return withPath(endpoint, "/info/lfs")
+ }
+ // The suffix is judged on the decoded path but appended to the escaped one,
+ // so a remote whose escaping hides it (re%2Egit) is not given a second
+ // suffix it did not need. It is matched case-insensitively, as the mirror's
+ // other .git handling does: a remote already ending in ".GIT" must not gain
+ // a differently cased one that no forge serves.
+ if !hasGitSuffix(strings.TrimSuffix(remoteURL.Path, "/")) {
+ escaped += ".git"
+ }
+ return withPath(endpoint, escaped+"/info/lfs")
+}
+
+// plainEndpoint derives the LFS API root at the remote's configured path,
+// without adding the repository suffix.
+func plainEndpoint(remoteURL *url.URL) string {
+ endpoint := endpointBase(remoteURL)
+ return withPath(endpoint, strings.TrimSuffix(remoteURL.EscapedPath(), "/")+"/info/lfs")
+}
+
+// endpointBase copies a remote URL with the parts an API root cannot carry
+// removed: a query, a forced empty query, and a fragment would otherwise sit
+// between the endpoint and the action path appended to it.
+func endpointBase(remoteURL *url.URL) url.URL {
+ endpoint := *remoteURL
+ endpoint.RawQuery = ""
+ endpoint.ForceQuery = false
+ endpoint.Fragment = ""
+ endpoint.RawFragment = ""
+ return endpoint
+}
+
+// withPath points an endpoint at an escaped path, keeping Path decoded so
+// String() does not escape the escaping a second time.
+func withPath(endpoint url.URL, escaped string) string {
+ endpoint.RawPath = escaped
+ endpoint.Path, _ = url.PathUnescape(escaped)
+ return endpoint.String()
+}
+
+// hasGitSuffix reports whether a remote path already ends in the repository
+// suffix, in any case.
+func hasGitSuffix(path string) bool {
+ return len(path) >= len(".git") && strings.EqualFold(path[len(path)-len(".git"):], ".git")
+}
+
+// isSchemeDefaultPort reports whether the URL's explicit port equals its
+// scheme's default (http 80, https 443).
+func isSchemeDefaultPort(u *url.URL) bool {
+ return (strings.EqualFold(u.Scheme, "http") && u.Port() == "80") ||
+ (strings.EqualFold(u.Scheme, "https") && u.Port() == "443")
+}
+
+// canonicalHost renders the URL's host lowercased with any scheme-default
+// port removed, so an explicit https://host:443 compares equal to
+// https://host while a redirect to any other port stays distinct.
+func canonicalHost(u *url.URL) string {
+ host := strings.ToLower(u.Hostname())
+ port := u.Port()
+ if port == "" || isSchemeDefaultPort(u) {
+ return host
+ }
+ return host + ":" + port
+}
diff --git a/internal/lfs/lfs.go b/internal/lfs/lfs.go
index 8e6f589..ebd6422 100644
--- a/internal/lfs/lfs.go
+++ b/internal/lfs/lfs.go
@@ -14,6 +14,7 @@ import (
"errors"
"fmt"
"io"
+ "log/slog"
"net/url"
"path/filepath"
"strings"
@@ -45,92 +46,353 @@ func NewFetcher() *Fetcher {
// repositoryPath, using remoteURL to derive the LFS endpoint. username and
// password authenticate the batch request only; object downloads use the
// server-provided (typically pre-signed) URLs.
+//
+// The work splits into three steps with one job each: reject a remote URL this
+// package will not talk to, scan the mirror for pointers, settle which endpoint
+// serves this repository's LFS API, and then fetch from that endpoint alone.
+// Settling the endpoint first is what keeps the fetch simple — no fallback to
+// reconcile, and no way for a working endpoint to be mistaken for a missing one
+// partway through a repository.
func (f *Fetcher) FetchAll(ctx context.Context, repositoryPath, remoteURL, username, password string) error {
repository, err := git.PlainOpen(repositoryPath)
if err != nil {
return fmt.Errorf("open repository: %w", err)
}
-
- endpoint, err := resolveEndpoint(repository, remoteURL)
+ parsed, err := parseRemoteURL(remoteURL)
if err != nil {
return err
}
- pointers, err := collectPointers(ctx, repository)
+ pointers, err := f.pointers(ctx, repository, remoteURL)
+ if err != nil || len(pointers) == 0 {
+ return err
+ }
+
+ candidates, err := endpointCandidates(repository, parsed)
if err != nil {
- return fmt.Errorf("scan for LFS pointers: %w", err)
+ return err
}
- if len(pointers) == 0 {
- // Nothing to fetch; never contact the endpoint so a forge without LFS
- // is not mistaken for one that disabled it.
- return nil
+ creds := credentials{username: username, password: password}
+ endpoint, err := f.selectEndpoint(ctx, candidates, creds, pointers[0])
+ if err != nil {
+ return err
}
- objects, err := f.client.batch(ctx, endpoint, username, password, pointers)
+ store := objectStoreDir(repository, repositoryPath)
+ outcome, err := f.fetchBatches(ctx, store, endpoint, creds, pointers)
if err != nil {
return err
}
+ if outcome.unavailable > 0 {
+ // The endpoint serves this repository but does not have every object a
+ // pointer names. The mirror keeps what it got; the gap is reported
+ // rather than passed over, because a mirror missing LFS content is not a
+ // complete backup.
+ slog.Warn("Some Git LFS objects could not be fetched; the repository was mirrored without them.",
+ "endpoint", redactedURL(endpoint), "objectsMissing", outcome.unavailable, "objectsRequested", len(pointers))
+ slog.Debug("Git LFS objects that could not be fetched.",
+ "endpoint", redactedURL(endpoint), "reason", outcome.skipped.Error())
+ return fmt.Errorf("%d of %d LFS objects could not be fetched: %w",
+ outcome.unavailable, len(pointers), outcome.skipped)
+ }
+ return nil
+}
- store := objectStoreDir(repository, repositoryPath)
- return downloadObjects(ctx, f.client, store, endpoint, username, password, objects)
-}
-
-// resolveEndpoint determines the LFS API root: an lfs.url override from the
-// repository's committed .lfsconfig when present, otherwise the remote URL's
-// standard /info/lfs root. The batch request authenticates with the remote's
-// credential, so an override is honored only when it is an absolute http(s)
-// URL on the remote's own host and scheme — a hostile .lfsconfig must not
-// redirect that credential elsewhere or downgrade it to plaintext. Reading
-// .lfsconfig is best-effort — any failure falls back to the derived endpoint,
-// matching the common deployment.
-func resolveEndpoint(repository *git.Repository, remoteURL string) (string, error) {
- remote := strings.TrimSuffix(remoteURL, "/")
- parsed, ok := paths.ParseHTTPURL(remote)
+// parseRemoteURL rejects a remote this package will not talk to: only http and
+// https are accepted, so a provider-supplied URL can never reach a transport
+// helper or a local path.
+func parseRemoteURL(remoteURL string) (*url.URL, error) {
+ parsed, ok := paths.ParseHTTPURL(remoteURL)
if !ok {
- return "", fmt.Errorf("unsupported remote URL '%s': only http and https are allowed", redactedURL(remoteURL))
+ return nil, fmt.Errorf("unsupported remote URL '%s': only http and https are allowed", redactedURL(remoteURL))
+ }
+ return parsed, nil
+}
+
+// pointers scans the mirror for the LFS pointers its refs reach, reporting the
+// parts of the repository it could not read. A scan that could not read
+// everything fails the fetch: a backup reported as complete while content it
+// could not scan is silently absent is worse than one that reports what it could
+// not do.
+func (f *Fetcher) pointers(ctx context.Context, repository *git.Repository, remoteURL string) ([]pointer, error) {
+ scan, err := collectPointers(ctx, repository)
+ if err != nil {
+ return nil, fmt.Errorf("scan for LFS pointers: %w", err)
+ }
+ if len(scan.skipped) > 0 {
+ for _, skipped := range scan.skipped {
+ slog.Warn("Could not read part of the repository while scanning for Git LFS pointers.",
+ "repository", redactedURL(remoteURL), "entry", skipped.name,
+ "oid", shortOID(skipped.hash.String()), "reason", skipped.err.Error())
+ }
+ return nil, fmt.Errorf("%d entries could not be read while scanning for LFS pointers, so the mirror would be incomplete", len(scan.skipped))
+ }
+ // Nothing to fetch means the endpoint is never contacted, so a forge without
+ // LFS is not mistaken for one that disabled it.
+ return scan.pointers, nil
+}
+
+// batchObjectLimit is how many pointers one batch request submits. Git LFS
+// clients use the same limit; a forge may reject a request that exceeds its own
+// smaller limit, which fetchChunk then narrows down.
+//
+// Narrowing is not remembered between chunks, deliberately. A refusal cannot
+// distinguish a server whose limit is low from a chunk carrying one object the
+// server will not serve, and a remembered bound derived from the second kind
+// would shrink every later chunk — including chunks of perfectly servable
+// objects. The cost of not remembering is bounded: at most two requests per
+// pointer in a chunk, and the sweep stops after a second failing chunk.
+const batchObjectLimit = 100
+
+// ErrNoEndpoint reports that nothing answered the LFS batch API at an endpoint,
+// which is how a forge that routes the API elsewhere — or at all — looks to a
+// client. FetchAll tries its next candidate endpoint on this answer, and a
+// repository with no service anywhere is reported with ErrDisabled, because from
+// here that is indistinguishable from LFS being switched off and the mirror
+// layer already knows how to record it.
+var ErrNoEndpoint = errors.New("no git lfs service answered at the endpoint")
+
+// fetchBatches downloads every pointer's object, splitting the pointers into
+// batch requests. A batch the endpoint rejects is narrowed down by halving, so
+// one object the server will not serve — a stale pointer, or a request above
+// the server's own limit — costs that object alone instead of the repository's
+// entire LFS mirror. Failing to mirror an object that exists still errors the
+// fetch.
+//
+// Narrowing a chunk costs up to two requests per pointer in it, not a
+// logarithmic count: a half the endpoint answered nothing for may hold nothing
+// but individually refused objects, so the only way to know whether its sibling
+// is servable is to ask it. A chunk the endpoint answered nothing for therefore
+// still reports, and the sweep continues, because that state may mean a broken
+// endpoint or merely a chunk of stale pointers. Once a second chunk fails, the
+// fetch stops and reports rather than repeating the failure across a large
+// repository.
+func (f *Fetcher) fetchBatches(
+ ctx context.Context,
+ store, endpoint string,
+ creds credentials,
+ pointers []pointer,
+) (chunkOutcome, error) {
+ var total chunkOutcome
+ var firstErr error
+ for start := 0; start < len(pointers); start += batchObjectLimit {
+ end := min(start+batchObjectLimit, len(pointers))
+ if err := ctx.Err(); err != nil {
+ return total, err
+ }
+
+ outcome, err := f.fetchChunk(ctx, store, endpoint, creds, pointers[start:end])
+ total.absorb(outcome)
+ if err == nil {
+ continue
+ }
+ // A chunk the endpoint answered nothing for is reported, but the rest of
+ // the repository is still attempted: that state may mean a broken
+ // endpoint or merely a chunk of stale pointers, and only the next chunk
+ // tells them apart.
+ if firstErr != nil {
+ // The endpoint failed a second chunk, so stop asking.
+ slog.Debug("Stopping the Git LFS fetch after another chunk failed.",
+ "endpoint", redactedURL(endpoint), "detail", err.Error())
+ break
+ }
+ firstErr = err
+ }
+
+ if firstErr != nil {
+ return total, firstErr
+ }
+ if missing := total.unavailable + total.rejected; missing > 0 {
+ return total, fmt.Errorf("%d of %d LFS objects could not be fetched: %w", missing, len(pointers), total.skipped)
}
- endpoint := remote + "/info/lfs"
+ return total, nil
+}
+
+// chunkOutcome reports what one batch request could not fetch — objects the
+// endpoint answered for but will not serve, and objects whose own batch request
+// the endpoint rejected outright — and whether it answered for any of them, which
+// is what separates a rejection beside real answers from one that stands alone.
+// skipped, when set, names the first object that could not be fetched.
+type chunkOutcome struct {
+ unavailable int
+ rejected int
+ answered bool
+ skipped error
+}
- config, err := readLFSConfig(repository)
- if err != nil || config == nil {
- return endpoint, nil
+// errNarrowedToSingleton marks the rejection of a single object's own batch
+// request. Callers compare it with a sibling half's outcome to decide whether
+// the endpoint refuses the request shape itself or merely that one object.
+var errNarrowedToSingleton = errors.New("batch request rejected down to a single object")
+
+// errAllRejected marks a chunk whose every request was refused while the
+// endpoint answered none of its objects, which is as close as a client can get to
+// an endpoint-wide rejection. It is reported and, once a second chunk repeats it,
+// treated as the endpoint being broken rather than as a chunk of stale pointers.
+var errAllRejected = errors.New("every request for this batch was rejected")
+
+// fetchChunk submits one batch request for pointers and downloads what it
+// schedules. A rejected batch is split in half and resubmitted, down to single
+// pointers, so the rejection is narrowed to the object that caused it: a request
+// the endpoint considers too large, or one poisonous object, then costs that
+// object alone instead of the chunk's whole LFS content.
+func (f *Fetcher) fetchChunk(
+ ctx context.Context,
+ store, endpoint string,
+ creds credentials,
+ pointers []pointer,
+) (chunkOutcome, error) {
+ objects, err := f.client.batch(ctx, creds, endpoint, pointers)
+ if err == nil {
+ return f.downloadChunk(ctx, store, endpoint, creds, pointers, objects)
}
- if override := strings.TrimSpace(config.Raw.Section("lfs").Option("url")); override != "" {
- overridden, ok := paths.ParseHTTPURL(override)
- if !ok {
- return "", fmt.Errorf("unsupported lfs.url '%s': only absolute http and https URLs are allowed", redactedURL(override))
+ if !errors.Is(err, errBatchRejected) {
+ return chunkOutcome{}, err
+ }
+ // Descending calls answer with their own errors, so the rejection that
+ // brought us here is kept for the reports below.
+ rejection := err
+
+ // A rejected chunk wider than one pointer is retried as two batches, and
+ // each half narrows further on its own. Both halves are always tried, so a
+ // failure in one cannot cost the other its objects, and the pair is what
+ // makes the chunk's outcome complete whichever half went wrong.
+ if len(pointers) > 1 {
+ middle := len(pointers) / 2
+ outcome, firstErr := f.fetchChunk(ctx, store, endpoint, creds, pointers[:middle])
+ if firstErr != nil && !isRefusal(firstErr) {
+ // A half that failed outright still must not cost its sibling.
+ rest, _ := f.fetchChunk(ctx, store, endpoint, creds, pointers[middle:])
+ outcome.absorb(rest)
+ return outcome, firstErr
}
- if !strings.EqualFold(overridden.Scheme, parsed.Scheme) || canonicalHost(overridden) != canonicalHost(parsed) {
- return "", fmt.Errorf("refusing lfs.url '%s': the override must stay on the remote host and scheme so the remote credential is not sent elsewhere", redactedURL(override))
+
+ rest, restErr := f.fetchChunk(ctx, store, endpoint, creds, pointers[middle:])
+ outcome.absorb(rest)
+ if restErr != nil && !isRefusal(restErr) {
+ return outcome, restErr
+ }
+ if firstErr == nil && restErr == nil {
+ return outcome, nil
}
- // Drop a redundant scheme-default port so the endpoint is canonical:
- // object-download host comparisons and logs then match hrefs rendered
- // without the explicit port.
- if overridden.Port() != "" && isSchemeDefaultPort(overridden) {
- overridden.Host = strings.TrimSuffix(overridden.Host, ":"+overridden.Port())
+ // A rejection reached a single pointer in one of the halves, and the
+ // whole chunk is now accounted for, so the rejection can be attributed.
+ return refuseOrReport(outcome, rejection)
+ }
+
+ // The endpoint refuses this single object's own request, so nothing narrower
+ // can be blamed. Whether that is this object's fault or the endpoint's is
+ // decided one level up, by whether the sibling half was served; this subtree
+ // has served nothing either way.
+ //
+ // The object's own report renders the rejection rather than wrapping it: an
+ // error matching both sentinels would read as the endpoint's refusal and as
+ // one object's unavailability at once, and callers weigh those differently.
+ slog.Debug("Git LFS batch request rejected for a single object.",
+ "endpoint", redactedURL(endpoint), "oid", shortOID(pointers[0].oid), "reason", rejection.Error())
+ return chunkOutcome{
+ rejected: 1,
+ skipped: fmt.Errorf("%w: LFS object %s download request rejected: %v",
+ errObjectUnavailable, shortOID(pointers[0].oid), rejection),
+ }, fmt.Errorf("%w: %w", errNarrowedToSingleton, rejection)
+}
+
+// isRefusal reports whether an error means the endpoint refused a request
+// rather than failing to serve it.
+func isRefusal(err error) bool {
+ return errors.Is(err, errNarrowedToSingleton) || errors.Is(err, errAllRejected)
+}
+
+// refuseOrReport settles a chunk whose rejection reached a single pointer. An
+// object the endpoint answered for settles it: whether that answer served bytes
+// or reported the object missing, the endpoint processed the batch, so a refused
+// sibling object is that object's fault rather than the endpoint's. A chunk the
+// endpoint answered nothing for is the alternative — allRejected wrapping
+// errBatchRejected — which callers report and stop the sweep after, since only a
+// second such chunk tells stale pointers apart from a broken endpoint.
+func refuseOrReport(outcome chunkOutcome, rejection error) (chunkOutcome, error) {
+ if outcome.answered || outcome.unavailable > 0 {
+ return outcome, nil
+ }
+ return outcome, fmt.Errorf("%w: %w", errAllRejected, rejection)
+}
+
+// downloadChunk streams every object the batch scheduled. Objects the endpoint
+// will not serve are recorded and skipped so their siblings still download, and
+// so are objects this client could not transfer — attempted, but unreachable —
+// which come back as an error once every object has had its turn. The chunk
+// counts as answered when the endpoint replied with content for any scheduled
+// object, which is what tells a rejection beside real answers from one that
+// stands alone.
+//
+// Objects whose scheduled URL had already lapsed are asked about once more: the
+// server is the only party that can issue a fresh URL, and a lapsed one would
+// otherwise fail a transfer that was ready to succeed.
+func (f *Fetcher) downloadChunk(
+ ctx context.Context,
+ store, endpoint string,
+ creds credentials,
+ want []pointer,
+ objects []batchResponseObject,
+) (chunkOutcome, error) {
+ expired, unavailable, failed := downloadObjects(ctx, f.client, store, endpoint, creds, want, objects)
+
+ var outcome chunkOutcome
+ if len(expired) > 0 {
+ rescheduled, err := f.client.batch(ctx, creds, endpoint, expired)
+ if err != nil {
+ // The endpoint scheduled these objects and then would not schedule
+ // them again, so each one is recorded as unserved with its own name
+ // and the reason the fresh URL never arrived. That record is the
+ // failure: reporting the request's error again would turn objects
+ // that were accounted for into an unaccounted chunk failure.
+ for _, object := range expired {
+ outcome.recordUnavailable(fmt.Errorf("%w: no fresh download URL was issued for %s: %w", errObjectUnavailable, object.oid, err))
+ }
+ } else {
+ againExpired, againUnavailable, againFailed := downloadObjects(ctx, f.client, store, endpoint, creds, expired, rescheduled)
+ for _, object := range againExpired {
+ // A server issuing an already-lapsed URL twice has nothing more
+ // to offer for these objects, and each is named so a caller can
+ // tell which content the mirror is missing.
+ outcome.recordUnavailable(fmt.Errorf("%w: the download URL for %s lapsed before it could be used", errObjectUnavailable, object.oid))
+ }
+ unavailable = append(unavailable, againUnavailable...)
+ failed = append(failed, againFailed...)
}
- return strings.TrimSuffix(overridden.String(), "/"), nil
}
- return endpoint, nil
+
+ for _, object := range unavailable {
+ outcome.recordUnavailable(fmt.Errorf("%w: %s", errObjectUnavailable, object.Error()))
+ }
+ // The endpoint answered for every object that is not unavailable, whether it
+ // streamed the bytes now or found them already in the store. A chunk it
+ // answered for is not a chunk it refused, so a rejection beside those answers
+ // is one object's fault rather than the endpoint's.
+ outcome.answered = len(objects) > len(unavailable)
+ if len(failed) > 0 {
+ return outcome, fmt.Errorf("%d of %d LFS objects could not be downloaded: %w", len(failed), len(objects), failed[0])
+ }
+ return outcome, nil
}
-// isSchemeDefaultPort reports whether the URL's explicit port equals its
-// scheme's default (http 80, https 443).
-func isSchemeDefaultPort(u *url.URL) bool {
- return (strings.EqualFold(u.Scheme, "http") && u.Port() == "80") ||
- (strings.EqualFold(u.Scheme, "https") && u.Port() == "443")
+// recordUnavailable counts one object the endpoint answered for but will not
+// serve, keeping the first reason for the caller's report.
+func (o *chunkOutcome) recordUnavailable(reason error) {
+ o.unavailable++
+ if o.skipped == nil {
+ o.skipped = reason
+ }
}
-// canonicalHost renders the URL's host lowercased with any scheme-default
-// port removed, so an explicit https://host:443 compares equal to
-// https://host while a redirect to any other port stays distinct.
-func canonicalHost(u *url.URL) string {
- host := strings.ToLower(u.Hostname())
- port := u.Port()
- if port == "" || isSchemeDefaultPort(u) {
- return host
+// absorb adds a split's second half to its first.
+func (o *chunkOutcome) absorb(half chunkOutcome) {
+ o.unavailable += half.unavailable
+ o.rejected += half.rejected
+ o.answered = o.answered || half.answered
+ if half.skipped != nil && o.skipped == nil {
+ o.skipped = half.skipped
}
- return host + ":" + port
}
// redactedURL renders a URL with any embedded password masked, for safe
diff --git a/internal/lfs/lfs_test.go b/internal/lfs/lfs_test.go
index 60bfd48..66fee61 100644
--- a/internal/lfs/lfs_test.go
+++ b/internal/lfs/lfs_test.go
@@ -7,20 +7,31 @@ import (
"encoding/json"
"errors"
"fmt"
+ "io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
+ "slices"
"strings"
"sync"
+ "sync/atomic"
"testing"
"time"
"github.com/go-git/go-git/v5"
+ "github.com/go-git/go-git/v5/plumbing"
+ "github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
)
+// roundTripFunc adapts a function to an http.RoundTripper, so a test can script
+// the exact responses the batch client sees.
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { return f(request) }
+
func pointerFor(content []byte) (string, string) {
sum := sha256.Sum256(content)
oid := hex.EncodeToString(sum[:])
@@ -70,15 +81,59 @@ type fakeLFSServer struct {
downloads map[string]int
// batchStatus, when non-zero, is the status returned for batch requests.
batchStatus int
+ // refusedOIDs are answered with a per-object error instead of a download
+ // action, like a pointer whose object is gone from the server.
+ refusedOIDs []string
+ // actionlessOIDs are answered with neither an error nor a download action,
+ // as a misbehaving endpoint may do.
+ actionlessOIDs []string
+ // rejectWhen, when set, decides the status of each batch request so a test
+ // can model an endpoint that rejects some request shapes.
+ rejectWhen func(batchRequest) int
// corruptDownload serves wrong bytes for every object.
corruptDownload bool
// batchPath overrides the expected batch path (lfs.url override tests).
batchPath string
+ // servePaths, when set, are the only batch paths the server answers, so any
+ // other path answers 404 like a host with no LFS service mounted there.
+ servePaths []string
+ // disabledPaths answer 403, the response for a repository with LFS switched
+ // off, which is not the same as a path with no service behind it.
+ disabledPaths []string
+ // serveProbe answers discovery probes normally, so a test can have discovery
+ // succeed while the batch requests that follow are rejected.
+ serveProbe bool
+ // failedPaths answer a status for one batch path only, so a test can model a
+ // candidate whose endpoint is broken while another answers.
+ failedPaths map[string]int
+ // requireAuth answers 401 to any request without an Authorization header,
+ // which is how a forge that wants credentials for the objects behaves.
+ requireAuth bool
+ // forbidAnonymous answers 403 to any request without an Authorization
+ // header, which is how a forge behaves when it refuses anonymous access
+ // outright rather than inviting a credential with 401. A 403 is also the
+ // status for LFS switched off, so a client that discovers the endpoint
+ // anonymously cannot tell the two apart.
+ forbidAnonymous bool
+ // lapseFirstAction makes the first schedule for each object point at an
+ // already-expired URL, so the client has to ask for a fresh one.
+ lapseFirstAction bool
+ // lapseEveryAction makes every schedule for each object point at an
+ // already-expired URL, so asking again cannot produce a usable one.
+ lapseEveryAction bool
+ // scheduled counts how many times each object has been scheduled.
+ scheduled map[string]int
+ // fetchStatus, when non-zero, is the status batch requests get once a probe
+ // has been answered, so a test can model an endpoint that serves discovery
+ // and then stops serving the fetch.
+ fetchStatus int
+ // probeCalls counts the discovery probes the handler answered.
+ probeCalls int
}
func newFakeLFSServer(t *testing.T, data map[string][]byte) *fakeLFSServer {
t.Helper()
- s := &fakeLFSServer{data: data, downloads: make(map[string]int)}
+ s := &fakeLFSServer{data: data, downloads: make(map[string]int), scheduled: make(map[string]int)}
mux := http.NewServeMux()
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
switch {
@@ -95,23 +150,119 @@ func newFakeLFSServer(t *testing.T, data map[string][]byte) *fakeLFSServer {
return s
}
+// writeUnmounted answers the way a forge's edge does for a repository path it
+// does not route to an LFS API: a page, which is what tells a client the path is
+// wrong rather than the repository LFS-free.
+func writeUnmounted(w http.ResponseWriter) {
+ w.Header().Set("Content-Type", "text/html; charset=utf-8")
+ w.WriteHeader(http.StatusUnprocessableEntity)
+ _, _ = w.Write([]byte("\n
Oh nounexpected"))
+}
+
// remoteURL is the remote a repository would use to reach the fake server.
func (s *fakeLFSServer) remoteURL() string {
return s.server.URL + "/repo.git"
}
+// plainRemoteURL is the same repository addressed without its .git suffix, the
+// form a configuration normally carries.
+func (s *fakeLFSServer) plainRemoteURL() string {
+ return s.server.URL + "/repo"
+}
+
+// endpointFor mounts the server's batch API at the path a remote addresses, so
+// a test chooses where the LFS service answers, and makes every other path
+// answer 404 like a host with nothing mounted there.
+func (s *fakeLFSServer) endpointFor(remoteURL string) string {
+ s.serve([]string{remoteURL})
+ return remoteURL
+}
+
+// disabledFor makes the given remotes answer 403 — the response a forge uses for
+// a repository with LFS switched off — while the rest answer 404.
+func (s *fakeLFSServer) disabledFor(remoteURLs ...string) {
+ disabled := make([]string, 0, len(remoteURLs))
+ for _, remoteURL := range remoteURLs {
+ disabled = append(disabled, batchPathFor(remoteURL))
+ }
+ s.mu.Lock()
+ s.disabledPaths = disabled
+ s.mu.Unlock()
+}
+
+// serve mounts the batch API at the paths the given remotes address.
+func (s *fakeLFSServer) serve(remoteURLs []string) {
+ served := make([]string, 0, len(remoteURLs))
+ for _, remoteURL := range remoteURLs {
+ served = append(served, batchPathFor(remoteURL))
+ }
+ s.mu.Lock()
+ s.servePaths = served
+ s.mu.Unlock()
+}
+
+// batchPathFor is the batch action path a remote addresses.
+func batchPathFor(remoteURL string) string {
+ parsed, err := url.Parse(remoteURL)
+ if err != nil {
+ return remoteURL
+ }
+ return parsed.Path + "/info/lfs/objects/batch"
+}
+
func (s *fakeLFSServer) handleBatch(w http.ResponseWriter, r *http.Request) {
s.mu.Lock()
s.batchCalls++
status, batchPath := s.batchStatus, s.batchPath
+ rejectWhen, serveProbe := s.rejectWhen, s.serveProbe
+ probed := s.probeCalls > 0
+ fetchStatus := s.fetchStatus
+ failedPaths := s.failedPaths
+ requireAuth := s.requireAuth
+ forbidAnonymous := s.forbidAnonymous
+ disabled := slices.Contains(s.disabledPaths, r.URL.Path)
+ servedHere := slices.Contains(s.servePaths, r.URL.Path)
+ hasServePaths := len(s.servePaths) > 0
s.mu.Unlock()
+ // A path the server does not serve answers the way the forges this package
+ // was written against do: a host page for the path it will not route, and
+ // markup rather than an LFS explanation. The disabled verdict is decided
+ // first, because a repository with LFS off answers 403 for its own path even
+ // when the test also mounts the service elsewhere.
if batchPath != "" && r.URL.Path != batchPath {
- w.WriteHeader(http.StatusNotFound)
+ writeUnmounted(w)
return
}
- if status != 0 {
- w.WriteHeader(status)
+ if requireAuth && r.Header.Get("Authorization") == "" {
+ // The API wants credentials it was not given, which the client is
+ // expected to retry with.
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+ if forbidAnonymous && r.Header.Get("Authorization") == "" {
+ // An anonymous request is refused outright, and the refusal is shaped
+ // exactly like a repository with LFS switched off.
+ w.Header().Set("Content-Type", lfsMediaType)
+ w.WriteHeader(http.StatusForbidden)
+ _, _ = w.Write([]byte(`{"message":"Git LFS is disabled for this repository."}`))
+ return
+ }
+ if disabled {
+ // A forge with LFS switched off explains itself in JSON.
+ w.Header().Set("Content-Type", lfsMediaType)
+ w.WriteHeader(http.StatusForbidden)
+ _, _ = w.Write([]byte(`{"message":"Git LFS is disabled for this repository."}`))
+ return
+ }
+ if failedStatus, fails := failedPaths[r.URL.Path]; fails {
+ // One candidate path fails while another answers, so a test can model a
+ // wrong guess that is not merely unmounted.
+ w.WriteHeader(failedStatus)
+ return
+ }
+ if hasServePaths && !servedHere {
+ writeUnmounted(w)
return
}
@@ -121,19 +272,92 @@ func (s *fakeLFSServer) handleBatch(w http.ResponseWriter, r *http.Request) {
return
}
+ // A discovery probe carries one object. A server configured to answer it
+ // does so before any rule about the fetch applies, so a test can model
+ // discovery succeeding and the fetch that follows failing.
+ if len(request.Objects) == 1 && serveProbe && !probed {
+ s.mu.Lock()
+ s.probeCalls++
+ s.mu.Unlock()
+ s.respond(w, request)
+ return
+ }
+ if status != 0 {
+ w.WriteHeader(status)
+ return
+ }
+ // fetchStatus, when set, is what the fetch gets once discovery has been
+ // answered, so a test can model an endpoint that serves discovery and then
+ // stops serving the fetch.
+ if fetchStatus != 0 && probed {
+ w.WriteHeader(fetchStatus)
+ return
+ }
+
+ // Mimic a forge that rejects some batch requests, such as one carrying more
+ // objects than its limit allows.
+ if rejectWhen != nil {
+ if code := rejectWhen(request); code != http.StatusOK {
+ w.WriteHeader(code)
+ return
+ }
+ }
+ s.respond(w, request)
+}
+
+// respond answers a batch request with the object verdicts the server holds.
+func (s *fakeLFSServer) respond(w http.ResponseWriter, request batchRequest) {
+ s.mu.Lock()
+ refused := make(map[string]struct{}, len(s.refusedOIDs))
+ for _, oid := range s.refusedOIDs {
+ refused[oid] = struct{}{}
+ }
+ actionless := make(map[string]struct{}, len(s.actionlessOIDs))
+ for _, oid := range s.actionlessOIDs {
+ actionless[oid] = struct{}{}
+ }
+ s.mu.Unlock()
+
response := batchResponse{Objects: make([]batchResponseObject, 0, len(request.Objects))}
for _, object := range request.Objects {
- if _, known := s.data[object.OID]; !known {
+ if _, isRefused := refused[object.OID]; isRefused {
+ response.Objects = append(response.Objects, batchResponseObject{
+ OID: object.OID,
+ Error: &batchError{Code: http.StatusUnprocessableEntity, Message: "refused"},
+ })
+ continue
+ }
+ content, known := s.data[object.OID]
+ if !known {
response.Objects = append(response.Objects, batchResponseObject{
OID: object.OID,
Error: &batchError{Code: http.StatusNotFound, Message: "Object does not exist"},
})
continue
}
+ if _, bare := actionless[object.OID]; bare {
+ // The server knows the object but schedules nothing for it: a
+ // successful response the client cannot act on.
+ response.Objects = append(response.Objects, batchResponseObject{
+ OID: object.OID,
+ Size: int64(len(content)),
+ })
+ continue
+ }
+ action := &batchAction{Href: s.server.URL + "/download/" + object.OID}
+ s.mu.Lock()
+ s.scheduled[object.OID]++
+ lapse := s.lapseEveryAction || (s.lapseFirstAction && s.scheduled[object.OID] == 1)
+ s.mu.Unlock()
+ if lapse {
+ // A URL that has already lapsed, as a server issuing a short-lived
+ // one before a slow scan would produce.
+ action.ExpiresAt = time.Now().Add(-time.Minute).UTC().Format(time.RFC3339)
+ }
response.Objects = append(response.Objects, batchResponseObject{
OID: object.OID,
- Size: int64(len(s.data[object.OID])),
- Actions: map[string]*batchAction{"download": {Href: s.server.URL + "/download/" + object.OID}},
+ Size: int64(len(content)),
+ Actions: map[string]*batchAction{"download": action},
})
}
w.Header().Set("Content-Type", lfsMediaType)
@@ -256,16 +480,541 @@ func TestFetchAllSkipsEndpointWhenNoPointers(t *testing.T) {
func TestFetchAllDisabledRemote(t *testing.T) {
oid, pointerText := pointerFor([]byte("content"))
- for _, status := range []int{http.StatusForbidden, http.StatusNotFound} {
- lfsServer := newFakeLFSServer(t, map[string][]byte{oid: []byte("content")})
- lfsServer.batchStatus = status
- repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText})
-
- err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "")
- if !errors.Is(err, ErrDisabled) {
- t.Errorf("status %d should map to ErrDisabled, got %v", status, err)
+ lfsServer := newFakeLFSServer(t, map[string][]byte{oid: []byte("content")})
+ // Both candidate paths answer as a repository with LFS switched off.
+ lfsServer.disabledFor(lfsServer.plainRemoteURL(), lfsServer.remoteURL())
+ repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText})
+
+ err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.plainRemoteURL(), "", "")
+ if !errors.Is(err, ErrDisabled) {
+ t.Errorf("a 403 from every candidate should map to ErrDisabled, got %v", err)
+ }
+}
+
+// TestFetchAllFallsBackToTheConfiguredPath covers a host that mounts its LFS
+// service exactly where the remote points rather than under the repository's
+// .git path: the suffixed endpoint answers 404, and the suffix-less one must
+// then be tried rather than the remote being written off as LFS-free.
+func TestFetchAllFallsBackToTheConfiguredPath(t *testing.T) {
+ content := []byte("content")
+ oid, pointerText := pointerFor(content)
+
+ lfsServer := newFakeLFSServer(t, map[string][]byte{oid: content})
+ remoteURL := lfsServer.endpointFor(lfsServer.plainRemoteURL())
+ repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText})
+
+ if err := NewFetcher().FetchAll(context.Background(), repositoryPath, remoteURL, "", ""); err != nil {
+ t.Fatalf("FetchAll failed: %v", err)
+ }
+ cached, err := os.ReadFile(filepath.Join(repositoryPath, ".git", "lfs", "objects",
+ oid[0:2], oid[2:4], oid))
+ if err != nil {
+ t.Fatalf("the object should be mirrored through the fallback endpoint: %v", err)
+ }
+ if string(cached) != string(content) {
+ t.Error("mirrored content should match the served bytes")
+ }
+}
+
+// TestFetchAllFallsBackWhenTheGuessIsForbidden covers the guessed .git path
+// answering 403 while the configured path serves normally. The 403 is about a
+// path that does not exist rather than about the repository's LFS setting, so it
+// must not be believed before the other candidate has been tried.
+func TestFetchAllFallsBackWhenTheGuessIsForbidden(t *testing.T) {
+ content := []byte("content")
+ oid, pointerText := pointerFor(content)
+
+ lfsServer := newFakeLFSServer(t, map[string][]byte{oid: content})
+ remoteURL := lfsServer.endpointFor(lfsServer.plainRemoteURL())
+ lfsServer.disabledFor(lfsServer.remoteURL())
+ repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText})
+
+ if err := NewFetcher().FetchAll(context.Background(), repositoryPath, remoteURL, "", ""); err != nil {
+ t.Fatalf("FetchAll failed: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects",
+ oid[0:2], oid[2:4], oid)); err != nil {
+ t.Errorf("the configured path should still be tried after a 403 on the guess: %v", err)
+ }
+}
+
+// TestFetchAllNoEndpointAnywhereIsReported covers a host with no LFS service at
+// either candidate path, which the mirror layer records as LFS being switched
+// off — the two are indistinguishable from here, and both are expected states
+// rather than a failed repository.
+func TestFetchAllNoEndpointAnywhereIsReported(t *testing.T) {
+ oid, pointerText := pointerFor([]byte("content"))
+
+ lfsServer := newFakeLFSServer(t, map[string][]byte{oid: []byte("content")})
+ lfsServer.endpointFor(lfsServer.server.URL + "/somewhere/else")
+ repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText})
+
+ err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.plainRemoteURL(), "", "")
+ if errors.Is(err, ErrNoEndpoint) {
+ t.Fatalf("err = %v, want the expected-skip verdict rather than an internal signal", err)
+ }
+ if !errors.Is(err, ErrDisabled) {
+ t.Fatalf("err = %v, want ErrDisabled", err)
+ }
+}
+
+// TestFetchAllNotServedAnywhereIsSkipped covers a host whose every candidate
+// path answers 404, which is both how a repository with LFS switched off looks
+// on a forge that does not distinguish the two and how a host with no LFS
+// service at all looks. Either way the mirror layer's expected skip is the right
+// outcome, not a repository-wide failure.
+func TestFetchAllNotServedAnywhereIsSkipped(t *testing.T) {
+ oid, pointerText := pointerFor([]byte("content"))
+
+ lfsServer := newFakeLFSServer(t, map[string][]byte{oid: []byte("content")})
+ lfsServer.endpointFor(lfsServer.server.URL + "/somewhere/else")
+ repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText})
+
+ err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.plainRemoteURL(), "", "")
+ if !errors.Is(err, ErrDisabled) {
+ t.Fatalf("err = %v, want ErrDisabled when no candidate serves the API", err)
+ }
+}
+
+// TestFetchAllConfiguredPathDecidesOverTheGuess covers the rule that only the
+// path the remote itself configures decides: a failure there is reported even
+// when a derived guess answers that LFS is off, because the mirror layer records
+// that verdict as a successful skip and would hide a repository whose LFS content
+// is missing.
+func TestFetchAllConfiguredPathDecidesOverTheGuess(t *testing.T) {
+ oid, pointerText := pointerFor([]byte("content"))
+
+ lfsServer := newFakeLFSServer(t, map[string][]byte{oid: []byte("content")})
+ // The remote is suffix-less, so the suffix-less path is configured and the
+ // suffixed guess is the one that reports LFS off.
+ lfsServer.disabledFor(lfsServer.remoteURL())
+ lfsServer.failedPaths = map[string]int{
+ batchPathFor(lfsServer.plainRemoteURL()): http.StatusInternalServerError,
+ }
+ repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText})
+
+ err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.plainRemoteURL(), "", "")
+ if err == nil || errors.Is(err, ErrDisabled) {
+ t.Fatalf("err = %v, want the configured path's failure reported rather than a skip", err)
+ }
+}
+
+// TestFetchAllGuessFailureDoesNotOverruleTheConfiguredPath covers the other
+// direction: the configured path reports LFS off while the derived guess fails,
+// so the configured path's verdict is what the repository is recorded as.
+func TestFetchAllGuessFailureDoesNotOverruleTheConfiguredPath(t *testing.T) {
+ oid, pointerText := pointerFor([]byte("content"))
+
+ lfsServer := newFakeLFSServer(t, map[string][]byte{oid: []byte("content")})
+ // The remote is suffix-less, so the suffix-less path is the configured one,
+ // and the suffixed guess is the one that fails.
+ lfsServer.disabledFor(lfsServer.plainRemoteURL())
+ lfsServer.failedPaths = map[string]int{
+ batchPathFor(lfsServer.remoteURL()): http.StatusInternalServerError,
+ }
+ repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText})
+
+ err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.plainRemoteURL(), "", "")
+ if !errors.Is(err, ErrDisabled) {
+ t.Fatalf("err = %v, want the configured path's disabled verdict to be the expected skip", err)
+ }
+}
+
+// TestFetchAllOffersCredentialsOnTheFirstRequest covers a forge that serves
+// nothing to an anonymous request. The client presents its credentials straight
+// away rather than waiting to be asked, so such a forge is read as one that
+// wanted the credential rather than as a repository with LFS switched off.
+func TestFetchAllOffersCredentialsOnTheFirstRequest(t *testing.T) {
+ content := []byte("content")
+ oid, pointerText := pointerFor(content)
+
+ lfsServer := newFakeLFSServer(t, map[string][]byte{oid: content})
+ lfsServer.requireAuth = true
+ repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText})
+
+ if err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "user", "token"); err != nil {
+ t.Fatalf("FetchAll failed: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects",
+ oid[0:2], oid[2:4], oid)); err != nil {
+ t.Errorf("the object should be mirrored after the authenticated request: %v", err)
+ }
+
+ // Without credentials there is nothing to present, so the 401 stands.
+ unauthenticated := newRepoWithLFS(t, map[string]string{"file.bin": pointerText})
+ err := NewFetcher().FetchAll(context.Background(), unauthenticated, lfsServer.remoteURL(), "", "")
+ if err == nil {
+ t.Fatal("a 401 answered without credentials should be reported")
+ }
+ if errors.Is(err, ErrDisabled) {
+ t.Errorf("err = %v, want a failure rather than a repository recorded as LFS-free", err)
+ }
+}
+
+// TestFetchAllReadsAForbiddenAnonymousRequestAsCredentialsNeeded covers the case
+// a 401-only client gets wrong: a forge that refuses anonymous requests with 403
+// instead of inviting a credential with 401. A 403 is also how a repository with
+// LFS switched off answers, so the credentials have to be offered on the first
+// request — discovering the endpoint anonymously would record the repository as
+// LFS-free and back it up without its LFS content.
+func TestFetchAllReadsAForbiddenAnonymousRequestAsCredentialsNeeded(t *testing.T) {
+ content := []byte("content")
+ oid, pointerText := pointerFor(content)
+
+ lfsServer := newFakeLFSServer(t, map[string][]byte{oid: content})
+ lfsServer.forbidAnonymous = true
+ repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText})
+
+ if err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "user", "token"); err != nil {
+ t.Fatalf("FetchAll failed: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects",
+ oid[0:2], oid[2:4], oid)); err != nil {
+ t.Errorf("the object should be mirrored rather than the repository recorded as LFS-free: %v", err)
+ }
+
+ // The same server without credentials says nothing about the credential, so
+ // its refusal is read the only way it can be: LFS is not available here.
+ unauthenticated := newRepoWithLFS(t, map[string]string{"file.bin": pointerText})
+ err := NewFetcher().FetchAll(context.Background(), unauthenticated, lfsServer.remoteURL(), "", "")
+ if !errors.Is(err, ErrDisabled) {
+ t.Errorf("err = %v, want a repository recorded as LFS-free when the forge refuses anonymously", err)
+ }
+}
+
+// TestFetchAllAsksAgainWhenTheDownloadURLLapsed covers a server that schedules a
+// URL which has already expired: the object needs a fresh one rather than a
+// transfer that can only fail, and the server is the only party that can issue it.
+func TestFetchAllAsksAgainWhenTheDownloadURLLapsed(t *testing.T) {
+ content := []byte("content")
+ oid, pointerText := pointerFor(content)
+
+ lfsServer := newFakeLFSServer(t, map[string][]byte{oid: content})
+ lfsServer.lapseFirstAction = true
+ repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText})
+
+ if err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", ""); err != nil {
+ t.Fatalf("FetchAll failed: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects",
+ oid[0:2], oid[2:4], oid)); err != nil {
+ t.Errorf("the object should be mirrored after the URL was refreshed: %v", err)
+ }
+
+ lfsServer.mu.Lock()
+ schedules := lfsServer.scheduled[oid]
+ lfsServer.mu.Unlock()
+ if schedules != 2 {
+ t.Errorf("object was scheduled %d times, want it asked about again after the lapsed URL", schedules)
+ }
+ if downloads := lfsServer.downloadCount(oid); downloads != 1 {
+ t.Errorf("download count = %d, want the object streamed exactly once", downloads)
+ }
+}
+
+// TestDownloadObjectsSkipsCachedObjectsWhoseURLLapsed covers content that is
+// already in the store being scheduled with an expired URL. The object needs
+// nothing from the endpoint, so it is settled from the cache before the expiry
+// can send it back for rescheduling, which is what made an already-mirrored
+// repository fail over a fresh URL it never needed.
+func TestDownloadObjectsSkipsCachedObjectsWhoseURLLapsed(t *testing.T) {
+ content := []byte("content")
+ oid, _ := pointerFor(content)
+
+ store := t.TempDir()
+ cached := filepath.Join(store, oid[0:2], oid[2:4])
+ if err := os.MkdirAll(cached, 0o755); err != nil {
+ t.Fatalf("prepare the store: %v", err)
+ }
+ if err := os.WriteFile(filepath.Join(cached, oid), content, 0o644); err != nil {
+ t.Fatalf("place the cached object: %v", err)
+ }
+
+ expired, unavailable, failed := downloadObjects(context.Background(), newBatchClient(nil), store,
+ "http://127.0.0.1:1/repo.git/info/lfs", credentials{},
+ []pointer{{oid: oid, size: int64(len(content))}},
+ []batchResponseObject{{
+ OID: oid,
+ Size: int64(len(content)),
+ Actions: map[string]*batchAction{"download": {
+ Href: "http://127.0.0.1:1/download/" + oid,
+ ExpiresAt: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
+ }},
+ }})
+
+ if len(expired) != 0 {
+ t.Errorf("expired = %v, want cached content not sent back for a fresh URL", expired)
+ }
+ if len(unavailable) != 0 || len(failed) != 0 {
+ t.Errorf("unavailable = %v, failed = %v, want cached content left alone", unavailable, failed)
+ }
+}
+
+// TestFetchAllNamesObjectsWhoseRefreshFailed covers an endpoint that only ever
+// schedules lapsed URLs and then refuses the request for fresh ones. Each object
+// has to be reported by name — a count tells a caller nothing about what is
+// missing from the mirror — and the refusal is that record's reason rather than
+// a second, chunk-level failure over objects already accounted for.
+func TestFetchAllNamesObjectsWhoseRefreshFailed(t *testing.T) {
+ content := []byte("content")
+ oid, pointerText := pointerFor(content)
+
+ lfsServer := newFakeLFSServer(t, map[string][]byte{oid: content})
+ lfsServer.lapseEveryAction = true
+ repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText})
+
+ err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "")
+ if err == nil {
+ t.Fatal("an object that never arrived should be reported")
+ }
+ if !strings.Contains(err.Error(), oid) {
+ t.Errorf("err = %v, want it to name the object that could not be fetched", err)
+ }
+ if !errors.Is(err, errObjectUnavailable) {
+ t.Errorf("err = %v, want the object recorded as unserved", err)
+ }
+ if _, statErr := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects",
+ oid[0:2], oid[2:4], oid)); statErr == nil {
+ t.Error("the object should not be in the store, since it was never downloaded")
+ }
+}
+
+// TestDownloadChunkReportsARefusedRefreshPerObject covers the endpoint scheduling
+// a lapsed URL and then refusing outright to issue a fresh one. The refusal is
+// the reason the object went unserved, so it is reported through that object —
+// named — rather than as a further chunk-level failure over objects that were
+// already accounted for.
+func TestDownloadChunkReportsARefusedRefreshPerObject(t *testing.T) {
+ const endpoint = "http://127.0.0.1:1/repo.git/info/lfs"
+ oid, _ := pointerFor([]byte("content"))
+
+ // downloadObjects settles the chunk's own objects from the response it is
+ // handed, so the first request this transport sees is the one asking for a
+ // fresh URL; that one is refused.
+ batches := 0
+ client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+ batches++
+ return &http.Response{
+ StatusCode: http.StatusInternalServerError,
+ Body: io.NopCloser(strings.NewReader("")),
+ Header: make(http.Header),
+ }, nil
+ })}
+
+ fetcher := &Fetcher{client: newBatchClient(client)}
+ outcome, err := fetcher.downloadChunk(context.Background(), t.TempDir(), endpoint, credentials{},
+ []pointer{{oid: oid, size: 7}},
+ []batchResponseObject{
+ {OID: oid, Size: 7, Actions: map[string]*batchAction{"download": {
+ Href: endpoint + "/download/" + oid,
+ ExpiresAt: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
+ }}},
+ })
+
+ if batches != 1 {
+ t.Errorf("batch requests = %d, want one asking for a fresh URL", batches)
+ }
+ if err != nil {
+ t.Errorf("downloadChunk err = %v, want the refusal carried by the object rather than the chunk", err)
+ }
+ if outcome.unavailable != 1 {
+ t.Errorf("unavailable = %d, want the object recorded as unserved", outcome.unavailable)
+ }
+ if outcome.skipped == nil {
+ t.Fatal("the object's reason should be reported")
+ }
+ if !errors.Is(outcome.skipped, errObjectUnavailable) {
+ t.Errorf("reason = %v, want the object recorded as unserved", outcome.skipped)
+ }
+ if !strings.Contains(outcome.skipped.Error(), oid) {
+ t.Errorf("reason = %v, want it to name the object", outcome.skipped)
+ }
+ if !strings.Contains(outcome.skipped.Error(), fmt.Sprint(http.StatusInternalServerError)) {
+ t.Errorf("reason = %v, want the endpoint's own status as why no fresh URL arrived", outcome.skipped)
+ }
+}
+
+// TestDownloadObjectsRefusesMalformedAndUnrequestedOIDs covers an endpoint whose
+// response names objects the client never asked for. The OID becomes a path
+// component and is sliced for the shard directories, so a malformed one has to be
+// refused before either happens: a short OID would panic the slice and cost the
+// whole backup, and a crafted one would name a path outside the store.
+func TestDownloadObjectsRefusesMalformedAndUnrequestedOIDs(t *testing.T) {
+ wanted := strings.Repeat("a", 64)
+ unrelated := strings.Repeat("b", 64)
+ downloads := 0
+ client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+ downloads++
+ return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil
+ })}
+
+ expired, unavailable, failed := downloadObjects(context.Background(), newBatchClient(client), t.TempDir(),
+ "http://127.0.0.1:1/repo.git/info/lfs", credentials{},
+ []pointer{{oid: wanted, size: 7}},
+ []batchResponseObject{
+ {OID: "ab", Size: 7, Actions: map[string]*batchAction{"download": {Href: "http://127.0.0.1:1/download/ab"}}},
+ {OID: "sha/../escape", Size: 7, Actions: map[string]*batchAction{"download": {Href: "http://127.0.0.1:1/download/escape"}}},
+ {OID: unrelated, Size: 7, Actions: map[string]*batchAction{"download": {Href: "http://127.0.0.1:1/download/" + unrelated}}},
+ })
+
+ if len(expired) != 0 || len(failed) != 0 {
+ t.Errorf("expired = %v, failed = %v, want the bad objects refused without a transfer", expired, failed)
+ }
+ if downloads != 0 {
+ t.Errorf("downloads = %d, want nothing fetched for objects the client did not ask for", downloads)
+ }
+ if len(unavailable) != 3 {
+ t.Fatalf("unavailable = %d, want every bad object reported rather than acted on", len(unavailable))
+ }
+ for _, object := range unavailable {
+ if !errors.Is(object, errObjectUnavailable) {
+ t.Errorf("object error = %v, want it recorded as unserved", object)
}
}
+ if !errors.Is(unavailable[2], errObjectUnavailable) || !strings.Contains(unavailable[2].Error(), "not requested") {
+ t.Errorf("third error = %v, want the unrequested object reported as such", unavailable[2])
+ }
+}
+
+// TestDownloadObjectsAcceptsAnUppercaseOID covers an endpoint that echoes a valid
+// digest in upper case. The OID is a path component and the store is keyed by it,
+// so the response has to fold to the spelling the store uses: taken literally it
+// would be written to a second directory, miss the cache it should hit, and be
+// rejected as an object that was never requested.
+func TestDownloadObjectsAcceptsAnUppercaseOID(t *testing.T) {
+ content := []byte("content")
+ oid, _ := pointerFor(content)
+
+ store := t.TempDir()
+ cached := filepath.Join(store, oid[0:2], oid[2:4])
+ if err := os.MkdirAll(cached, 0o755); err != nil {
+ t.Fatalf("prepare the store: %v", err)
+ }
+ if err := os.WriteFile(filepath.Join(cached, oid), content, 0o644); err != nil {
+ t.Fatalf("place the cached object: %v", err)
+ }
+
+ downloads := 0
+ client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+ downloads++
+ return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil
+ })}
+
+ expired, unavailable, failed := downloadObjects(context.Background(), newBatchClient(client), store,
+ "http://127.0.0.1:1/repo.git/info/lfs", credentials{},
+ []pointer{{oid: oid, size: int64(len(content))}},
+ []batchResponseObject{{
+ OID: strings.ToUpper(oid),
+ Size: int64(len(content)),
+ Actions: map[string]*batchAction{"download": {Href: "http://127.0.0.1:1/download/" + oid}},
+ }})
+
+ if len(expired) != 0 || len(unavailable) != 0 || len(failed) != 0 {
+ t.Errorf("expired = %v, unavailable = %v, failed = %v, want the uppercase digest accepted", expired, unavailable, failed)
+ }
+ if downloads != 0 {
+ t.Errorf("downloads = %d, want the cached object recognised rather than fetched again", downloads)
+ }
+}
+
+// TestDownloadObjectsTrustsTheCacheWhenTheEndpointReportsNoSize covers an
+// endpoint that omits the object's size. The length is then unstated rather than
+// contradicted, so content the store already holds must not be re-downloaded —
+// with a lapsed URL that would escalate into failing a backup whose LFS content
+// is complete.
+func TestDownloadObjectsTrustsTheCacheWhenTheEndpointReportsNoSize(t *testing.T) {
+ content := []byte("content")
+ oid, _ := pointerFor(content)
+
+ store := t.TempDir()
+ cached := filepath.Join(store, oid[0:2], oid[2:4])
+ if err := os.MkdirAll(cached, 0o755); err != nil {
+ t.Fatalf("prepare the store: %v", err)
+ }
+ if err := os.WriteFile(filepath.Join(cached, oid), content, 0o644); err != nil {
+ t.Fatalf("place the cached object: %v", err)
+ }
+
+ expired, unavailable, failed := downloadObjects(context.Background(), newBatchClient(nil), store,
+ "http://127.0.0.1:1/repo.git/info/lfs", credentials{},
+ []pointer{{oid: oid, size: int64(len(content))}},
+ []batchResponseObject{{
+ OID: oid,
+ Size: 0,
+ Actions: map[string]*batchAction{"download": {
+ Href: "http://127.0.0.1:1/download/" + oid,
+ ExpiresAt: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
+ }},
+ }})
+
+ if len(expired) != 0 {
+ t.Errorf("expired = %v, want cached content not rescheduled over an unstated size", expired)
+ }
+ if len(unavailable) != 0 || len(failed) != 0 {
+ t.Errorf("unavailable = %v, failed = %v, want cached content left alone", unavailable, failed)
+ }
+
+ // A size the endpoint does state has to be honoured, so a partial file is
+ // not mistaken for the object.
+ shortStore := t.TempDir()
+ if err := os.MkdirAll(filepath.Join(shortStore, oid[0:2], oid[2:4]), 0o755); err != nil {
+ t.Fatalf("prepare the store: %v", err)
+ }
+ if err := os.WriteFile(filepath.Join(shortStore, oid[0:2], oid[2:4], oid), content[:3], 0o644); err != nil {
+ t.Fatalf("place the truncated object: %v", err)
+ }
+ _, _, truncatedFailed := downloadObjects(context.Background(), newBatchClient(nil), shortStore,
+ "http://127.0.0.1:1/repo.git/info/lfs", credentials{},
+ []pointer{{oid: oid, size: int64(len(content))}},
+ []batchResponseObject{{
+ OID: oid,
+ Size: int64(len(content)),
+ Actions: map[string]*batchAction{"download": {
+ Href: "http://127.0.0.1:1/download/" + oid,
+ }},
+ }})
+ if len(truncatedFailed) == 0 {
+ t.Error("a cached file the endpoint's size contradicts should fall through to a download")
+ }
+
+ // The pointer file's size is authoritative, so a truncated file is not the
+ // object even when the endpoint states no size at all — the case that would
+ // otherwise let an interrupted copy be counted as mirrored.
+ _, _, unstatedFailed := downloadObjects(context.Background(), newBatchClient(nil), shortStore,
+ "http://127.0.0.1:1/repo.git/info/lfs", credentials{},
+ []pointer{{oid: oid, size: int64(len(content))}},
+ []batchResponseObject{{
+ OID: oid,
+ Size: 0,
+ Actions: map[string]*batchAction{"download": {
+ Href: "http://127.0.0.1:1/download/" + oid,
+ }},
+ }})
+ if len(unstatedFailed) == 0 {
+ t.Error("a cached file the pointer's size contradicts should fall through to a download")
+ }
+
+ // The resolved size has to travel with an object that is rescheduled, so the
+ // refresh request and the pass over its answer keep the pointer's length
+ // instead of falling back to an endpoint that stated none.
+ expiredPointers, _, _ := downloadObjects(context.Background(), newBatchClient(nil), shortStore,
+ "http://127.0.0.1:1/repo.git/info/lfs", credentials{},
+ []pointer{{oid: oid, size: int64(len(content))}},
+ []batchResponseObject{{
+ OID: oid,
+ Size: 0,
+ Actions: map[string]*batchAction{"download": {
+ Href: "http://127.0.0.1:1/download/" + oid,
+ ExpiresAt: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
+ }},
+ }})
+ if len(expiredPointers) != 1 {
+ t.Fatalf("expired = %v, want the lapsed object rescheduled", expiredPointers)
+ }
+ if expiredPointers[0].size != int64(len(content)) {
+ t.Errorf("rescheduled size = %d, want the pointer's %d rather than the endpoint's unstated 0",
+ expiredPointers[0].size, len(content))
+ }
}
func TestFetchAllUnauthorizedIsNotDisabled(t *testing.T) {
@@ -319,21 +1068,723 @@ func TestFetchAllUnknownObjectFails(t *testing.T) {
if err == nil || errors.Is(err, ErrDisabled) {
t.Fatalf("unknown object should be a genuine error, got %v", err)
}
+ if !errors.Is(err, errObjectUnavailable) {
+ t.Fatalf("err = %v, want it to report the object as unavailable", err)
+ }
+ if !strings.Contains(err.Error(), "could not be fetched") {
+ t.Fatalf("err = %v, want it to say how many objects were skipped", err)
+ }
}
-func TestFetchAllRejectsUnsafeLFSConfigOverrides(t *testing.T) {
- oid, pointerText := pointerFor([]byte("content"))
+// TestFetchAllFallsBackWhenBatchRejected covers forges that reject a batch
+// request outright — an object limit, or a request they refuse to process. The
+// pointers must be narrowed down so a single refused object cannot cost the
+// repository's whole LFS mirror, while a refusal that hits every object is
+// reported as the endpoint's failure.
+func TestFetchAllFallsBackWhenBatchRejected(t *testing.T) {
+ available := []byte("available content")
+ availableOID, availablePointer := pointerFor(available)
+ refused := []byte("refused content")
+ refusedOID, refusedPointer := pointerFor(refused)
+ result := func(oids ...string) string { return strings.Join(oids, ",") }
+
cases := []struct {
- name string
- lfsURL string
+ name string
+ request func(batchRequest) int
+ wantErr string
+ wantBatches int
+ wantCached string
}{
- {"off-host override", "https://evil.example.com/lfs"},
- {"non-http scheme", "ssh://git@evil.example.com/repo.git/lfs"},
- {"relative value", "/custom/lfs"},
+ {
+ name: "server object limit",
+ request: func(request batchRequest) int {
+ if len(request.Objects) > 1 {
+ return http.StatusUnprocessableEntity
+ }
+ return http.StatusOK
+ },
+ // Discovery answers the one-object probe, so the fetch runs: the
+ // chunk is refused once and then narrowed into two served singletons.
+ wantBatches: 4,
+ wantCached: result(availableOID, refusedOID),
+ },
+ {
+ name: "one object rejected, one refused",
+ request: func(request batchRequest) int {
+ for _, object := range request.Objects {
+ if object.OID == refusedOID {
+ return http.StatusUnprocessableEntity
+ }
+ }
+ return http.StatusOK
+ },
+ // The chunk splits once and the refused half's lone pointer is
+ // rejected on its own request.
+ wantErr: "unavailable",
+ wantBatches: 4,
+ wantCached: result(availableOID),
+ },
+ {
+ name: "the endpoint refuses every batch",
+ request: func(batchRequest) int {
+ return http.StatusBadRequest
+ },
+ // Discovery is answered, then every fetch request is refused, so the
+ // rejection outlives the split and is reported as the endpoint's.
+ wantErr: "rejected",
+ wantBatches: 4,
+ },
}
+
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
- lfsServer := newFakeLFSServer(t, map[string][]byte{oid: []byte("content")})
+ lfsServer := newFakeLFSServer(t, map[string][]byte{
+ availableOID: available,
+ refusedOID: refused,
+ })
+ // Discovery is answered so the fetch runs; the rejection rules then
+ // decide what the fetch sees.
+ lfsServer.serveProbe = true
+ lfsServer.rejectWhen = c.request
+ repositoryPath := newRepoWithLFS(t, map[string]string{
+ "big.bin": availablePointer,
+ "stale.bin": refusedPointer,
+ })
+
+ err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "")
+ if c.wantErr == "" {
+ if err != nil {
+ t.Fatalf("FetchAll = %v, want success", err)
+ }
+ } else if err == nil || !strings.Contains(err.Error(), c.wantErr) {
+ t.Fatalf("FetchAll = %v, want an error containing %q", err, c.wantErr)
+ }
+
+ if calls := lfsServer.batchCallCount(); calls != c.wantBatches {
+ t.Errorf("batch calls = %d, want %d", calls, c.wantBatches)
+ }
+ for _, oid := range strings.Split(c.wantCached, ",") {
+ if oid == "" {
+ continue
+ }
+ path := filepath.Join(repositoryPath, ".git", "lfs", "objects", oid[0:2], oid[2:4], oid)
+ if _, err := os.Stat(path); err != nil {
+ t.Errorf("object %s should be mirrored: %v", shortOID(oid), err)
+ }
+ }
+ })
+ }
+}
+
+// TestFetchAllSkipsObjectsWithoutDownloadAction covers a successful batch
+// response that schedules no action for one object: the object the client
+// cannot act on must be reported, and the objects scheduled beside it must
+// still reach the mirror (internal/lfs/batch.go turns the missing action into
+// the unavailable sentinel that downloadChunk has to isolate).
+func TestFetchAllSkipsObjectsWithoutDownloadAction(t *testing.T) {
+ first := []byte("first content")
+ firstOID, firstPointer := pointerFor(first)
+ second := []byte("second content")
+ secondOID, secondPointer := pointerFor(second)
+ third := []byte("third content")
+ thirdOID, thirdPointer := pointerFor(third)
+
+ lfsServer := newFakeLFSServer(t, map[string][]byte{
+ firstOID: first,
+ secondOID: second,
+ thirdOID: third,
+ })
+ lfsServer.actionlessOIDs = []string{secondOID}
+ repositoryPath := newRepoWithLFS(t, map[string]string{
+ "first.bin": firstPointer,
+ "second.bin": secondPointer,
+ "third.bin": thirdPointer,
+ })
+
+ err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "")
+ if err == nil {
+ t.Fatal("an object without a download action should be reported")
+ }
+ if !strings.Contains(err.Error(), "could not be fetched") {
+ t.Fatalf("err = %v, want it to say how many objects were skipped", err)
+ }
+ if !errors.Is(err, errObjectUnavailable) {
+ t.Fatalf("err = %v, want it to report the object as unavailable", err)
+ }
+
+ for _, oid := range []string{firstOID, thirdOID} {
+ path := filepath.Join(repositoryPath, ".git", "lfs", "objects", oid[0:2], oid[2:4], oid)
+ if _, err := os.Stat(path); err != nil {
+ t.Errorf("object %s should be mirrored beside the actionless one: %v", shortOID(oid), err)
+ }
+ }
+ if path := filepath.Join(repositoryPath, ".git", "lfs", "objects", secondOID[0:2], secondOID[2:4], secondOID); func() bool {
+ _, err := os.Stat(path)
+ return err == nil
+ }() {
+ t.Error("the actionless object must not be stored")
+ }
+}
+
+func TestFetchAllChunksLargePointerSets(t *testing.T) {
+ files := make(map[string]string, batchObjectLimit+2)
+ data := make(map[string][]byte, batchObjectLimit+2)
+ var lastOID string
+ for index := range batchObjectLimit + 2 {
+ content := []byte(fmt.Sprintf("object %d", index))
+ oid, pointerText := pointerFor(content)
+ data[oid] = content
+ files[fmt.Sprintf("file-%03d.bin", index)] = pointerText
+ lastOID = oid
+ }
+
+ lfsServer := newFakeLFSServer(t, data)
+ repositoryPath := newRepoWithLFS(t, files)
+
+ if err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", ""); err != nil {
+ t.Fatalf("FetchAll failed: %v", err)
+ }
+
+ // One discovery probe plus one request per chunk of batchObjectLimit.
+ wantCalls := 1 + (batchObjectLimit+2+batchObjectLimit-1)/batchObjectLimit
+ if calls := lfsServer.batchCallCount(); calls != wantCalls {
+ t.Errorf("batch calls = %d, want a probe plus one request per chunk of %d", calls, batchObjectLimit)
+ }
+ if _, err := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects",
+ lastOID[0:2], lastOID[2:4], lastOID)); err != nil {
+ t.Errorf("the last chunk should be mirrored too: %v", err)
+ }
+}
+
+// TestFetchAllStopsAfterFailedDiscovery covers an endpoint whose failures are not
+// about the request's shape — dropped connections, and the credentials and server
+// statuses that mean the same thing. Discovery asks once and reports, so the fetch
+// never starts rather than narrowing a chunk against an endpoint that is not
+// answering.
+func TestFetchAllStopsAfterFailedDiscovery(t *testing.T) {
+ files := make(map[string]string, batchObjectLimit*5)
+ for index := range batchObjectLimit * 5 {
+ _, pointerText := pointerFor([]byte(fmt.Sprintf("object %d", index)))
+ files[fmt.Sprintf("file-%04d.bin", index)] = pointerText
+ }
+ repositoryPath := newRepoWithLFS(t, files)
+
+ t.Run("dropped connections", func(t *testing.T) {
+ var requests int64
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ atomic.AddInt64(&requests, 1)
+ hijacked, _, err := w.(http.Hijacker).Hijack()
+ if err != nil {
+ return
+ }
+ _ = hijacked.Close()
+ }))
+ t.Cleanup(server.Close)
+
+ err := NewFetcher().FetchAll(context.Background(), repositoryPath, server.URL+"/repo.git", "", "")
+ if err == nil {
+ t.Fatal("an endpoint that never answers should be reported")
+ }
+ if errors.Is(err, errObjectUnavailable) {
+ t.Errorf("err = %v, want a transport failure rather than per-object unavailability", err)
+ }
+ // The remote carries its suffix, so there is one candidate to ask.
+ if got := atomic.LoadInt64(&requests); got != 1 {
+ t.Errorf("batch requests = %d, want a single discovery attempt", got)
+ }
+ })
+
+ for _, status := range []int{http.StatusUnauthorized, http.StatusTooManyRequests, http.StatusInternalServerError} {
+ t.Run(http.StatusText(status), func(t *testing.T) {
+ lfsServer := newFakeLFSServer(t, nil)
+ lfsServer.batchStatus = status
+
+ err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "")
+ if err == nil || !strings.Contains(err.Error(), fmt.Sprint(status)) {
+ t.Fatalf("FetchAll = %v, want the endpoint's status", err)
+ }
+ if errors.Is(err, errObjectUnavailable) {
+ t.Errorf("err = %v, want the endpoint failure rather than per-object unavailability", err)
+ }
+ // No halving, no chunks: discovery answered for the whole repository.
+ if calls := lfsServer.batchCallCount(); calls != 1 {
+ t.Errorf("batch calls = %d, want a single discovery attempt", calls)
+ }
+ })
+ }
+}
+
+// TestFetchAllReportsSystematicRejection covers an endpoint that answers
+// discovery and then rejects every batch as unprocessable, including
+// single-object ones, so the rejection outlives every split. It must be reported
+// as the endpoint's — not attributed to each object as unavailable — and the
+// sweep must stop once a second chunk confirms it rather than narrowing its way
+// through the whole repository.
+func TestFetchAllReportsSystematicRejection(t *testing.T) {
+ const pointers = batchObjectLimit * 5
+ files := make(map[string]string, pointers)
+ for index := range pointers {
+ _, pointerText := pointerFor([]byte(fmt.Sprintf("object %d", index)))
+ files[fmt.Sprintf("file-%04d.bin", index)] = pointerText
+ }
+
+ lfsServer := newFakeLFSServer(t, nil)
+ // Discovery is answered, then every fetch request is refused.
+ lfsServer.batchStatus = http.StatusUnprocessableEntity
+ lfsServer.serveProbe = true
+ repositoryPath := newRepoWithLFS(t, files)
+
+ err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "")
+ if err == nil || !strings.Contains(err.Error(), "rejected") {
+ t.Fatalf("FetchAll = %v, want the endpoint rejection reported", err)
+ }
+ if errors.Is(err, errObjectUnavailable) {
+ t.Errorf("err = %v, want an endpoint rejection rather than per-object unavailability", err)
+ }
+ if !strings.Contains(err.Error(), "422") {
+ t.Errorf("err = %v, want the endpoint's status", err)
+ }
+
+ // Two chunks are narrowed — the first to establish the rejection and the
+ // second to confirm it is the endpoint's — and the rest are left alone. A
+ // per-pointer sweep of this repository would take five hundred requests.
+ if calls := lfsServer.batchCallCount(); calls > 450 {
+ t.Errorf("batch calls = %d, want the systematic rejection to stop the sweep", calls)
+ }
+}
+
+// TestFetchAllNarrowsBothHalves covers a rejection that narrows onto a broken
+// half: the sibling half must still be requested and downloaded, rather than
+// being abandoned because the first half reported the endpoint's rejection.
+func TestFetchAllNarrowsBothHalves(t *testing.T) {
+ bad := []byte("bad content")
+ badOID, badPointer := pointerFor(bad)
+ good := []byte("good content")
+ goodOID, goodPointer := pointerFor(good)
+
+ lfsServer := newFakeLFSServer(t, map[string][]byte{
+ badOID: bad,
+ goodOID: good,
+ })
+ // Discovery is answered, so the fetch runs; every request naming the bad
+ // object is then refused, narrowing the chunk onto it.
+ lfsServer.serveProbe = true
+ lfsServer.rejectWhen = func(request batchRequest) int {
+ for _, object := range request.Objects {
+ if object.OID == badOID {
+ return http.StatusUnprocessableEntity
+ }
+ }
+ return http.StatusOK
+ }
+ repositoryPath := newRepoWithLFS(t, map[string]string{
+ "bad.bin": badPointer,
+ "good.bin": goodPointer,
+ })
+
+ err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "")
+ if err == nil || !strings.Contains(err.Error(), "unavailable") {
+ t.Fatalf("FetchAll = %v, want the refused object reported", err)
+ }
+ if _, statErr := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects",
+ goodOID[0:2], goodOID[2:4], goodOID)); statErr != nil {
+ t.Errorf("the sibling half should be downloaded: %v", statErr)
+ }
+}
+
+// TestFetchAllKeepsObjectsBesideAnAnsweredRejection covers a chunk whose halves
+// disagree: one half's batch is refused down to a single object, while the other
+// half's batch is answered with an object the endpoint will not serve. The
+// answer proves the endpoint processes batches, so the refused object must not be
+// mistaken for a broken endpoint — which would end the fetch and leave the
+// answered half's siblings unattempted.
+func TestFetchAllKeepsObjectsBesideAnAnsweredRejection(t *testing.T) {
+ held := []byte("held content")
+ heldOID, heldPointer := pointerFor(held)
+ phantom := []byte("phantom content")
+ phantomOID, phantomPointer := pointerFor(phantom)
+ refused := []byte("refused content")
+ refusedOID, refusedPointer := pointerFor(refused)
+
+ lfsServer := newFakeLFSServer(t, map[string][]byte{
+ heldOID: held,
+ phantomOID: phantom,
+ refusedOID: refused,
+ })
+ lfsServer.refusedOIDs = []string{phantomOID}
+ lfsServer.serveProbe = true
+ lfsServer.rejectWhen = func(request batchRequest) int {
+ for _, object := range request.Objects {
+ if object.OID == refusedOID {
+ return http.StatusUnprocessableEntity
+ }
+ }
+ return http.StatusOK
+ }
+ repositoryPath := newRepoWithLFS(t, map[string]string{
+ "held.bin": heldPointer,
+ "phantom.bin": phantomPointer,
+ "refused.bin": refusedPointer,
+ })
+
+ err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "")
+ if err == nil || !strings.Contains(err.Error(), "unavailable") {
+ t.Fatalf("FetchAll = %v, want the two unfetchable objects reported", err)
+ }
+
+ for _, oid := range []string{heldOID} {
+ if _, statErr := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects",
+ oid[0:2], oid[2:4], oid)); statErr != nil {
+ t.Errorf("object %s should be mirrored: %v", shortOID(oid), statErr)
+ }
+ }
+}
+
+// TestFetchAllKeepsSweepingAfterARefusedChunk covers a chunk the endpoint
+// answers nothing for — every object refused, which alone looks like an endpoint
+// rejecting every request — followed by a chunk it serves. The refused chunk
+// must not end the sweep, or a chunk of stale pointers would cost the repository
+// everything after it.
+func TestFetchAllKeepsSweepingAfterARefusedChunk(t *testing.T) {
+ const staleInFirstChunk = 3
+ files := make(map[string]string, batchObjectLimit*2)
+ data := make(map[string][]byte, batchObjectLimit*2)
+ var staleOIDs []string
+ for index := range batchObjectLimit * 2 {
+ content := []byte(fmt.Sprintf("object %d", index))
+ oid, pointerText := pointerFor(content)
+ data[oid] = content
+ files[fmt.Sprintf("file-%04d.bin", index)] = pointerText
+ if index < staleInFirstChunk {
+ staleOIDs = append(staleOIDs, oid)
+ }
+ }
+
+ lfsServer := newFakeLFSServer(t, data)
+ lfsServer.serveProbe = true
+ lfsServer.rejectWhen = func(request batchRequest) int {
+ for _, object := range request.Objects {
+ for _, stale := range staleOIDs {
+ if object.OID == stale {
+ return http.StatusUnprocessableEntity
+ }
+ }
+ }
+ return http.StatusOK
+ }
+ repositoryPath := newRepoWithLFS(t, files)
+
+ err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "")
+ if err == nil || !strings.Contains(err.Error(), "could not be fetched") {
+ t.Fatalf("FetchAll = %v, want the refused objects summarised", err)
+ }
+
+ // The chunk after the refused one is still fetched.
+ last := fmt.Sprintf("object %d", batchObjectLimit*2-1)
+ lastOID, _ := pointerFor([]byte(last))
+ if _, statErr := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects",
+ lastOID[0:2], lastOID[2:4], lastOID)); statErr != nil {
+ t.Errorf("the chunk after a refused one should still be fetched: %v", statErr)
+ }
+ for _, oid := range staleOIDs {
+ if _, statErr := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects",
+ oid[0:2], oid[2:4], oid)); statErr == nil {
+ t.Errorf("refused object %s must not be stored", shortOID(oid))
+ }
+ }
+}
+
+// TestFetchAllKeepsObjectsBesideARefusedHalf covers a chunk whose first half is
+// refused in full while its sibling is servable: the first half answers nothing
+// at all, which looks like an endpoint-wide rejection, and concluding that from
+// one half alone would abandon the sibling — and, one chunk later, the rest of
+// the repository.
+func TestFetchAllKeepsObjectsBesideARefusedHalf(t *testing.T) {
+ staleA := []byte("stale a")
+ staleAOID, staleAPointer := pointerFor(staleA)
+ staleB := []byte("stale b")
+ staleBOID, staleBPointer := pointerFor(staleB)
+ goodC := []byte("good c")
+ goodCOID, goodCPointer := pointerFor(goodC)
+
+ lfsServer := newFakeLFSServer(t, map[string][]byte{
+ staleAOID: staleA,
+ staleBOID: staleB,
+ goodCOID: goodC,
+ })
+ // Requests naming either stale object are refused, so a half holding both
+ // narrows to rejections that the endpoint never answers.
+ lfsServer.rejectWhen = func(request batchRequest) int {
+ for _, object := range request.Objects {
+ if object.OID == staleAOID || object.OID == staleBOID {
+ return http.StatusUnprocessableEntity
+ }
+ }
+ return http.StatusOK
+ }
+ repositoryPath := newRepoWithLFS(t, map[string]string{
+ "stale-a.bin": staleAPointer,
+ "stale-b.bin": staleBPointer,
+ "good-c.bin": goodCPointer,
+ })
+
+ err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "")
+ if err == nil || !strings.Contains(err.Error(), "unavailable") {
+ t.Fatalf("FetchAll = %v, want the refused objects reported", err)
+ }
+ if _, statErr := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects",
+ goodCOID[0:2], goodCOID[2:4], goodCOID)); statErr != nil {
+ t.Errorf("the servable sibling half should still be fetched: %v", statErr)
+ }
+}
+
+// TestFetchAllReportsAnEndpointThatStopsPartway covers a service that answers
+// discovery and then stops serving the fetch. Part of the repository is mirrored
+// by then, so the failure must be reported rather than recorded as a clean skip:
+// that would present a half-finished mirror as complete.
+func TestFetchAllReportsAnEndpointThatStopsPartway(t *testing.T) {
+ const pointers = batchObjectLimit * 2
+ files := make(map[string]string, pointers)
+ data := make(map[string][]byte, pointers)
+ for index := range pointers {
+ content := []byte(fmt.Sprintf("object %d", index))
+ oid, pointerText := pointerFor(content)
+ data[oid] = content
+ files[fmt.Sprintf("file-%04d.bin", index)] = pointerText
+ }
+
+ lfsServer := newFakeLFSServer(t, data)
+ // Discovery is answered; the fetch that follows looks like a host with
+ // nothing mounted at the path any more.
+ lfsServer.serveProbe = true
+ lfsServer.fetchStatus = http.StatusNotFound
+ repositoryPath := newRepoWithLFS(t, files)
+
+ err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "")
+ if err == nil {
+ t.Fatal("an endpoint that stops serving the fetch should be reported")
+ }
+ // ErrDisabled is how the mirror layer recognises an expected skip, so this
+ // failure must not match it through the chain.
+ if errors.Is(err, ErrDisabled) || errors.Is(err, ErrNoEndpoint) {
+ t.Fatalf("err = %v, want a failure rather than a clean skip", err)
+ }
+}
+
+func TestCollectPointersHandlesWindowsIllegalNames(t *testing.T) {
+ dir := t.TempDir()
+ repository, err := git.PlainInit(dir, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ worktree, err := repository.Worktree()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ content := []byte("weights")
+ oid, pointerText := pointerFor(content)
+ if err := os.MkdirAll(filepath.Join(dir, "nested"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dir, "nested", "weights.bin"), []byte(pointerText), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := worktree.Add("nested/weights.bin"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := worktree.Commit("add pointer", &git.CommitOptions{
+ Author: &object.Signature{Name: "test", Email: "test@example.com", When: time.Now()},
+ }); err != nil {
+ t.Fatal(err)
+ }
+
+ head, err := repository.Head()
+ if err != nil {
+ t.Fatal(err)
+ }
+ commit, err := repository.CommitObject(head.Hash())
+ if err != nil {
+ t.Fatal(err)
+ }
+ root, err := repository.TreeObject(commit.TreeHash)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var nested object.TreeEntry
+ for _, entry := range root.Entries {
+ if entry.Name == "nested" {
+ nested = entry
+ }
+ }
+ if nested.Hash.IsZero() {
+ t.Fatal("nested subtree not found")
+ }
+
+ // An empty subtree the hostile tree names twice, so the deduplication is
+ // exercised without competing with the commit's own trees for the visit.
+ empty := &object.Tree{}
+ emptyObject := repository.Storer.NewEncodedObject()
+ if err := empty.Encode(emptyObject); err != nil {
+ t.Fatal(err)
+ }
+ emptyHash, err := repository.Storer.SetEncodedObject(emptyObject)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ hostile := &object.Tree{Entries: []object.TreeEntry{
+ {Name: "empty-again", Hash: emptyHash, Mode: filemode.Dir},
+ {Name: "empty", Hash: emptyHash, Mode: filemode.Dir},
+ {Name: `src\windows.cpp`, Hash: plumbing.NewHash(strings.Repeat("a", 40)), Mode: filemode.Regular},
+ }}
+ encoded := repository.Storer.NewEncodedObject()
+ if err := hostile.Encode(encoded); err != nil {
+ t.Fatal(err)
+ }
+ hostileHash, err := repository.Storer.SetEncodedObject(encoded)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // A ref points at a commit, and the scan reaches trees through commits, so
+ // the hostile tree is committed before the ref names it.
+ signature := object.Signature{Name: "test", Email: "test@example.com", When: time.Now()}
+ hostileCommit := &object.Commit{
+ Author: signature,
+ Committer: signature,
+ Message: "hostile tree",
+ TreeHash: hostileHash,
+ }
+ commitObject := repository.Storer.NewEncodedObject()
+ if err := hostileCommit.Encode(commitObject); err != nil {
+ t.Fatal(err)
+ }
+ hostileCommitHash, err := repository.Storer.SetEncodedObject(commitObject)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ headRef := plumbing.NewHashReference(plumbing.NewBranchReferenceName("hostile"), hostileCommitHash)
+ if err := repository.Storer.SetReference(headRef); err != nil {
+ t.Fatal(err)
+ }
+
+ scan, err := collectPointers(context.Background(), repository)
+ if err != nil {
+ t.Fatalf("collectPointers failed on a tree with host-specific names: %v", err)
+ }
+ if len(scan.pointers) != 1 || scan.pointers[0].oid != oid {
+ t.Fatalf("pointers = %+v, want the single pointer %s", scan.pointers, oid)
+ }
+ // The hostile entry names a blob that was never stored, so the scan reports
+ // it rather than quietly reading a smaller set. The empty subtree behind the
+ // two directory entries is visited once, so it is not reported as skipped.
+ if len(scan.skipped) != 1 || scan.skipped[0].name != `src\windows.cpp` {
+ t.Errorf("skipped = %+v, want just the entry whose blob is missing", scan.skipped)
+ }
+}
+
+// TestCollectPointersReportsUnreadableSubtrees covers a subtree whose object is
+// missing — a partial fetch, say. Its pointers cannot be fetched, so the scan
+// has to name them as skipped rather than reporting a smaller pointer set as if
+// it were the whole truth.
+func TestCollectPointersReportsUnreadableSubtrees(t *testing.T) {
+ const missingName = "missing"
+ repositoryPath := addMissingSubtreeCommit(t, missingName)
+
+ repository, err := git.PlainOpen(repositoryPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ scan, err := collectPointers(context.Background(), repository)
+ if err != nil {
+ t.Fatalf("a missing subtree should not fail the scan: %v", err)
+ }
+ if len(scan.skipped) != 1 || scan.skipped[0].name != missingName {
+ t.Fatalf("skipped = %+v, want the unreadable subtree named", scan.skipped)
+ }
+ if scan.skipped[0].err == nil {
+ t.Error("the skip should carry the reason it was skipped")
+ }
+}
+
+// addMissingSubtreeCommit builds a repository whose history names a subtree
+// object that was never stored, so the pointer scan has something it cannot
+// read, and returns its path.
+func addMissingSubtreeCommit(t *testing.T, name string) string {
+ t.Helper()
+
+ repositoryPath := newRepoWithLFS(t, map[string]string{"README.md": "plain repo"})
+ repository, err := git.PlainOpen(repositoryPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ tree := &object.Tree{Entries: []object.TreeEntry{
+ {Name: name, Hash: plumbing.NewHash(strings.Repeat("b", 40)), Mode: filemode.Dir},
+ }}
+ treeObject := repository.Storer.NewEncodedObject()
+ if err := tree.Encode(treeObject); err != nil {
+ t.Fatal(err)
+ }
+ treeHash, err := repository.Storer.SetEncodedObject(treeObject)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ signature := object.Signature{Name: "test", Email: "test@example.com", When: time.Now()}
+ commit := &object.Commit{
+ Author: signature,
+ Committer: signature,
+ Message: "point at a missing subtree",
+ TreeHash: treeHash,
+ }
+ commitObject := repository.Storer.NewEncodedObject()
+ if err := commit.Encode(commitObject); err != nil {
+ t.Fatal(err)
+ }
+ commitHash, err := repository.Storer.SetEncodedObject(commitObject)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := repository.Storer.SetReference(plumbing.NewHashReference(plumbing.NewBranchReferenceName("hostile"), commitHash)); err != nil {
+ t.Fatal(err)
+ }
+ return repositoryPath
+}
+
+// TestFetchAllFailsWhenTheScanCouldNotReadEverything covers what the mirror
+// layer sees when part of the repository cannot be read: the pointers behind it
+// cannot be fetched either, so the fetch reports rather than returning success
+// for a mirror that is quietly incomplete.
+func TestFetchAllFailsWhenTheScanCouldNotReadEverything(t *testing.T) {
+ lfsServer := newFakeLFSServer(t, nil)
+ repositoryPath := addMissingSubtreeCommit(t, "missing")
+
+ err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "")
+ if err == nil || !strings.Contains(err.Error(), "could not be read") {
+ t.Fatalf("FetchAll = %v, want the unreadable entry reported", err)
+ }
+ if calls := lfsServer.batchCallCount(); calls != 0 {
+ t.Errorf("batch calls = %d, want no endpoint work for a scan that cannot complete", calls)
+ }
+}
+
+func TestFetchAllRejectsUnsafeLFSConfigOverrides(t *testing.T) {
+ oid, pointerText := pointerFor([]byte("content"))
+ cases := []struct {
+ name string
+ lfsURL string
+ }{
+ {"off-host override", "https://evil.example.com/lfs"},
+ {"non-http scheme", "ssh://git@evil.example.com/repo.git/lfs"},
+ {"relative value", "/custom/lfs"},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ lfsServer := newFakeLFSServer(t, map[string][]byte{oid: []byte("content")})
repositoryPath := newRepoWithLFS(t, map[string]string{
".lfsconfig": "[lfs]\n\turl = " + c.lfsURL + "\n",
"file.bin": pointerText,
@@ -379,7 +1830,74 @@ func TestCanonicalHostStripsSchemeDefaultPorts(t *testing.T) {
}
}
-func TestResolveEndpointOverrideHostAndScheme(t *testing.T) {
+// TestSuffixedEndpointAddsGitSuffix pins the endpoint a forge expects: GitHub
+// and GitLab answer a suffix-less /info/lfs with 422, so a configured remote
+// without .git still has to request the suffixed path. Everything else the
+// remote URL carries has to survive the rewrite.
+func TestSuffixedEndpointAddsGitSuffix(t *testing.T) {
+ cases := []struct {
+ remoteURL string
+ want string
+ }{
+ {"https://github.com/owner/repo", "https://github.com/owner/repo.git/info/lfs"},
+ {"https://github.com/owner/repo.git", "https://github.com/owner/repo.git/info/lfs"},
+ {"https://github.com/owner/repo.GIT", "https://github.com/owner/repo.GIT/info/lfs"},
+ {"https://github.com/owner/repo/", "https://github.com/owner/repo.git/info/lfs"},
+ {"https://git.example.com:8443/group/sub/repo", "https://git.example.com:8443/group/sub/repo.git/info/lfs"},
+ {"https://user:secret@git.example.com/owner/repo", "https://user:secret@git.example.com/owner/repo.git/info/lfs"},
+ {"https://git.example.com", "https://git.example.com/info/lfs"},
+ {"https://git.example.com/", "https://git.example.com/info/lfs"},
+ {"https://git.example.com/owner/repo?", "https://git.example.com/owner/repo.git/info/lfs"},
+ {"https://git.example.com/owner/repo#frag", "https://git.example.com/owner/repo.git/info/lfs"},
+ {"https://git.example.com/owner/re%2Fpo", "https://git.example.com/owner/re%2Fpo.git/info/lfs"},
+ {"https://git.example.com/owner/re po", "https://git.example.com/owner/re%20po.git/info/lfs"},
+ }
+
+ for _, c := range cases {
+ t.Run(redactedURL(c.remoteURL), func(t *testing.T) {
+ parsed, err := url.Parse(c.remoteURL)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := suffixedEndpoint(parsed); got != c.want {
+ t.Errorf("suffixedEndpoint(%q) = %q, want %q", redactedURL(c.remoteURL), redactedURL(got), redactedURL(c.want))
+ }
+ })
+ }
+}
+
+// TestEndpointCandidatesCollapseDuplicates covers remotes that reach the
+// LFS API at one path whichever candidate is derived — a remote that already
+// carries the repository suffix, and one with no path at all. Probing the same
+// URL twice would repeat every request of a whole repository for nothing.
+func TestEndpointCandidatesCollapseDuplicates(t *testing.T) {
+ for _, remoteURL := range []string{
+ "https://git.example.com/owner/repo.git",
+ "https://git.example.com",
+ } {
+ t.Run(redactedURL(remoteURL), func(t *testing.T) {
+ repositoryPath := newRepoWithLFS(t, map[string]string{"README.md": "plain repo"})
+ repository, err := git.PlainOpen(repositoryPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ parsed, parseErr := parseRemoteURL(remoteURL)
+ if parseErr != nil {
+ t.Fatal(parseErr)
+ }
+ endpoints, err := endpointCandidates(repository, parsed)
+ if err != nil {
+ t.Fatalf("resolveEndpoints failed: %v", err)
+ }
+ if len(endpoints) != 1 {
+ t.Errorf("endpoints = %q, want one candidate for a remote that has only one path", redactedURL(strings.Join(candidateURLs(endpoints), ",")))
+ }
+ })
+ }
+}
+
+func TestEndpointCandidatesOverrideHostAndScheme(t *testing.T) {
oid, pointerText := pointerFor([]byte("content"))
cases := []struct {
name string
@@ -394,6 +1912,12 @@ func TestResolveEndpointOverrideHostAndScheme(t *testing.T) {
remoteURL: "https://example.com/repo.git",
wantAddress: "https://example.com/custom/lfs",
},
+ {
+ name: "endpoint rooted at the host",
+ lfsURL: "https://example.com:443/",
+ remoteURL: "https://example.com/repo.git",
+ wantAddress: "https://example.com",
+ },
{
name: "http override downgrades the https remote",
lfsURL: "http://example.com/custom/lfs",
@@ -426,22 +1950,35 @@ func TestResolveEndpointOverrideHostAndScheme(t *testing.T) {
t.Fatal(err)
}
- address, err := resolveEndpoint(repository, c.remoteURL)
+ parsed, parseErr := parseRemoteURL(c.remoteURL)
+ if parseErr != nil {
+ t.Fatal(parseErr)
+ }
+ endpoints, err := endpointCandidates(repository, parsed)
if c.wantReject {
if err == nil {
- t.Fatalf("resolveEndpoint = %q, want rejection", address)
+ t.Fatalf("resolveEndpoints = %q, want rejection", redactedURL(strings.Join(candidateURLs(endpoints), ",")))
}
return
}
if err != nil {
- t.Fatalf("resolveEndpoint failed: %v", err)
+ t.Fatalf("resolveEndpoints failed: %v", err)
}
- if address != c.wantAddress {
- t.Errorf("endpoint = %q, want %q", address, c.wantAddress)
+ if len(endpoints) != 1 || endpoints[0].url != c.wantAddress {
+ t.Errorf("endpoints = %q, want just %q", redactedURL(strings.Join(candidateURLs(endpoints), ",")), redactedURL(c.wantAddress))
}
if lfsServer.batchCallCount() != 0 {
- t.Error("resolveEndpoint must not contact any endpoint")
+ t.Error("resolveEndpoints must not contact any endpoint")
}
})
}
}
+
+// candidateURLs renders candidates for a test message.
+func candidateURLs(candidates []endpointCandidate) []string {
+ urls := make([]string, 0, len(candidates))
+ for _, candidate := range candidates {
+ urls = append(urls, candidate.url)
+ }
+ return urls
+}
diff --git a/internal/lfs/scan.go b/internal/lfs/scan.go
index d9dab76..e9da170 100644
--- a/internal/lfs/scan.go
+++ b/internal/lfs/scan.go
@@ -10,6 +10,7 @@ import (
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
+ "github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
)
@@ -19,6 +20,21 @@ type pointer struct {
size int64
}
+// skippedSubtree records a subtree the scan could not read, so its pointers are
+// known to be missing from the result rather than silently absent.
+type skippedSubtree struct {
+ name string
+ hash plumbing.Hash
+ err error
+}
+
+// scanResult is what one scan of a repository found: the pointers it collected
+// and the subtrees it could not read.
+type scanResult struct {
+ pointers []pointer
+ skipped []skippedSubtree
+}
+
// pointerVersionValue is the version scheme URI on the first line of every
// LFS pointer file.
const pointerVersionValue = "https://git-lfs.github.com/spec/v1"
@@ -31,12 +47,13 @@ const pointerMaxBytes = 4 * 1024
// inspected once. Commit history is traversed explicitly (rather than per-ref
// iterators) so a repository with many refs costs one pass over its history,
// not one per ref.
-func collectPointers(ctx context.Context, repository *git.Repository) ([]pointer, error) {
+func collectPointers(ctx context.Context, repository *git.Repository) (scanResult, error) {
seenCommits := make(map[plumbing.Hash]struct{})
seenTrees := make(map[plumbing.Hash]bool)
seenBlobs := make(map[plumbing.Hash]struct{})
var pointers []pointer
+ var skipped []skippedSubtree
seenPointers := make(map[string]struct{})
enqueue := func(queue []plumbing.Hash, hash plumbing.Hash) []plumbing.Hash {
@@ -52,7 +69,7 @@ func collectPointers(ctx context.Context, repository *git.Repository) ([]pointer
queue := make([]plumbing.Hash, 0, 64)
refs, err := repository.References()
if err != nil {
- return nil, err
+ return scanResult{}, err
}
err = refs.ForEach(func(ref *plumbing.Reference) error {
if ref.Type() == plumbing.SymbolicReference || ref.Name().IsRemote() {
@@ -62,12 +79,12 @@ func collectPointers(ctx context.Context, repository *git.Repository) ([]pointer
return ctx.Err()
})
if err != nil {
- return nil, err
+ return scanResult{}, err
}
for len(queue) > 0 {
if err := ctx.Err(); err != nil {
- return nil, err
+ return scanResult{}, err
}
hash := queue[len(queue)-1]
@@ -85,8 +102,8 @@ func collectPointers(ctx context.Context, repository *git.Repository) ([]pointer
continue
}
- if err := scanTree(repository, commit.TreeHash, seenTrees, seenBlobs, seenPointers, &pointers); err != nil {
- return nil, err
+ if err := scanTree(repository, commit.TreeHash, seenTrees, seenBlobs, seenPointers, &skipped, &pointers); err != nil {
+ return scanResult{}, err
}
for _, parent := range commit.ParentHashes {
@@ -94,7 +111,7 @@ func collectPointers(ctx context.Context, repository *git.Repository) ([]pointer
}
}
- return pointers, nil
+ return scanResult{pointers: pointers, skipped: skipped}, nil
}
// peelToCommitHash resolves a ref tip hash to the commit it designates,
@@ -119,11 +136,24 @@ func peelToCommitHash(repository *git.Repository, hash plumbing.Hash) plumbing.H
}
}
+// scanTree collects pointers from every blob under treeHash, descending into
+// subtrees iteratively.
+//
+// Subtrees are walked from their tree objects directly rather than through
+// object.TreeWalker: that walker validates each entry against the rules for
+// materialising a working tree, so a repository containing a path that is
+// illegal on the host — a backslash in an entry name on Windows, for instance —
+// would abort the whole backup even though the mirror clones and uploads
+// perfectly well. A tree object only ever holds entry names and subtree hashes,
+// so enumerating them needs no path handling at all. seenTrees bounds the walk
+// for repositories whose history repeats or self-references a tree, and makes
+// the cross-ref scan visit each tree once (see collectPointers).
func scanTree(
repository *git.Repository,
treeHash plumbing.Hash,
seenTrees map[plumbing.Hash]bool, seenBlobs map[plumbing.Hash]struct{},
seenPointers map[string]struct{},
+ skipped *[]skippedSubtree,
pointers *[]pointer,
) error {
tree, err := repository.TreeObject(treeHash)
@@ -131,52 +161,72 @@ func scanTree(
return fmt.Errorf("read tree: %w", err)
}
- walker := object.NewTreeWalker(tree, true, seenTrees)
- defer walker.Close()
+ pending := []*object.Tree{tree}
+ seenTrees[treeHash] = true
- for {
- name, entry, err := walker.Next()
- if err == io.EOF {
- return nil
- }
- if err != nil {
- return fmt.Errorf("walk tree: %w", err)
- }
- _ = name
+ for len(pending) > 0 {
+ current := pending[len(pending)-1]
+ pending = pending[:len(pending)-1]
- if !entry.Mode.IsFile() {
- continue
- }
- if _, seen := seenBlobs[entry.Hash]; seen {
- continue
- }
- seenBlobs[entry.Hash] = struct{}{}
+ for _, entry := range current.Entries {
+ if entry.Mode == filemode.Dir {
+ if seenTrees[entry.Hash] {
+ continue
+ }
+ seenTrees[entry.Hash] = true
- blob, err := repository.BlobObject(entry.Hash)
- if err != nil {
- continue
- }
- if blob.Size > pointerMaxBytes {
- continue
- }
+ subtree, err := repository.TreeObject(entry.Hash)
+ if err != nil {
+ // A subtree missing from a partial fetch costs its
+ // pointers but leaves the rest of the scan intact; record
+ // it so the loss is reported rather than silent.
+ *skipped = append(*skipped, skippedSubtree{name: entry.Name, hash: entry.Hash, err: err})
+ continue
+ }
+ pending = append(pending, subtree)
+ continue
+ }
+ if !entry.Mode.IsFile() {
+ continue
+ }
+ if _, seen := seenBlobs[entry.Hash]; seen {
+ continue
+ }
+ seenBlobs[entry.Hash] = struct{}{}
- reader, err := blob.Reader()
- if err != nil {
- continue
- }
- content, err := io.ReadAll(io.LimitReader(reader, pointerMaxBytes))
- _ = reader.Close()
- if err != nil {
- continue
- }
+ blob, err := repository.BlobObject(entry.Hash)
+ if err != nil {
+ // A blob missing from a partial fetch costs whatever pointer it
+ // held; record it so the loss is reported rather than silent.
+ *skipped = append(*skipped, skippedSubtree{name: entry.Name, hash: entry.Hash, err: err})
+ continue
+ }
+ if blob.Size > pointerMaxBytes {
+ continue
+ }
- if parsed, ok := parsePointer(content); ok {
- if _, duplicate := seenPointers[parsed.oid]; !duplicate {
- seenPointers[parsed.oid] = struct{}{}
- *pointers = append(*pointers, parsed)
+ reader, err := blob.Reader()
+ if err != nil {
+ *skipped = append(*skipped, skippedSubtree{name: entry.Name, hash: entry.Hash, err: err})
+ continue
+ }
+ content, err := io.ReadAll(io.LimitReader(reader, pointerMaxBytes))
+ _ = reader.Close()
+ if err != nil {
+ *skipped = append(*skipped, skippedSubtree{name: entry.Name, hash: entry.Hash, err: err})
+ continue
+ }
+
+ if parsed, ok := parsePointer(content); ok {
+ if _, duplicate := seenPointers[parsed.oid]; !duplicate {
+ seenPointers[parsed.oid] = struct{}{}
+ *pointers = append(*pointers, parsed)
+ }
}
}
}
+
+ return nil
}
// parsePointer parses an LFS pointer file: the version line followed by