diff --git a/internal/lfs/batch.go b/internal/lfs/batch.go index ba2de10..a08f62e 100644 --- a/internal/lfs/batch.go +++ b/internal/lfs/batch.go @@ -6,12 +6,14 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "net/http" "net/url" "os" "path/filepath" + "strings" "time" ) @@ -30,9 +32,25 @@ type batchRequest struct { // batchAction is one server-provided action, here the basic-transfer download // instruction with its (typically pre-signed) URL. +// +// The expiry fields matter to a long run: a pre-signed URL stops working once it +// lapses, and the server is the only party that can issue a fresh one. type batchAction struct { - Href string `json:"href"` - Header map[string]string `json:"header"` + Href string `json:"href"` + Header map[string]string `json:"header"` + ExpiresIn int64 `json:"expires_in"` + ExpiresAt string `json:"expires_at"` +} + +// expired reports whether the action's URL has already lapsed. A past ExpiresAt +// is authoritative; otherwise the relative expiry is measured from now. +func (a *batchAction) expired() bool { + if a.ExpiresAt != "" { + if at, err := time.Parse(time.RFC3339, a.ExpiresAt); err == nil { + return !at.After(time.Now()) + } + } + return a.ExpiresIn < 0 } // batchResponseObject is the server's verdict for one object; exactly one of @@ -53,6 +71,30 @@ type batchResponse struct { Objects []batchResponseObject `json:"objects"` } +// errBatchRejected marks a batch request the endpoint answered with an +// unexpected status instead of scheduling the objects. Callers narrow such a +// request down to individual objects, because a rejection is usually one +// object's fault (the endpoint's object limit, a stale pointer) rather than the +// whole batch's. +var errBatchRejected = errors.New("batch request rejected by the endpoint") + +// errObjectUnavailable reports that the endpoint answered the batch request but +// will not serve one of the objects it named, so the object cannot be mirrored. +var errObjectUnavailable = errors.New("lfs object unavailable on the remote") + +// batchObjectError is one object the endpoint refused: a pointer kept in the +// repository after its object was removed or garbage-collected on the server. +type batchObjectError struct { + oid string + message string +} + +func (e *batchObjectError) Error() string { + return fmt.Sprintf("LFS object %s unavailable: %s", shortOID(e.oid), e.message) +} + +func (e *batchObjectError) Unwrap() error { return errObjectUnavailable } + // batchClient performs the LFS batch protocol over HTTP. type batchClient struct { client *http.Client @@ -67,10 +109,140 @@ func newBatchClient(client *http.Client) *batchClient { const lfsMediaType = "application/vnd.git-lfs+json" -// batch submits every pointer for download scheduling. A 403 or 404 from the -// endpoint means the remote has Git LFS switched off, which callers treat as -// an expected skip rather than a failure. -func (c *batchClient) batch(ctx context.Context, endpoint, username, password string, pointers []pointer) ([]batchResponseObject, error) { +// authorize offers the credentials to a request when there are any to offer. +// Every batch request goes out authenticated rather than waiting for a 401, +// because a forge is free to answer 403 to an anonymous request, and a 403 is +// also how a repository with LFS switched off answers. +func (c credentials) authorize(request *http.Request) { + if !c.available() { + return + } + request.SetBasicAuth(c.username, c.password) +} + +// probe asks an endpoint for one object to find out whether it serves this +// repository's LFS API at all. Discovery happens once, before any object is +// fetched, so the fetch itself runs against a single known-good endpoint: the +// answer separates "the API answered for this object" — including that the +// server does not have it — from "LFS is switched off here" and from "nothing is +// mounted at this path". +// +// A single object keeps the probe from tripping a server's object limit, so a +// refused probe is about the path rather than the request's shape. A nil return +// therefore means the API root is right, and the object's own fate is the +// fetch's business. +// +// The credentials are offered on the first request when there are any. Waiting +// for a 401 to offer them would read a forge that answers 403 to anonymous +// requests as a repository with LFS switched off, which backs it up without its +// LFS content instead of reporting that the credential was refused. +func (c *batchClient) probe(ctx context.Context, endpoint string, creds credentials, object pointer) error { + return c.doProbe(ctx, endpoint, creds, object) +} + +// doProbe performs one probe request, offering the credentials when there are +// any to offer. +func (c *batchClient) doProbe(ctx context.Context, endpoint string, creds credentials, object pointer) error { + body, err := json.Marshal(batchRequest{ + Operation: "download", + Transfers: []string{"basic"}, + Objects: toBatchObjects([]pointer{object}), + }) + if err != nil { + return err + } + + request, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint+"/objects/batch", bytes.NewReader(body)) + if err != nil { + return err + } + request.Header.Set("Content-Type", lfsMediaType) + request.Header.Set("Accept", lfsMediaType) + creds.authorize(request) + + response, err := c.client.Do(request) + if err != nil { + return fmt.Errorf("batch request: %w", err) + } + defer drainAndClose(response) + + switch response.StatusCode { + case http.StatusOK: + // The API answered for the object, whether by scheduling it or by + // reporting that the server does not have it. + return nil + case http.StatusUnauthorized: + // The endpoint refused what it was sent, and the credentials were + // already offered, so this is reported rather than retried: repeating + // the request would send the same credentials again. Reading it as a + // path problem would be worse still, because the path answered. + return fmt.Errorf("batch request failed with status %d", response.StatusCode) + case http.StatusForbidden: + // Two different things answer 403: a forge saying LFS is switched off + // for this repository, and a host page refusing a path it does not + // serve. The LFS API explains itself — in JSON, and sometimes with no + // body at all, which is still a verdict about the repository — while an + // edge page does not. + return classifyRefusal(response, ErrDisabled) + case http.StatusNotFound: + // Nothing is mounted at this path: a host that routes by path answers + // 404. + return ErrNoEndpoint + case http.StatusUnprocessableEntity: + // Unprocessable is how an edge refuses a path it does not route as much + // as it is how the API refuses a request's shape. A single object cannot + // trip an object limit, so a page here is a wrong path and anything else + // is the API refusing the probe. + return classifyRefusal(response, fmt.Errorf("batch request failed with status %d", response.StatusCode)) + default: + // Every other status describes the endpoint or the credential rather + // than the path — a rate limit, an authentication failure, the server's + // own error — so it stays a reported failure. Reading an edge's error + // page as "no endpoint here" would let the mirror layer record a broken + // or unauthorized service as a repository with LFS switched off. + return fmt.Errorf("batch request failed with status %d", response.StatusCode) + } +} + +// classifyRefusal reads a refusal the API did not phrase as a verdict about one +// object. Markup means a host page answered — the path is not the API — and +// anything else is the API speaking, which the caller reads as verdict. +func classifyRefusal(response *http.Response, verdict error) error { + body, err := io.ReadAll(io.LimitReader(response.Body, 4096)) + if err == nil && looksLikeHTML(body) { + return ErrNoEndpoint + } + return verdict +} + +// credentials are the HTTP basic-auth pair the remote's own API accepts. They +// travel as a value rather than as raw strings because a client may need to +// offer them more than once: the batch API answers 401 when credentials are +// needed but were not sent, and the request is then retried once with them. +type credentials struct { + username string + password string +} + +// available reports whether there is anything to offer. A request with no +// credentials to offer goes out anonymous, because there is nothing to add. +func (c credentials) available() bool { + return c.username != "" || c.password != "" +} + +// batch submits every pointer for download scheduling. +// +// A status describing the request's shape is reported as errBatchRejected so +// callers can retry the pointers in smaller batches; every other status +// describes the endpoint rather than what was asked of it, so splitting the +// chunk would only repeat the same answer more slowly. Discovery already +// established which endpoint serves the API, so a 403 or 404 here describes a +// service that has stopped answering rather than a path worth retrying. +// +// A 401 means the endpoint would not accept what was sent, so it is reported +// rather than retried: the credentials were already offered, and repeating the +// request would send the same ones. +func (c *batchClient) batch(ctx context.Context, creds credentials, endpoint string, pointers []pointer) ([]batchResponseObject, error) { body, err := json.Marshal(batchRequest{ Operation: "download", Transfers: []string{"basic"}, @@ -86,9 +258,7 @@ func (c *batchClient) batch(ctx context.Context, endpoint, username, password st } request.Header.Set("Content-Type", lfsMediaType) request.Header.Set("Accept", lfsMediaType) - if username != "" || password != "" { - request.SetBasicAuth(username, password) - } + creds.authorize(request) response, err := c.client.Do(request) if err != nil { @@ -96,11 +266,11 @@ func (c *batchClient) batch(ctx context.Context, endpoint, username, password st } defer drainAndClose(response) - switch response.StatusCode { - case http.StatusOK: + switch { + case response.StatusCode == http.StatusOK: // Handled below. - case http.StatusForbidden, http.StatusNotFound: - return nil, ErrDisabled + case describesRequestShape(response.StatusCode): + return nil, fmt.Errorf("%w: status %d", errBatchRejected, response.StatusCode) default: return nil, fmt.Errorf("batch request failed with status %d", response.StatusCode) } @@ -115,49 +285,141 @@ func (c *batchClient) batch(ctx context.Context, endpoint, username, password st return decoded.Objects, nil } +// describesRequestShape reports whether a status is how servers answer a request +// they will not process as sent, which a smaller batch may avoid: an unprocessable +// entity, a payload the server considers too large, or a body it cannot accept. +// Credentials, rate limiting, and the endpoint's own failures are not among them. +func describesRequestShape(status int) bool { + switch status { + case http.StatusBadRequest, + http.StatusRequestEntityTooLarge, + http.StatusUnprocessableEntity, + http.StatusNotAcceptable, + http.StatusUnsupportedMediaType: + return true + default: + return false + } +} + +// looksLikeHTML reports whether a body is a markup document, which is what an +// edge or proxy answers with and what the LFS API never does. A JSON document or +// an empty body is therefore read as the API speaking; markup is read as the path +// being refused by something that is not the API. +func looksLikeHTML(body []byte) bool { + trimmed := bytes.TrimSpace(bytes.ToLower(body)) + return bytes.HasPrefix(trimmed, []byte(" 0 { + size = wanted + } + if cached, err := os.Stat(filepath.Join(store, oid[0:2], oid[2:4], oid)); err == nil && + cached.Mode().IsRegular() && (size == 0 || cached.Size() == size) { + // Already cached by a previous snapshot; git-lfs also skips it, and + // content already held needs no fresh URL, so this is settled before + // the expiry below can send it back for rescheduling. A file of a + // length the recorded size contradicts is not this object, so it + // falls through to the download, which verifies the digest as bytes + // arrive. No size anywhere leaves the length unstated rather than + // contradicted, and re-downloading content the store already holds + // would risk failing over it. + continue } - action := object.Actions["download"] - if action == nil || action.Href == "" { - // The server knows the object but scheduled nothing; without a - // href there is nothing this client can do beyond reporting it. - return fmt.Errorf("LFS object %s has no download action", shortOID(object.OID)) + if action.expired() { + // The URL lapsed before it was used — a long scan, a slow batch, a + // small expires_in — so the object needs a fresh one rather than a + // transfer that can only fail. The resolved size travels with it, so + // the refresh request and the pass over its answer keep the length + // the pointer records rather than falling back to an endpoint that + // stated none. + expired = append(expired, pointer{oid: oid, size: size}) + continue } - - if err := downloadObject(ctx, client.client, store, endpointHost, username, password, object.OID, action); err != nil { - return err + if err := downloadObject(ctx, client.client, store, endpointHost, creds, oid, action); err != nil { + failed = append(failed, err) } } - return nil + return expired, unavailable, failed } +// downloadObject streams one object into the LFS cache, verifying its SHA-256 as +// bytes arrive. func downloadObject( ctx context.Context, client *http.Client, - store, endpointHost, username, password, oid string, + store, endpointHost string, + creds credentials, + oid string, action *batchAction, ) error { destination := filepath.Join(store, oid[0:2], oid[2:4], oid) - if _, err := os.Stat(destination); err == nil { - // Already cached by a previous snapshot; git-lfs also skips it. - return nil - } request, err := http.NewRequestWithContext(ctx, http.MethodGet, action.Href, nil) if err != nil { @@ -169,8 +431,8 @@ func downloadObject( // Pre-signed download URLs carry their own authorization; the basic // credential only applies while the request stays on the LFS endpoint's // host. - if hostOf(action.Href) == endpointHost && (username != "" || password != "") { - request.SetBasicAuth(username, password) + if hostOf(action.Href) == endpointHost && creds.available() { + request.SetBasicAuth(creds.username, creds.password) } response, err := client.Do(request) diff --git a/internal/lfs/endpoint.go b/internal/lfs/endpoint.go new file mode 100644 index 0000000..be57045 --- /dev/null +++ b/internal/lfs/endpoint.go @@ -0,0 +1,251 @@ +package lfs + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/url" + "strings" + + "github.com/go-git/go-git/v5" + "github.com/neurekadev/git-backup/internal/paths" +) + +// endpointCandidates lists, in the order they should be asked, the API roots a +// repository's LFS objects may live under: an lfs.url override from the +// repository's committed .lfsconfig when present, otherwise the remote URL's +// standard /info/lfs root with and without the repository's .git suffix. +// +// Which of the derived roots answers is a property of the host, and no rule +// decides it: GitHub and GitLab route only the suffixed path to their LFS +// service and answer a suffix-less /info/lfs with 422, while a host may mount +// the service exactly where its remote points. The distinction is therefore +// settled by asking, once, before any object is fetched — see selectEndpoint. +// +// The batch request authenticates with the remote's credential, so an override +// is honored only when it is an absolute http(s) URL on the remote's own host +// and scheme: a hostile .lfsconfig must not redirect that credential elsewhere +// or downgrade it to plaintext. Reading .lfsconfig is best-effort — any failure +// falls back to the derived candidates, matching the common deployment. +// endpointCandidate is one API root worth asking, and whether this package +// derived it rather than taking it from the configured remote. +// +// The distinction decides how much an answer weighs. A path the remote actually +// points at is the repository's own root, so its verdict speaks for the +// repository; a derived path is a guess about how the host routes LFS, and a +// guess answering "LFS is off" is weaker evidence than a real failure on the +// path the remote configures. +type endpointCandidate struct { + url string + // derived marks the suffixed root this package adds when the configured + // remote does not carry one. + derived bool +} + +// endpointCandidates lists, in the order they should be asked, the API roots a +// repository's LFS objects may live under: an lfs.url override from the +// repository's committed .lfsconfig when present, otherwise the remote URL's +// standard /info/lfs root with and without the repository's .git suffix. +// +// Which of the derived roots answers is a property of the host, and no rule +// decides it: GitHub and GitLab route only the suffixed path to their LFS +// service and answer a suffix-less /info/lfs with 422, while a host may mount +// the service exactly where its remote points. The distinction is therefore +// settled by asking, once, before any object is fetched — see selectEndpoint. +// +// The batch request authenticates with the remote's credential, so an override +// is honored only when it is an absolute http(s) URL on the remote's own host +// and scheme: a hostile .lfsconfig must not redirect that credential elsewhere +// or downgrade it to plaintext. Reading .lfsconfig is best-effort — any failure +// falls back to the derived candidates, matching the common deployment. +func endpointCandidates(repository *git.Repository, parsed *url.URL) ([]endpointCandidate, error) { + config, err := readLFSConfig(repository) + if err == nil && config != nil { + if override := strings.TrimSpace(config.Raw.Section("lfs").Option("url")); override != "" { + overridden, ok := paths.ParseHTTPURL(override) + if !ok { + return nil, fmt.Errorf("unsupported lfs.url '%s': only absolute http and https URLs are allowed", redactedURL(override)) + } + if !strings.EqualFold(overridden.Scheme, parsed.Scheme) || canonicalHost(overridden) != canonicalHost(parsed) { + return nil, fmt.Errorf("refusing lfs.url '%s': the override must stay on the remote host and scheme so the remote credential is not sent elsewhere", redactedURL(override)) + } + // Drop a redundant scheme-default port so the endpoint is canonical: + // object-download host comparisons and logs then match hrefs rendered + // without the explicit port. + if overridden.Port() != "" && isSchemeDefaultPort(overridden) { + overridden.Host = strings.TrimSuffix(overridden.Host, ":"+overridden.Port()) + } + // An override's own query, forced query, and fragment cannot ride + // along: the action path is appended to the endpoint, so anything + // after the path would land in the middle of the request line. + cleaned := endpointBase(overridden) + return []endpointCandidate{{url: strings.TrimSuffix(cleaned.String(), "/")}}, nil + } + } + + // A remote that already carries the repository suffix, or has no path at + // all, reaches the API at one path, so there is nothing to choose between. + suffixed, plain := suffixedEndpoint(parsed), plainEndpoint(parsed) + if suffixed == plain { + return []endpointCandidate{{url: suffixed}}, nil + } + // The configured remote's own path is asked second only because a forge that + // requires the suffix is the more common deployment; it is not the weaker + // candidate. + return []endpointCandidate{{url: suffixed, derived: true}, {url: plain}}, nil +} + +// selectEndpoint answers which candidate serves this repository's LFS API, by +// asking each one for a single object before anything is downloaded. +// +// Asking first is what keeps the fetch itself simple: a probe separates the +// answers a host can give — the API answered for the object, LFS is switched +// off, nothing is mounted at that path, or the endpoint failed — so the fetch +// then runs against one known-good endpoint with no fallback to reconcile. +// +// Every candidate is asked, and what decides the repository is the answer from +// the path the remote itself configures. A derived guess never overrides that +// path's answer in either direction: a guess reporting "LFS is off" must not mask +// a credential, rate-limit, server, or transport failure on the configured path +// — the mirror layer records disabled as a successful skip, so that would mirror +// the repository with no LFS content and raise nothing — and a guess failing must +// not turn a configured path's "LFS is off" into a failed backup. The guess only +// decides when the configured path said nothing itself, which is the case it +// exists for: a host that routes LFS solely under the suffixed path. +func (f *Fetcher) selectEndpoint( + ctx context.Context, + candidates []endpointCandidate, + creds credentials, + first pointer, +) (string, error) { + var configured, derived error + for _, candidate := range candidates { + switch err := f.client.probe(ctx, candidate.url, creds, first); { + case err == nil: + slog.Debug("Git LFS API answered.", "endpoint", redactedURL(candidate.url)) + return candidate.url, nil + case errors.Is(err, ErrDisabled): + slog.Debug("Git LFS is switched off for this repository.", "endpoint", redactedURL(candidate.url)) + if candidate.derived { + if derived == nil { + derived = err + } + } else if configured == nil { + configured = err + } + case errors.Is(err, ErrNoEndpoint): + slog.Debug("No Git LFS API is mounted at this endpoint.", "endpoint", redactedURL(candidate.url)) + default: + // The endpoint and the error both describe URLs, and the error's own + // text repeats the one the request went to, so it is redacted too + // before it reaches the log. + slog.Debug("Git LFS API did not answer.", + "endpoint", redactedURL(candidate.url), "detail", redactedURL(err.Error())) + if candidate.derived { + if derived == nil { + derived = err + } + } else if configured == nil { + configured = err + } + } + } + + if configured != nil { + if derived != nil { + // The guess's answer is kept for the debug trail rather than + // discarded — it is just not what decides the fetch. + slog.Debug("The derived path answered differently from the configured one.", + "detail", redactedURL(derived.Error())) + } + return "", configured + } + if derived != nil { + return "", derived + } + // No candidate serves the API. The mirror layer records that as LFS being + // switched off rather than failing the repository, which is the closest + // truthful reading available from a client. + return "", ErrDisabled +} + +// suffixedEndpoint derives the remote's standard LFS API root, including the +// repository's .git suffix. +// +// The parsed URL is copied and only its path rewritten, from its escaped form so +// percent-encoding survives: a remote whose path holds a reserved byte — an +// encoded slash in a repository name, say — must keep requesting that exact +// path. Everything else the configured remote carries, userinfo for a deployment +// that embeds credentials and a non-default port among it, still reaches the +// endpoint. A remote with no path has no repository to suffix, but the API root +// is still /info/lfs on that host. +func suffixedEndpoint(remoteURL *url.URL) string { + endpoint := endpointBase(remoteURL) + + escaped := strings.TrimSuffix(remoteURL.EscapedPath(), "/") + if escaped == "" { + return withPath(endpoint, "/info/lfs") + } + // The suffix is judged on the decoded path but appended to the escaped one, + // so a remote whose escaping hides it (re%2Egit) is not given a second + // suffix it did not need. It is matched case-insensitively, as the mirror's + // other .git handling does: a remote already ending in ".GIT" must not gain + // a differently cased one that no forge serves. + if !hasGitSuffix(strings.TrimSuffix(remoteURL.Path, "/")) { + escaped += ".git" + } + return withPath(endpoint, escaped+"/info/lfs") +} + +// plainEndpoint derives the LFS API root at the remote's configured path, +// without adding the repository suffix. +func plainEndpoint(remoteURL *url.URL) string { + endpoint := endpointBase(remoteURL) + return withPath(endpoint, strings.TrimSuffix(remoteURL.EscapedPath(), "/")+"/info/lfs") +} + +// endpointBase copies a remote URL with the parts an API root cannot carry +// removed: a query, a forced empty query, and a fragment would otherwise sit +// between the endpoint and the action path appended to it. +func endpointBase(remoteURL *url.URL) url.URL { + endpoint := *remoteURL + endpoint.RawQuery = "" + endpoint.ForceQuery = false + endpoint.Fragment = "" + endpoint.RawFragment = "" + return endpoint +} + +// withPath points an endpoint at an escaped path, keeping Path decoded so +// String() does not escape the escaping a second time. +func withPath(endpoint url.URL, escaped string) string { + endpoint.RawPath = escaped + endpoint.Path, _ = url.PathUnescape(escaped) + return endpoint.String() +} + +// hasGitSuffix reports whether a remote path already ends in the repository +// suffix, in any case. +func hasGitSuffix(path string) bool { + return len(path) >= len(".git") && strings.EqualFold(path[len(path)-len(".git"):], ".git") +} + +// isSchemeDefaultPort reports whether the URL's explicit port equals its +// scheme's default (http 80, https 443). +func isSchemeDefaultPort(u *url.URL) bool { + return (strings.EqualFold(u.Scheme, "http") && u.Port() == "80") || + (strings.EqualFold(u.Scheme, "https") && u.Port() == "443") +} + +// canonicalHost renders the URL's host lowercased with any scheme-default +// port removed, so an explicit https://host:443 compares equal to +// https://host while a redirect to any other port stays distinct. +func canonicalHost(u *url.URL) string { + host := strings.ToLower(u.Hostname()) + port := u.Port() + if port == "" || isSchemeDefaultPort(u) { + return host + } + return host + ":" + port +} diff --git a/internal/lfs/lfs.go b/internal/lfs/lfs.go index 8e6f589..ebd6422 100644 --- a/internal/lfs/lfs.go +++ b/internal/lfs/lfs.go @@ -14,6 +14,7 @@ import ( "errors" "fmt" "io" + "log/slog" "net/url" "path/filepath" "strings" @@ -45,92 +46,353 @@ func NewFetcher() *Fetcher { // repositoryPath, using remoteURL to derive the LFS endpoint. username and // password authenticate the batch request only; object downloads use the // server-provided (typically pre-signed) URLs. +// +// The work splits into three steps with one job each: reject a remote URL this +// package will not talk to, scan the mirror for pointers, settle which endpoint +// serves this repository's LFS API, and then fetch from that endpoint alone. +// Settling the endpoint first is what keeps the fetch simple — no fallback to +// reconcile, and no way for a working endpoint to be mistaken for a missing one +// partway through a repository. func (f *Fetcher) FetchAll(ctx context.Context, repositoryPath, remoteURL, username, password string) error { repository, err := git.PlainOpen(repositoryPath) if err != nil { return fmt.Errorf("open repository: %w", err) } - - endpoint, err := resolveEndpoint(repository, remoteURL) + parsed, err := parseRemoteURL(remoteURL) if err != nil { return err } - pointers, err := collectPointers(ctx, repository) + pointers, err := f.pointers(ctx, repository, remoteURL) + if err != nil || len(pointers) == 0 { + return err + } + + candidates, err := endpointCandidates(repository, parsed) if err != nil { - return fmt.Errorf("scan for LFS pointers: %w", err) + return err } - if len(pointers) == 0 { - // Nothing to fetch; never contact the endpoint so a forge without LFS - // is not mistaken for one that disabled it. - return nil + creds := credentials{username: username, password: password} + endpoint, err := f.selectEndpoint(ctx, candidates, creds, pointers[0]) + if err != nil { + return err } - objects, err := f.client.batch(ctx, endpoint, username, password, pointers) + store := objectStoreDir(repository, repositoryPath) + outcome, err := f.fetchBatches(ctx, store, endpoint, creds, pointers) if err != nil { return err } + if outcome.unavailable > 0 { + // The endpoint serves this repository but does not have every object a + // pointer names. The mirror keeps what it got; the gap is reported + // rather than passed over, because a mirror missing LFS content is not a + // complete backup. + slog.Warn("Some Git LFS objects could not be fetched; the repository was mirrored without them.", + "endpoint", redactedURL(endpoint), "objectsMissing", outcome.unavailable, "objectsRequested", len(pointers)) + slog.Debug("Git LFS objects that could not be fetched.", + "endpoint", redactedURL(endpoint), "reason", outcome.skipped.Error()) + return fmt.Errorf("%d of %d LFS objects could not be fetched: %w", + outcome.unavailable, len(pointers), outcome.skipped) + } + return nil +} - store := objectStoreDir(repository, repositoryPath) - return downloadObjects(ctx, f.client, store, endpoint, username, password, objects) -} - -// resolveEndpoint determines the LFS API root: an lfs.url override from the -// repository's committed .lfsconfig when present, otherwise the remote URL's -// standard /info/lfs root. The batch request authenticates with the remote's -// credential, so an override is honored only when it is an absolute http(s) -// URL on the remote's own host and scheme — a hostile .lfsconfig must not -// redirect that credential elsewhere or downgrade it to plaintext. Reading -// .lfsconfig is best-effort — any failure falls back to the derived endpoint, -// matching the common deployment. -func resolveEndpoint(repository *git.Repository, remoteURL string) (string, error) { - remote := strings.TrimSuffix(remoteURL, "/") - parsed, ok := paths.ParseHTTPURL(remote) +// parseRemoteURL rejects a remote this package will not talk to: only http and +// https are accepted, so a provider-supplied URL can never reach a transport +// helper or a local path. +func parseRemoteURL(remoteURL string) (*url.URL, error) { + parsed, ok := paths.ParseHTTPURL(remoteURL) if !ok { - return "", fmt.Errorf("unsupported remote URL '%s': only http and https are allowed", redactedURL(remoteURL)) + return nil, fmt.Errorf("unsupported remote URL '%s': only http and https are allowed", redactedURL(remoteURL)) + } + return parsed, nil +} + +// pointers scans the mirror for the LFS pointers its refs reach, reporting the +// parts of the repository it could not read. A scan that could not read +// everything fails the fetch: a backup reported as complete while content it +// could not scan is silently absent is worse than one that reports what it could +// not do. +func (f *Fetcher) pointers(ctx context.Context, repository *git.Repository, remoteURL string) ([]pointer, error) { + scan, err := collectPointers(ctx, repository) + if err != nil { + return nil, fmt.Errorf("scan for LFS pointers: %w", err) + } + if len(scan.skipped) > 0 { + for _, skipped := range scan.skipped { + slog.Warn("Could not read part of the repository while scanning for Git LFS pointers.", + "repository", redactedURL(remoteURL), "entry", skipped.name, + "oid", shortOID(skipped.hash.String()), "reason", skipped.err.Error()) + } + return nil, fmt.Errorf("%d entries could not be read while scanning for LFS pointers, so the mirror would be incomplete", len(scan.skipped)) + } + // Nothing to fetch means the endpoint is never contacted, so a forge without + // LFS is not mistaken for one that disabled it. + return scan.pointers, nil +} + +// batchObjectLimit is how many pointers one batch request submits. Git LFS +// clients use the same limit; a forge may reject a request that exceeds its own +// smaller limit, which fetchChunk then narrows down. +// +// Narrowing is not remembered between chunks, deliberately. A refusal cannot +// distinguish a server whose limit is low from a chunk carrying one object the +// server will not serve, and a remembered bound derived from the second kind +// would shrink every later chunk — including chunks of perfectly servable +// objects. The cost of not remembering is bounded: at most two requests per +// pointer in a chunk, and the sweep stops after a second failing chunk. +const batchObjectLimit = 100 + +// ErrNoEndpoint reports that nothing answered the LFS batch API at an endpoint, +// which is how a forge that routes the API elsewhere — or at all — looks to a +// client. FetchAll tries its next candidate endpoint on this answer, and a +// repository with no service anywhere is reported with ErrDisabled, because from +// here that is indistinguishable from LFS being switched off and the mirror +// layer already knows how to record it. +var ErrNoEndpoint = errors.New("no git lfs service answered at the endpoint") + +// fetchBatches downloads every pointer's object, splitting the pointers into +// batch requests. A batch the endpoint rejects is narrowed down by halving, so +// one object the server will not serve — a stale pointer, or a request above +// the server's own limit — costs that object alone instead of the repository's +// entire LFS mirror. Failing to mirror an object that exists still errors the +// fetch. +// +// Narrowing a chunk costs up to two requests per pointer in it, not a +// logarithmic count: a half the endpoint answered nothing for may hold nothing +// but individually refused objects, so the only way to know whether its sibling +// is servable is to ask it. A chunk the endpoint answered nothing for therefore +// still reports, and the sweep continues, because that state may mean a broken +// endpoint or merely a chunk of stale pointers. Once a second chunk fails, the +// fetch stops and reports rather than repeating the failure across a large +// repository. +func (f *Fetcher) fetchBatches( + ctx context.Context, + store, endpoint string, + creds credentials, + pointers []pointer, +) (chunkOutcome, error) { + var total chunkOutcome + var firstErr error + for start := 0; start < len(pointers); start += batchObjectLimit { + end := min(start+batchObjectLimit, len(pointers)) + if err := ctx.Err(); err != nil { + return total, err + } + + outcome, err := f.fetchChunk(ctx, store, endpoint, creds, pointers[start:end]) + total.absorb(outcome) + if err == nil { + continue + } + // A chunk the endpoint answered nothing for is reported, but the rest of + // the repository is still attempted: that state may mean a broken + // endpoint or merely a chunk of stale pointers, and only the next chunk + // tells them apart. + if firstErr != nil { + // The endpoint failed a second chunk, so stop asking. + slog.Debug("Stopping the Git LFS fetch after another chunk failed.", + "endpoint", redactedURL(endpoint), "detail", err.Error()) + break + } + firstErr = err + } + + if firstErr != nil { + return total, firstErr + } + if missing := total.unavailable + total.rejected; missing > 0 { + return total, fmt.Errorf("%d of %d LFS objects could not be fetched: %w", missing, len(pointers), total.skipped) } - endpoint := remote + "/info/lfs" + return total, nil +} + +// chunkOutcome reports what one batch request could not fetch — objects the +// endpoint answered for but will not serve, and objects whose own batch request +// the endpoint rejected outright — and whether it answered for any of them, which +// is what separates a rejection beside real answers from one that stands alone. +// skipped, when set, names the first object that could not be fetched. +type chunkOutcome struct { + unavailable int + rejected int + answered bool + skipped error +} - config, err := readLFSConfig(repository) - if err != nil || config == nil { - return endpoint, nil +// errNarrowedToSingleton marks the rejection of a single object's own batch +// request. Callers compare it with a sibling half's outcome to decide whether +// the endpoint refuses the request shape itself or merely that one object. +var errNarrowedToSingleton = errors.New("batch request rejected down to a single object") + +// errAllRejected marks a chunk whose every request was refused while the +// endpoint answered none of its objects, which is as close as a client can get to +// an endpoint-wide rejection. It is reported and, once a second chunk repeats it, +// treated as the endpoint being broken rather than as a chunk of stale pointers. +var errAllRejected = errors.New("every request for this batch was rejected") + +// fetchChunk submits one batch request for pointers and downloads what it +// schedules. A rejected batch is split in half and resubmitted, down to single +// pointers, so the rejection is narrowed to the object that caused it: a request +// the endpoint considers too large, or one poisonous object, then costs that +// object alone instead of the chunk's whole LFS content. +func (f *Fetcher) fetchChunk( + ctx context.Context, + store, endpoint string, + creds credentials, + pointers []pointer, +) (chunkOutcome, error) { + objects, err := f.client.batch(ctx, creds, endpoint, pointers) + if err == nil { + return f.downloadChunk(ctx, store, endpoint, creds, pointers, objects) } - if override := strings.TrimSpace(config.Raw.Section("lfs").Option("url")); override != "" { - overridden, ok := paths.ParseHTTPURL(override) - if !ok { - return "", fmt.Errorf("unsupported lfs.url '%s': only absolute http and https URLs are allowed", redactedURL(override)) + if !errors.Is(err, errBatchRejected) { + return chunkOutcome{}, err + } + // Descending calls answer with their own errors, so the rejection that + // brought us here is kept for the reports below. + rejection := err + + // A rejected chunk wider than one pointer is retried as two batches, and + // each half narrows further on its own. Both halves are always tried, so a + // failure in one cannot cost the other its objects, and the pair is what + // makes the chunk's outcome complete whichever half went wrong. + if len(pointers) > 1 { + middle := len(pointers) / 2 + outcome, firstErr := f.fetchChunk(ctx, store, endpoint, creds, pointers[:middle]) + if firstErr != nil && !isRefusal(firstErr) { + // A half that failed outright still must not cost its sibling. + rest, _ := f.fetchChunk(ctx, store, endpoint, creds, pointers[middle:]) + outcome.absorb(rest) + return outcome, firstErr } - if !strings.EqualFold(overridden.Scheme, parsed.Scheme) || canonicalHost(overridden) != canonicalHost(parsed) { - return "", fmt.Errorf("refusing lfs.url '%s': the override must stay on the remote host and scheme so the remote credential is not sent elsewhere", redactedURL(override)) + + rest, restErr := f.fetchChunk(ctx, store, endpoint, creds, pointers[middle:]) + outcome.absorb(rest) + if restErr != nil && !isRefusal(restErr) { + return outcome, restErr + } + if firstErr == nil && restErr == nil { + return outcome, nil } - // Drop a redundant scheme-default port so the endpoint is canonical: - // object-download host comparisons and logs then match hrefs rendered - // without the explicit port. - if overridden.Port() != "" && isSchemeDefaultPort(overridden) { - overridden.Host = strings.TrimSuffix(overridden.Host, ":"+overridden.Port()) + // A rejection reached a single pointer in one of the halves, and the + // whole chunk is now accounted for, so the rejection can be attributed. + return refuseOrReport(outcome, rejection) + } + + // The endpoint refuses this single object's own request, so nothing narrower + // can be blamed. Whether that is this object's fault or the endpoint's is + // decided one level up, by whether the sibling half was served; this subtree + // has served nothing either way. + // + // The object's own report renders the rejection rather than wrapping it: an + // error matching both sentinels would read as the endpoint's refusal and as + // one object's unavailability at once, and callers weigh those differently. + slog.Debug("Git LFS batch request rejected for a single object.", + "endpoint", redactedURL(endpoint), "oid", shortOID(pointers[0].oid), "reason", rejection.Error()) + return chunkOutcome{ + rejected: 1, + skipped: fmt.Errorf("%w: LFS object %s download request rejected: %v", + errObjectUnavailable, shortOID(pointers[0].oid), rejection), + }, fmt.Errorf("%w: %w", errNarrowedToSingleton, rejection) +} + +// isRefusal reports whether an error means the endpoint refused a request +// rather than failing to serve it. +func isRefusal(err error) bool { + return errors.Is(err, errNarrowedToSingleton) || errors.Is(err, errAllRejected) +} + +// refuseOrReport settles a chunk whose rejection reached a single pointer. An +// object the endpoint answered for settles it: whether that answer served bytes +// or reported the object missing, the endpoint processed the batch, so a refused +// sibling object is that object's fault rather than the endpoint's. A chunk the +// endpoint answered nothing for is the alternative — allRejected wrapping +// errBatchRejected — which callers report and stop the sweep after, since only a +// second such chunk tells stale pointers apart from a broken endpoint. +func refuseOrReport(outcome chunkOutcome, rejection error) (chunkOutcome, error) { + if outcome.answered || outcome.unavailable > 0 { + return outcome, nil + } + return outcome, fmt.Errorf("%w: %w", errAllRejected, rejection) +} + +// downloadChunk streams every object the batch scheduled. Objects the endpoint +// will not serve are recorded and skipped so their siblings still download, and +// so are objects this client could not transfer — attempted, but unreachable — +// which come back as an error once every object has had its turn. The chunk +// counts as answered when the endpoint replied with content for any scheduled +// object, which is what tells a rejection beside real answers from one that +// stands alone. +// +// Objects whose scheduled URL had already lapsed are asked about once more: the +// server is the only party that can issue a fresh URL, and a lapsed one would +// otherwise fail a transfer that was ready to succeed. +func (f *Fetcher) downloadChunk( + ctx context.Context, + store, endpoint string, + creds credentials, + want []pointer, + objects []batchResponseObject, +) (chunkOutcome, error) { + expired, unavailable, failed := downloadObjects(ctx, f.client, store, endpoint, creds, want, objects) + + var outcome chunkOutcome + if len(expired) > 0 { + rescheduled, err := f.client.batch(ctx, creds, endpoint, expired) + if err != nil { + // The endpoint scheduled these objects and then would not schedule + // them again, so each one is recorded as unserved with its own name + // and the reason the fresh URL never arrived. That record is the + // failure: reporting the request's error again would turn objects + // that were accounted for into an unaccounted chunk failure. + for _, object := range expired { + outcome.recordUnavailable(fmt.Errorf("%w: no fresh download URL was issued for %s: %w", errObjectUnavailable, object.oid, err)) + } + } else { + againExpired, againUnavailable, againFailed := downloadObjects(ctx, f.client, store, endpoint, creds, expired, rescheduled) + for _, object := range againExpired { + // A server issuing an already-lapsed URL twice has nothing more + // to offer for these objects, and each is named so a caller can + // tell which content the mirror is missing. + outcome.recordUnavailable(fmt.Errorf("%w: the download URL for %s lapsed before it could be used", errObjectUnavailable, object.oid)) + } + unavailable = append(unavailable, againUnavailable...) + failed = append(failed, againFailed...) } - return strings.TrimSuffix(overridden.String(), "/"), nil } - return endpoint, nil + + for _, object := range unavailable { + outcome.recordUnavailable(fmt.Errorf("%w: %s", errObjectUnavailable, object.Error())) + } + // The endpoint answered for every object that is not unavailable, whether it + // streamed the bytes now or found them already in the store. A chunk it + // answered for is not a chunk it refused, so a rejection beside those answers + // is one object's fault rather than the endpoint's. + outcome.answered = len(objects) > len(unavailable) + if len(failed) > 0 { + return outcome, fmt.Errorf("%d of %d LFS objects could not be downloaded: %w", len(failed), len(objects), failed[0]) + } + return outcome, nil } -// isSchemeDefaultPort reports whether the URL's explicit port equals its -// scheme's default (http 80, https 443). -func isSchemeDefaultPort(u *url.URL) bool { - return (strings.EqualFold(u.Scheme, "http") && u.Port() == "80") || - (strings.EqualFold(u.Scheme, "https") && u.Port() == "443") +// recordUnavailable counts one object the endpoint answered for but will not +// serve, keeping the first reason for the caller's report. +func (o *chunkOutcome) recordUnavailable(reason error) { + o.unavailable++ + if o.skipped == nil { + o.skipped = reason + } } -// canonicalHost renders the URL's host lowercased with any scheme-default -// port removed, so an explicit https://host:443 compares equal to -// https://host while a redirect to any other port stays distinct. -func canonicalHost(u *url.URL) string { - host := strings.ToLower(u.Hostname()) - port := u.Port() - if port == "" || isSchemeDefaultPort(u) { - return host +// absorb adds a split's second half to its first. +func (o *chunkOutcome) absorb(half chunkOutcome) { + o.unavailable += half.unavailable + o.rejected += half.rejected + o.answered = o.answered || half.answered + if half.skipped != nil && o.skipped == nil { + o.skipped = half.skipped } - return host + ":" + port } // redactedURL renders a URL with any embedded password masked, for safe diff --git a/internal/lfs/lfs_test.go b/internal/lfs/lfs_test.go index 60bfd48..66fee61 100644 --- a/internal/lfs/lfs_test.go +++ b/internal/lfs/lfs_test.go @@ -7,20 +7,31 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "net/http/httptest" "net/url" "os" "path/filepath" + "slices" "strings" "sync" + "sync/atomic" "testing" "time" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/filemode" "github.com/go-git/go-git/v5/plumbing/object" ) +// roundTripFunc adapts a function to an http.RoundTripper, so a test can script +// the exact responses the batch client sees. +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { return f(request) } + func pointerFor(content []byte) (string, string) { sum := sha256.Sum256(content) oid := hex.EncodeToString(sum[:]) @@ -70,15 +81,59 @@ type fakeLFSServer struct { downloads map[string]int // batchStatus, when non-zero, is the status returned for batch requests. batchStatus int + // refusedOIDs are answered with a per-object error instead of a download + // action, like a pointer whose object is gone from the server. + refusedOIDs []string + // actionlessOIDs are answered with neither an error nor a download action, + // as a misbehaving endpoint may do. + actionlessOIDs []string + // rejectWhen, when set, decides the status of each batch request so a test + // can model an endpoint that rejects some request shapes. + rejectWhen func(batchRequest) int // corruptDownload serves wrong bytes for every object. corruptDownload bool // batchPath overrides the expected batch path (lfs.url override tests). batchPath string + // servePaths, when set, are the only batch paths the server answers, so any + // other path answers 404 like a host with no LFS service mounted there. + servePaths []string + // disabledPaths answer 403, the response for a repository with LFS switched + // off, which is not the same as a path with no service behind it. + disabledPaths []string + // serveProbe answers discovery probes normally, so a test can have discovery + // succeed while the batch requests that follow are rejected. + serveProbe bool + // failedPaths answer a status for one batch path only, so a test can model a + // candidate whose endpoint is broken while another answers. + failedPaths map[string]int + // requireAuth answers 401 to any request without an Authorization header, + // which is how a forge that wants credentials for the objects behaves. + requireAuth bool + // forbidAnonymous answers 403 to any request without an Authorization + // header, which is how a forge behaves when it refuses anonymous access + // outright rather than inviting a credential with 401. A 403 is also the + // status for LFS switched off, so a client that discovers the endpoint + // anonymously cannot tell the two apart. + forbidAnonymous bool + // lapseFirstAction makes the first schedule for each object point at an + // already-expired URL, so the client has to ask for a fresh one. + lapseFirstAction bool + // lapseEveryAction makes every schedule for each object point at an + // already-expired URL, so asking again cannot produce a usable one. + lapseEveryAction bool + // scheduled counts how many times each object has been scheduled. + scheduled map[string]int + // fetchStatus, when non-zero, is the status batch requests get once a probe + // has been answered, so a test can model an endpoint that serves discovery + // and then stops serving the fetch. + fetchStatus int + // probeCalls counts the discovery probes the handler answered. + probeCalls int } func newFakeLFSServer(t *testing.T, data map[string][]byte) *fakeLFSServer { t.Helper() - s := &fakeLFSServer{data: data, downloads: make(map[string]int)} + s := &fakeLFSServer{data: data, downloads: make(map[string]int), scheduled: make(map[string]int)} mux := http.NewServeMux() mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { switch { @@ -95,23 +150,119 @@ func newFakeLFSServer(t *testing.T, data map[string][]byte) *fakeLFSServer { return s } +// writeUnmounted answers the way a forge's edge does for a repository path it +// does not route to an LFS API: a page, which is what tells a client the path is +// wrong rather than the repository LFS-free. +func writeUnmounted(w http.ResponseWriter) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.WriteHeader(http.StatusUnprocessableEntity) + _, _ = w.Write([]byte("\nOh nounexpected")) +} + // remoteURL is the remote a repository would use to reach the fake server. func (s *fakeLFSServer) remoteURL() string { return s.server.URL + "/repo.git" } +// plainRemoteURL is the same repository addressed without its .git suffix, the +// form a configuration normally carries. +func (s *fakeLFSServer) plainRemoteURL() string { + return s.server.URL + "/repo" +} + +// endpointFor mounts the server's batch API at the path a remote addresses, so +// a test chooses where the LFS service answers, and makes every other path +// answer 404 like a host with nothing mounted there. +func (s *fakeLFSServer) endpointFor(remoteURL string) string { + s.serve([]string{remoteURL}) + return remoteURL +} + +// disabledFor makes the given remotes answer 403 — the response a forge uses for +// a repository with LFS switched off — while the rest answer 404. +func (s *fakeLFSServer) disabledFor(remoteURLs ...string) { + disabled := make([]string, 0, len(remoteURLs)) + for _, remoteURL := range remoteURLs { + disabled = append(disabled, batchPathFor(remoteURL)) + } + s.mu.Lock() + s.disabledPaths = disabled + s.mu.Unlock() +} + +// serve mounts the batch API at the paths the given remotes address. +func (s *fakeLFSServer) serve(remoteURLs []string) { + served := make([]string, 0, len(remoteURLs)) + for _, remoteURL := range remoteURLs { + served = append(served, batchPathFor(remoteURL)) + } + s.mu.Lock() + s.servePaths = served + s.mu.Unlock() +} + +// batchPathFor is the batch action path a remote addresses. +func batchPathFor(remoteURL string) string { + parsed, err := url.Parse(remoteURL) + if err != nil { + return remoteURL + } + return parsed.Path + "/info/lfs/objects/batch" +} + func (s *fakeLFSServer) handleBatch(w http.ResponseWriter, r *http.Request) { s.mu.Lock() s.batchCalls++ status, batchPath := s.batchStatus, s.batchPath + rejectWhen, serveProbe := s.rejectWhen, s.serveProbe + probed := s.probeCalls > 0 + fetchStatus := s.fetchStatus + failedPaths := s.failedPaths + requireAuth := s.requireAuth + forbidAnonymous := s.forbidAnonymous + disabled := slices.Contains(s.disabledPaths, r.URL.Path) + servedHere := slices.Contains(s.servePaths, r.URL.Path) + hasServePaths := len(s.servePaths) > 0 s.mu.Unlock() + // A path the server does not serve answers the way the forges this package + // was written against do: a host page for the path it will not route, and + // markup rather than an LFS explanation. The disabled verdict is decided + // first, because a repository with LFS off answers 403 for its own path even + // when the test also mounts the service elsewhere. if batchPath != "" && r.URL.Path != batchPath { - w.WriteHeader(http.StatusNotFound) + writeUnmounted(w) return } - if status != 0 { - w.WriteHeader(status) + if requireAuth && r.Header.Get("Authorization") == "" { + // The API wants credentials it was not given, which the client is + // expected to retry with. + w.WriteHeader(http.StatusUnauthorized) + return + } + if forbidAnonymous && r.Header.Get("Authorization") == "" { + // An anonymous request is refused outright, and the refusal is shaped + // exactly like a repository with LFS switched off. + w.Header().Set("Content-Type", lfsMediaType) + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"message":"Git LFS is disabled for this repository."}`)) + return + } + if disabled { + // A forge with LFS switched off explains itself in JSON. + w.Header().Set("Content-Type", lfsMediaType) + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"message":"Git LFS is disabled for this repository."}`)) + return + } + if failedStatus, fails := failedPaths[r.URL.Path]; fails { + // One candidate path fails while another answers, so a test can model a + // wrong guess that is not merely unmounted. + w.WriteHeader(failedStatus) + return + } + if hasServePaths && !servedHere { + writeUnmounted(w) return } @@ -121,19 +272,92 @@ func (s *fakeLFSServer) handleBatch(w http.ResponseWriter, r *http.Request) { return } + // A discovery probe carries one object. A server configured to answer it + // does so before any rule about the fetch applies, so a test can model + // discovery succeeding and the fetch that follows failing. + if len(request.Objects) == 1 && serveProbe && !probed { + s.mu.Lock() + s.probeCalls++ + s.mu.Unlock() + s.respond(w, request) + return + } + if status != 0 { + w.WriteHeader(status) + return + } + // fetchStatus, when set, is what the fetch gets once discovery has been + // answered, so a test can model an endpoint that serves discovery and then + // stops serving the fetch. + if fetchStatus != 0 && probed { + w.WriteHeader(fetchStatus) + return + } + + // Mimic a forge that rejects some batch requests, such as one carrying more + // objects than its limit allows. + if rejectWhen != nil { + if code := rejectWhen(request); code != http.StatusOK { + w.WriteHeader(code) + return + } + } + s.respond(w, request) +} + +// respond answers a batch request with the object verdicts the server holds. +func (s *fakeLFSServer) respond(w http.ResponseWriter, request batchRequest) { + s.mu.Lock() + refused := make(map[string]struct{}, len(s.refusedOIDs)) + for _, oid := range s.refusedOIDs { + refused[oid] = struct{}{} + } + actionless := make(map[string]struct{}, len(s.actionlessOIDs)) + for _, oid := range s.actionlessOIDs { + actionless[oid] = struct{}{} + } + s.mu.Unlock() + response := batchResponse{Objects: make([]batchResponseObject, 0, len(request.Objects))} for _, object := range request.Objects { - if _, known := s.data[object.OID]; !known { + if _, isRefused := refused[object.OID]; isRefused { + response.Objects = append(response.Objects, batchResponseObject{ + OID: object.OID, + Error: &batchError{Code: http.StatusUnprocessableEntity, Message: "refused"}, + }) + continue + } + content, known := s.data[object.OID] + if !known { response.Objects = append(response.Objects, batchResponseObject{ OID: object.OID, Error: &batchError{Code: http.StatusNotFound, Message: "Object does not exist"}, }) continue } + if _, bare := actionless[object.OID]; bare { + // The server knows the object but schedules nothing for it: a + // successful response the client cannot act on. + response.Objects = append(response.Objects, batchResponseObject{ + OID: object.OID, + Size: int64(len(content)), + }) + continue + } + action := &batchAction{Href: s.server.URL + "/download/" + object.OID} + s.mu.Lock() + s.scheduled[object.OID]++ + lapse := s.lapseEveryAction || (s.lapseFirstAction && s.scheduled[object.OID] == 1) + s.mu.Unlock() + if lapse { + // A URL that has already lapsed, as a server issuing a short-lived + // one before a slow scan would produce. + action.ExpiresAt = time.Now().Add(-time.Minute).UTC().Format(time.RFC3339) + } response.Objects = append(response.Objects, batchResponseObject{ OID: object.OID, - Size: int64(len(s.data[object.OID])), - Actions: map[string]*batchAction{"download": {Href: s.server.URL + "/download/" + object.OID}}, + Size: int64(len(content)), + Actions: map[string]*batchAction{"download": action}, }) } w.Header().Set("Content-Type", lfsMediaType) @@ -256,16 +480,541 @@ func TestFetchAllSkipsEndpointWhenNoPointers(t *testing.T) { func TestFetchAllDisabledRemote(t *testing.T) { oid, pointerText := pointerFor([]byte("content")) - for _, status := range []int{http.StatusForbidden, http.StatusNotFound} { - lfsServer := newFakeLFSServer(t, map[string][]byte{oid: []byte("content")}) - lfsServer.batchStatus = status - repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText}) - - err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "") - if !errors.Is(err, ErrDisabled) { - t.Errorf("status %d should map to ErrDisabled, got %v", status, err) + lfsServer := newFakeLFSServer(t, map[string][]byte{oid: []byte("content")}) + // Both candidate paths answer as a repository with LFS switched off. + lfsServer.disabledFor(lfsServer.plainRemoteURL(), lfsServer.remoteURL()) + repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText}) + + err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.plainRemoteURL(), "", "") + if !errors.Is(err, ErrDisabled) { + t.Errorf("a 403 from every candidate should map to ErrDisabled, got %v", err) + } +} + +// TestFetchAllFallsBackToTheConfiguredPath covers a host that mounts its LFS +// service exactly where the remote points rather than under the repository's +// .git path: the suffixed endpoint answers 404, and the suffix-less one must +// then be tried rather than the remote being written off as LFS-free. +func TestFetchAllFallsBackToTheConfiguredPath(t *testing.T) { + content := []byte("content") + oid, pointerText := pointerFor(content) + + lfsServer := newFakeLFSServer(t, map[string][]byte{oid: content}) + remoteURL := lfsServer.endpointFor(lfsServer.plainRemoteURL()) + repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText}) + + if err := NewFetcher().FetchAll(context.Background(), repositoryPath, remoteURL, "", ""); err != nil { + t.Fatalf("FetchAll failed: %v", err) + } + cached, err := os.ReadFile(filepath.Join(repositoryPath, ".git", "lfs", "objects", + oid[0:2], oid[2:4], oid)) + if err != nil { + t.Fatalf("the object should be mirrored through the fallback endpoint: %v", err) + } + if string(cached) != string(content) { + t.Error("mirrored content should match the served bytes") + } +} + +// TestFetchAllFallsBackWhenTheGuessIsForbidden covers the guessed .git path +// answering 403 while the configured path serves normally. The 403 is about a +// path that does not exist rather than about the repository's LFS setting, so it +// must not be believed before the other candidate has been tried. +func TestFetchAllFallsBackWhenTheGuessIsForbidden(t *testing.T) { + content := []byte("content") + oid, pointerText := pointerFor(content) + + lfsServer := newFakeLFSServer(t, map[string][]byte{oid: content}) + remoteURL := lfsServer.endpointFor(lfsServer.plainRemoteURL()) + lfsServer.disabledFor(lfsServer.remoteURL()) + repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText}) + + if err := NewFetcher().FetchAll(context.Background(), repositoryPath, remoteURL, "", ""); err != nil { + t.Fatalf("FetchAll failed: %v", err) + } + if _, err := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects", + oid[0:2], oid[2:4], oid)); err != nil { + t.Errorf("the configured path should still be tried after a 403 on the guess: %v", err) + } +} + +// TestFetchAllNoEndpointAnywhereIsReported covers a host with no LFS service at +// either candidate path, which the mirror layer records as LFS being switched +// off — the two are indistinguishable from here, and both are expected states +// rather than a failed repository. +func TestFetchAllNoEndpointAnywhereIsReported(t *testing.T) { + oid, pointerText := pointerFor([]byte("content")) + + lfsServer := newFakeLFSServer(t, map[string][]byte{oid: []byte("content")}) + lfsServer.endpointFor(lfsServer.server.URL + "/somewhere/else") + repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText}) + + err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.plainRemoteURL(), "", "") + if errors.Is(err, ErrNoEndpoint) { + t.Fatalf("err = %v, want the expected-skip verdict rather than an internal signal", err) + } + if !errors.Is(err, ErrDisabled) { + t.Fatalf("err = %v, want ErrDisabled", err) + } +} + +// TestFetchAllNotServedAnywhereIsSkipped covers a host whose every candidate +// path answers 404, which is both how a repository with LFS switched off looks +// on a forge that does not distinguish the two and how a host with no LFS +// service at all looks. Either way the mirror layer's expected skip is the right +// outcome, not a repository-wide failure. +func TestFetchAllNotServedAnywhereIsSkipped(t *testing.T) { + oid, pointerText := pointerFor([]byte("content")) + + lfsServer := newFakeLFSServer(t, map[string][]byte{oid: []byte("content")}) + lfsServer.endpointFor(lfsServer.server.URL + "/somewhere/else") + repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText}) + + err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.plainRemoteURL(), "", "") + if !errors.Is(err, ErrDisabled) { + t.Fatalf("err = %v, want ErrDisabled when no candidate serves the API", err) + } +} + +// TestFetchAllConfiguredPathDecidesOverTheGuess covers the rule that only the +// path the remote itself configures decides: a failure there is reported even +// when a derived guess answers that LFS is off, because the mirror layer records +// that verdict as a successful skip and would hide a repository whose LFS content +// is missing. +func TestFetchAllConfiguredPathDecidesOverTheGuess(t *testing.T) { + oid, pointerText := pointerFor([]byte("content")) + + lfsServer := newFakeLFSServer(t, map[string][]byte{oid: []byte("content")}) + // The remote is suffix-less, so the suffix-less path is configured and the + // suffixed guess is the one that reports LFS off. + lfsServer.disabledFor(lfsServer.remoteURL()) + lfsServer.failedPaths = map[string]int{ + batchPathFor(lfsServer.plainRemoteURL()): http.StatusInternalServerError, + } + repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText}) + + err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.plainRemoteURL(), "", "") + if err == nil || errors.Is(err, ErrDisabled) { + t.Fatalf("err = %v, want the configured path's failure reported rather than a skip", err) + } +} + +// TestFetchAllGuessFailureDoesNotOverruleTheConfiguredPath covers the other +// direction: the configured path reports LFS off while the derived guess fails, +// so the configured path's verdict is what the repository is recorded as. +func TestFetchAllGuessFailureDoesNotOverruleTheConfiguredPath(t *testing.T) { + oid, pointerText := pointerFor([]byte("content")) + + lfsServer := newFakeLFSServer(t, map[string][]byte{oid: []byte("content")}) + // The remote is suffix-less, so the suffix-less path is the configured one, + // and the suffixed guess is the one that fails. + lfsServer.disabledFor(lfsServer.plainRemoteURL()) + lfsServer.failedPaths = map[string]int{ + batchPathFor(lfsServer.remoteURL()): http.StatusInternalServerError, + } + repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText}) + + err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.plainRemoteURL(), "", "") + if !errors.Is(err, ErrDisabled) { + t.Fatalf("err = %v, want the configured path's disabled verdict to be the expected skip", err) + } +} + +// TestFetchAllOffersCredentialsOnTheFirstRequest covers a forge that serves +// nothing to an anonymous request. The client presents its credentials straight +// away rather than waiting to be asked, so such a forge is read as one that +// wanted the credential rather than as a repository with LFS switched off. +func TestFetchAllOffersCredentialsOnTheFirstRequest(t *testing.T) { + content := []byte("content") + oid, pointerText := pointerFor(content) + + lfsServer := newFakeLFSServer(t, map[string][]byte{oid: content}) + lfsServer.requireAuth = true + repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText}) + + if err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "user", "token"); err != nil { + t.Fatalf("FetchAll failed: %v", err) + } + if _, err := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects", + oid[0:2], oid[2:4], oid)); err != nil { + t.Errorf("the object should be mirrored after the authenticated request: %v", err) + } + + // Without credentials there is nothing to present, so the 401 stands. + unauthenticated := newRepoWithLFS(t, map[string]string{"file.bin": pointerText}) + err := NewFetcher().FetchAll(context.Background(), unauthenticated, lfsServer.remoteURL(), "", "") + if err == nil { + t.Fatal("a 401 answered without credentials should be reported") + } + if errors.Is(err, ErrDisabled) { + t.Errorf("err = %v, want a failure rather than a repository recorded as LFS-free", err) + } +} + +// TestFetchAllReadsAForbiddenAnonymousRequestAsCredentialsNeeded covers the case +// a 401-only client gets wrong: a forge that refuses anonymous requests with 403 +// instead of inviting a credential with 401. A 403 is also how a repository with +// LFS switched off answers, so the credentials have to be offered on the first +// request — discovering the endpoint anonymously would record the repository as +// LFS-free and back it up without its LFS content. +func TestFetchAllReadsAForbiddenAnonymousRequestAsCredentialsNeeded(t *testing.T) { + content := []byte("content") + oid, pointerText := pointerFor(content) + + lfsServer := newFakeLFSServer(t, map[string][]byte{oid: content}) + lfsServer.forbidAnonymous = true + repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText}) + + if err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "user", "token"); err != nil { + t.Fatalf("FetchAll failed: %v", err) + } + if _, err := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects", + oid[0:2], oid[2:4], oid)); err != nil { + t.Errorf("the object should be mirrored rather than the repository recorded as LFS-free: %v", err) + } + + // The same server without credentials says nothing about the credential, so + // its refusal is read the only way it can be: LFS is not available here. + unauthenticated := newRepoWithLFS(t, map[string]string{"file.bin": pointerText}) + err := NewFetcher().FetchAll(context.Background(), unauthenticated, lfsServer.remoteURL(), "", "") + if !errors.Is(err, ErrDisabled) { + t.Errorf("err = %v, want a repository recorded as LFS-free when the forge refuses anonymously", err) + } +} + +// TestFetchAllAsksAgainWhenTheDownloadURLLapsed covers a server that schedules a +// URL which has already expired: the object needs a fresh one rather than a +// transfer that can only fail, and the server is the only party that can issue it. +func TestFetchAllAsksAgainWhenTheDownloadURLLapsed(t *testing.T) { + content := []byte("content") + oid, pointerText := pointerFor(content) + + lfsServer := newFakeLFSServer(t, map[string][]byte{oid: content}) + lfsServer.lapseFirstAction = true + repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText}) + + if err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", ""); err != nil { + t.Fatalf("FetchAll failed: %v", err) + } + if _, err := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects", + oid[0:2], oid[2:4], oid)); err != nil { + t.Errorf("the object should be mirrored after the URL was refreshed: %v", err) + } + + lfsServer.mu.Lock() + schedules := lfsServer.scheduled[oid] + lfsServer.mu.Unlock() + if schedules != 2 { + t.Errorf("object was scheduled %d times, want it asked about again after the lapsed URL", schedules) + } + if downloads := lfsServer.downloadCount(oid); downloads != 1 { + t.Errorf("download count = %d, want the object streamed exactly once", downloads) + } +} + +// TestDownloadObjectsSkipsCachedObjectsWhoseURLLapsed covers content that is +// already in the store being scheduled with an expired URL. The object needs +// nothing from the endpoint, so it is settled from the cache before the expiry +// can send it back for rescheduling, which is what made an already-mirrored +// repository fail over a fresh URL it never needed. +func TestDownloadObjectsSkipsCachedObjectsWhoseURLLapsed(t *testing.T) { + content := []byte("content") + oid, _ := pointerFor(content) + + store := t.TempDir() + cached := filepath.Join(store, oid[0:2], oid[2:4]) + if err := os.MkdirAll(cached, 0o755); err != nil { + t.Fatalf("prepare the store: %v", err) + } + if err := os.WriteFile(filepath.Join(cached, oid), content, 0o644); err != nil { + t.Fatalf("place the cached object: %v", err) + } + + expired, unavailable, failed := downloadObjects(context.Background(), newBatchClient(nil), store, + "http://127.0.0.1:1/repo.git/info/lfs", credentials{}, + []pointer{{oid: oid, size: int64(len(content))}}, + []batchResponseObject{{ + OID: oid, + Size: int64(len(content)), + Actions: map[string]*batchAction{"download": { + Href: "http://127.0.0.1:1/download/" + oid, + ExpiresAt: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339), + }}, + }}) + + if len(expired) != 0 { + t.Errorf("expired = %v, want cached content not sent back for a fresh URL", expired) + } + if len(unavailable) != 0 || len(failed) != 0 { + t.Errorf("unavailable = %v, failed = %v, want cached content left alone", unavailable, failed) + } +} + +// TestFetchAllNamesObjectsWhoseRefreshFailed covers an endpoint that only ever +// schedules lapsed URLs and then refuses the request for fresh ones. Each object +// has to be reported by name — a count tells a caller nothing about what is +// missing from the mirror — and the refusal is that record's reason rather than +// a second, chunk-level failure over objects already accounted for. +func TestFetchAllNamesObjectsWhoseRefreshFailed(t *testing.T) { + content := []byte("content") + oid, pointerText := pointerFor(content) + + lfsServer := newFakeLFSServer(t, map[string][]byte{oid: content}) + lfsServer.lapseEveryAction = true + repositoryPath := newRepoWithLFS(t, map[string]string{"file.bin": pointerText}) + + err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "") + if err == nil { + t.Fatal("an object that never arrived should be reported") + } + if !strings.Contains(err.Error(), oid) { + t.Errorf("err = %v, want it to name the object that could not be fetched", err) + } + if !errors.Is(err, errObjectUnavailable) { + t.Errorf("err = %v, want the object recorded as unserved", err) + } + if _, statErr := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects", + oid[0:2], oid[2:4], oid)); statErr == nil { + t.Error("the object should not be in the store, since it was never downloaded") + } +} + +// TestDownloadChunkReportsARefusedRefreshPerObject covers the endpoint scheduling +// a lapsed URL and then refusing outright to issue a fresh one. The refusal is +// the reason the object went unserved, so it is reported through that object — +// named — rather than as a further chunk-level failure over objects that were +// already accounted for. +func TestDownloadChunkReportsARefusedRefreshPerObject(t *testing.T) { + const endpoint = "http://127.0.0.1:1/repo.git/info/lfs" + oid, _ := pointerFor([]byte("content")) + + // downloadObjects settles the chunk's own objects from the response it is + // handed, so the first request this transport sees is the one asking for a + // fresh URL; that one is refused. + batches := 0 + client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + batches++ + return &http.Response{ + StatusCode: http.StatusInternalServerError, + Body: io.NopCloser(strings.NewReader("")), + Header: make(http.Header), + }, nil + })} + + fetcher := &Fetcher{client: newBatchClient(client)} + outcome, err := fetcher.downloadChunk(context.Background(), t.TempDir(), endpoint, credentials{}, + []pointer{{oid: oid, size: 7}}, + []batchResponseObject{ + {OID: oid, Size: 7, Actions: map[string]*batchAction{"download": { + Href: endpoint + "/download/" + oid, + ExpiresAt: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339), + }}}, + }) + + if batches != 1 { + t.Errorf("batch requests = %d, want one asking for a fresh URL", batches) + } + if err != nil { + t.Errorf("downloadChunk err = %v, want the refusal carried by the object rather than the chunk", err) + } + if outcome.unavailable != 1 { + t.Errorf("unavailable = %d, want the object recorded as unserved", outcome.unavailable) + } + if outcome.skipped == nil { + t.Fatal("the object's reason should be reported") + } + if !errors.Is(outcome.skipped, errObjectUnavailable) { + t.Errorf("reason = %v, want the object recorded as unserved", outcome.skipped) + } + if !strings.Contains(outcome.skipped.Error(), oid) { + t.Errorf("reason = %v, want it to name the object", outcome.skipped) + } + if !strings.Contains(outcome.skipped.Error(), fmt.Sprint(http.StatusInternalServerError)) { + t.Errorf("reason = %v, want the endpoint's own status as why no fresh URL arrived", outcome.skipped) + } +} + +// TestDownloadObjectsRefusesMalformedAndUnrequestedOIDs covers an endpoint whose +// response names objects the client never asked for. The OID becomes a path +// component and is sliced for the shard directories, so a malformed one has to be +// refused before either happens: a short OID would panic the slice and cost the +// whole backup, and a crafted one would name a path outside the store. +func TestDownloadObjectsRefusesMalformedAndUnrequestedOIDs(t *testing.T) { + wanted := strings.Repeat("a", 64) + unrelated := strings.Repeat("b", 64) + downloads := 0 + client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + downloads++ + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil + })} + + expired, unavailable, failed := downloadObjects(context.Background(), newBatchClient(client), t.TempDir(), + "http://127.0.0.1:1/repo.git/info/lfs", credentials{}, + []pointer{{oid: wanted, size: 7}}, + []batchResponseObject{ + {OID: "ab", Size: 7, Actions: map[string]*batchAction{"download": {Href: "http://127.0.0.1:1/download/ab"}}}, + {OID: "sha/../escape", Size: 7, Actions: map[string]*batchAction{"download": {Href: "http://127.0.0.1:1/download/escape"}}}, + {OID: unrelated, Size: 7, Actions: map[string]*batchAction{"download": {Href: "http://127.0.0.1:1/download/" + unrelated}}}, + }) + + if len(expired) != 0 || len(failed) != 0 { + t.Errorf("expired = %v, failed = %v, want the bad objects refused without a transfer", expired, failed) + } + if downloads != 0 { + t.Errorf("downloads = %d, want nothing fetched for objects the client did not ask for", downloads) + } + if len(unavailable) != 3 { + t.Fatalf("unavailable = %d, want every bad object reported rather than acted on", len(unavailable)) + } + for _, object := range unavailable { + if !errors.Is(object, errObjectUnavailable) { + t.Errorf("object error = %v, want it recorded as unserved", object) } } + if !errors.Is(unavailable[2], errObjectUnavailable) || !strings.Contains(unavailable[2].Error(), "not requested") { + t.Errorf("third error = %v, want the unrequested object reported as such", unavailable[2]) + } +} + +// TestDownloadObjectsAcceptsAnUppercaseOID covers an endpoint that echoes a valid +// digest in upper case. The OID is a path component and the store is keyed by it, +// so the response has to fold to the spelling the store uses: taken literally it +// would be written to a second directory, miss the cache it should hit, and be +// rejected as an object that was never requested. +func TestDownloadObjectsAcceptsAnUppercaseOID(t *testing.T) { + content := []byte("content") + oid, _ := pointerFor(content) + + store := t.TempDir() + cached := filepath.Join(store, oid[0:2], oid[2:4]) + if err := os.MkdirAll(cached, 0o755); err != nil { + t.Fatalf("prepare the store: %v", err) + } + if err := os.WriteFile(filepath.Join(cached, oid), content, 0o644); err != nil { + t.Fatalf("place the cached object: %v", err) + } + + downloads := 0 + client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + downloads++ + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil + })} + + expired, unavailable, failed := downloadObjects(context.Background(), newBatchClient(client), store, + "http://127.0.0.1:1/repo.git/info/lfs", credentials{}, + []pointer{{oid: oid, size: int64(len(content))}}, + []batchResponseObject{{ + OID: strings.ToUpper(oid), + Size: int64(len(content)), + Actions: map[string]*batchAction{"download": {Href: "http://127.0.0.1:1/download/" + oid}}, + }}) + + if len(expired) != 0 || len(unavailable) != 0 || len(failed) != 0 { + t.Errorf("expired = %v, unavailable = %v, failed = %v, want the uppercase digest accepted", expired, unavailable, failed) + } + if downloads != 0 { + t.Errorf("downloads = %d, want the cached object recognised rather than fetched again", downloads) + } +} + +// TestDownloadObjectsTrustsTheCacheWhenTheEndpointReportsNoSize covers an +// endpoint that omits the object's size. The length is then unstated rather than +// contradicted, so content the store already holds must not be re-downloaded — +// with a lapsed URL that would escalate into failing a backup whose LFS content +// is complete. +func TestDownloadObjectsTrustsTheCacheWhenTheEndpointReportsNoSize(t *testing.T) { + content := []byte("content") + oid, _ := pointerFor(content) + + store := t.TempDir() + cached := filepath.Join(store, oid[0:2], oid[2:4]) + if err := os.MkdirAll(cached, 0o755); err != nil { + t.Fatalf("prepare the store: %v", err) + } + if err := os.WriteFile(filepath.Join(cached, oid), content, 0o644); err != nil { + t.Fatalf("place the cached object: %v", err) + } + + expired, unavailable, failed := downloadObjects(context.Background(), newBatchClient(nil), store, + "http://127.0.0.1:1/repo.git/info/lfs", credentials{}, + []pointer{{oid: oid, size: int64(len(content))}}, + []batchResponseObject{{ + OID: oid, + Size: 0, + Actions: map[string]*batchAction{"download": { + Href: "http://127.0.0.1:1/download/" + oid, + ExpiresAt: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339), + }}, + }}) + + if len(expired) != 0 { + t.Errorf("expired = %v, want cached content not rescheduled over an unstated size", expired) + } + if len(unavailable) != 0 || len(failed) != 0 { + t.Errorf("unavailable = %v, failed = %v, want cached content left alone", unavailable, failed) + } + + // A size the endpoint does state has to be honoured, so a partial file is + // not mistaken for the object. + shortStore := t.TempDir() + if err := os.MkdirAll(filepath.Join(shortStore, oid[0:2], oid[2:4]), 0o755); err != nil { + t.Fatalf("prepare the store: %v", err) + } + if err := os.WriteFile(filepath.Join(shortStore, oid[0:2], oid[2:4], oid), content[:3], 0o644); err != nil { + t.Fatalf("place the truncated object: %v", err) + } + _, _, truncatedFailed := downloadObjects(context.Background(), newBatchClient(nil), shortStore, + "http://127.0.0.1:1/repo.git/info/lfs", credentials{}, + []pointer{{oid: oid, size: int64(len(content))}}, + []batchResponseObject{{ + OID: oid, + Size: int64(len(content)), + Actions: map[string]*batchAction{"download": { + Href: "http://127.0.0.1:1/download/" + oid, + }}, + }}) + if len(truncatedFailed) == 0 { + t.Error("a cached file the endpoint's size contradicts should fall through to a download") + } + + // The pointer file's size is authoritative, so a truncated file is not the + // object even when the endpoint states no size at all — the case that would + // otherwise let an interrupted copy be counted as mirrored. + _, _, unstatedFailed := downloadObjects(context.Background(), newBatchClient(nil), shortStore, + "http://127.0.0.1:1/repo.git/info/lfs", credentials{}, + []pointer{{oid: oid, size: int64(len(content))}}, + []batchResponseObject{{ + OID: oid, + Size: 0, + Actions: map[string]*batchAction{"download": { + Href: "http://127.0.0.1:1/download/" + oid, + }}, + }}) + if len(unstatedFailed) == 0 { + t.Error("a cached file the pointer's size contradicts should fall through to a download") + } + + // The resolved size has to travel with an object that is rescheduled, so the + // refresh request and the pass over its answer keep the pointer's length + // instead of falling back to an endpoint that stated none. + expiredPointers, _, _ := downloadObjects(context.Background(), newBatchClient(nil), shortStore, + "http://127.0.0.1:1/repo.git/info/lfs", credentials{}, + []pointer{{oid: oid, size: int64(len(content))}}, + []batchResponseObject{{ + OID: oid, + Size: 0, + Actions: map[string]*batchAction{"download": { + Href: "http://127.0.0.1:1/download/" + oid, + ExpiresAt: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339), + }}, + }}) + if len(expiredPointers) != 1 { + t.Fatalf("expired = %v, want the lapsed object rescheduled", expiredPointers) + } + if expiredPointers[0].size != int64(len(content)) { + t.Errorf("rescheduled size = %d, want the pointer's %d rather than the endpoint's unstated 0", + expiredPointers[0].size, len(content)) + } } func TestFetchAllUnauthorizedIsNotDisabled(t *testing.T) { @@ -319,21 +1068,723 @@ func TestFetchAllUnknownObjectFails(t *testing.T) { if err == nil || errors.Is(err, ErrDisabled) { t.Fatalf("unknown object should be a genuine error, got %v", err) } + if !errors.Is(err, errObjectUnavailable) { + t.Fatalf("err = %v, want it to report the object as unavailable", err) + } + if !strings.Contains(err.Error(), "could not be fetched") { + t.Fatalf("err = %v, want it to say how many objects were skipped", err) + } } -func TestFetchAllRejectsUnsafeLFSConfigOverrides(t *testing.T) { - oid, pointerText := pointerFor([]byte("content")) +// TestFetchAllFallsBackWhenBatchRejected covers forges that reject a batch +// request outright — an object limit, or a request they refuse to process. The +// pointers must be narrowed down so a single refused object cannot cost the +// repository's whole LFS mirror, while a refusal that hits every object is +// reported as the endpoint's failure. +func TestFetchAllFallsBackWhenBatchRejected(t *testing.T) { + available := []byte("available content") + availableOID, availablePointer := pointerFor(available) + refused := []byte("refused content") + refusedOID, refusedPointer := pointerFor(refused) + result := func(oids ...string) string { return strings.Join(oids, ",") } + cases := []struct { - name string - lfsURL string + name string + request func(batchRequest) int + wantErr string + wantBatches int + wantCached string }{ - {"off-host override", "https://evil.example.com/lfs"}, - {"non-http scheme", "ssh://git@evil.example.com/repo.git/lfs"}, - {"relative value", "/custom/lfs"}, + { + name: "server object limit", + request: func(request batchRequest) int { + if len(request.Objects) > 1 { + return http.StatusUnprocessableEntity + } + return http.StatusOK + }, + // Discovery answers the one-object probe, so the fetch runs: the + // chunk is refused once and then narrowed into two served singletons. + wantBatches: 4, + wantCached: result(availableOID, refusedOID), + }, + { + name: "one object rejected, one refused", + request: func(request batchRequest) int { + for _, object := range request.Objects { + if object.OID == refusedOID { + return http.StatusUnprocessableEntity + } + } + return http.StatusOK + }, + // The chunk splits once and the refused half's lone pointer is + // rejected on its own request. + wantErr: "unavailable", + wantBatches: 4, + wantCached: result(availableOID), + }, + { + name: "the endpoint refuses every batch", + request: func(batchRequest) int { + return http.StatusBadRequest + }, + // Discovery is answered, then every fetch request is refused, so the + // rejection outlives the split and is reported as the endpoint's. + wantErr: "rejected", + wantBatches: 4, + }, } + for _, c := range cases { t.Run(c.name, func(t *testing.T) { - lfsServer := newFakeLFSServer(t, map[string][]byte{oid: []byte("content")}) + lfsServer := newFakeLFSServer(t, map[string][]byte{ + availableOID: available, + refusedOID: refused, + }) + // Discovery is answered so the fetch runs; the rejection rules then + // decide what the fetch sees. + lfsServer.serveProbe = true + lfsServer.rejectWhen = c.request + repositoryPath := newRepoWithLFS(t, map[string]string{ + "big.bin": availablePointer, + "stale.bin": refusedPointer, + }) + + err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "") + if c.wantErr == "" { + if err != nil { + t.Fatalf("FetchAll = %v, want success", err) + } + } else if err == nil || !strings.Contains(err.Error(), c.wantErr) { + t.Fatalf("FetchAll = %v, want an error containing %q", err, c.wantErr) + } + + if calls := lfsServer.batchCallCount(); calls != c.wantBatches { + t.Errorf("batch calls = %d, want %d", calls, c.wantBatches) + } + for _, oid := range strings.Split(c.wantCached, ",") { + if oid == "" { + continue + } + path := filepath.Join(repositoryPath, ".git", "lfs", "objects", oid[0:2], oid[2:4], oid) + if _, err := os.Stat(path); err != nil { + t.Errorf("object %s should be mirrored: %v", shortOID(oid), err) + } + } + }) + } +} + +// TestFetchAllSkipsObjectsWithoutDownloadAction covers a successful batch +// response that schedules no action for one object: the object the client +// cannot act on must be reported, and the objects scheduled beside it must +// still reach the mirror (internal/lfs/batch.go turns the missing action into +// the unavailable sentinel that downloadChunk has to isolate). +func TestFetchAllSkipsObjectsWithoutDownloadAction(t *testing.T) { + first := []byte("first content") + firstOID, firstPointer := pointerFor(first) + second := []byte("second content") + secondOID, secondPointer := pointerFor(second) + third := []byte("third content") + thirdOID, thirdPointer := pointerFor(third) + + lfsServer := newFakeLFSServer(t, map[string][]byte{ + firstOID: first, + secondOID: second, + thirdOID: third, + }) + lfsServer.actionlessOIDs = []string{secondOID} + repositoryPath := newRepoWithLFS(t, map[string]string{ + "first.bin": firstPointer, + "second.bin": secondPointer, + "third.bin": thirdPointer, + }) + + err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "") + if err == nil { + t.Fatal("an object without a download action should be reported") + } + if !strings.Contains(err.Error(), "could not be fetched") { + t.Fatalf("err = %v, want it to say how many objects were skipped", err) + } + if !errors.Is(err, errObjectUnavailable) { + t.Fatalf("err = %v, want it to report the object as unavailable", err) + } + + for _, oid := range []string{firstOID, thirdOID} { + path := filepath.Join(repositoryPath, ".git", "lfs", "objects", oid[0:2], oid[2:4], oid) + if _, err := os.Stat(path); err != nil { + t.Errorf("object %s should be mirrored beside the actionless one: %v", shortOID(oid), err) + } + } + if path := filepath.Join(repositoryPath, ".git", "lfs", "objects", secondOID[0:2], secondOID[2:4], secondOID); func() bool { + _, err := os.Stat(path) + return err == nil + }() { + t.Error("the actionless object must not be stored") + } +} + +func TestFetchAllChunksLargePointerSets(t *testing.T) { + files := make(map[string]string, batchObjectLimit+2) + data := make(map[string][]byte, batchObjectLimit+2) + var lastOID string + for index := range batchObjectLimit + 2 { + content := []byte(fmt.Sprintf("object %d", index)) + oid, pointerText := pointerFor(content) + data[oid] = content + files[fmt.Sprintf("file-%03d.bin", index)] = pointerText + lastOID = oid + } + + lfsServer := newFakeLFSServer(t, data) + repositoryPath := newRepoWithLFS(t, files) + + if err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", ""); err != nil { + t.Fatalf("FetchAll failed: %v", err) + } + + // One discovery probe plus one request per chunk of batchObjectLimit. + wantCalls := 1 + (batchObjectLimit+2+batchObjectLimit-1)/batchObjectLimit + if calls := lfsServer.batchCallCount(); calls != wantCalls { + t.Errorf("batch calls = %d, want a probe plus one request per chunk of %d", calls, batchObjectLimit) + } + if _, err := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects", + lastOID[0:2], lastOID[2:4], lastOID)); err != nil { + t.Errorf("the last chunk should be mirrored too: %v", err) + } +} + +// TestFetchAllStopsAfterFailedDiscovery covers an endpoint whose failures are not +// about the request's shape — dropped connections, and the credentials and server +// statuses that mean the same thing. Discovery asks once and reports, so the fetch +// never starts rather than narrowing a chunk against an endpoint that is not +// answering. +func TestFetchAllStopsAfterFailedDiscovery(t *testing.T) { + files := make(map[string]string, batchObjectLimit*5) + for index := range batchObjectLimit * 5 { + _, pointerText := pointerFor([]byte(fmt.Sprintf("object %d", index))) + files[fmt.Sprintf("file-%04d.bin", index)] = pointerText + } + repositoryPath := newRepoWithLFS(t, files) + + t.Run("dropped connections", func(t *testing.T) { + var requests int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt64(&requests, 1) + hijacked, _, err := w.(http.Hijacker).Hijack() + if err != nil { + return + } + _ = hijacked.Close() + })) + t.Cleanup(server.Close) + + err := NewFetcher().FetchAll(context.Background(), repositoryPath, server.URL+"/repo.git", "", "") + if err == nil { + t.Fatal("an endpoint that never answers should be reported") + } + if errors.Is(err, errObjectUnavailable) { + t.Errorf("err = %v, want a transport failure rather than per-object unavailability", err) + } + // The remote carries its suffix, so there is one candidate to ask. + if got := atomic.LoadInt64(&requests); got != 1 { + t.Errorf("batch requests = %d, want a single discovery attempt", got) + } + }) + + for _, status := range []int{http.StatusUnauthorized, http.StatusTooManyRequests, http.StatusInternalServerError} { + t.Run(http.StatusText(status), func(t *testing.T) { + lfsServer := newFakeLFSServer(t, nil) + lfsServer.batchStatus = status + + err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "") + if err == nil || !strings.Contains(err.Error(), fmt.Sprint(status)) { + t.Fatalf("FetchAll = %v, want the endpoint's status", err) + } + if errors.Is(err, errObjectUnavailable) { + t.Errorf("err = %v, want the endpoint failure rather than per-object unavailability", err) + } + // No halving, no chunks: discovery answered for the whole repository. + if calls := lfsServer.batchCallCount(); calls != 1 { + t.Errorf("batch calls = %d, want a single discovery attempt", calls) + } + }) + } +} + +// TestFetchAllReportsSystematicRejection covers an endpoint that answers +// discovery and then rejects every batch as unprocessable, including +// single-object ones, so the rejection outlives every split. It must be reported +// as the endpoint's — not attributed to each object as unavailable — and the +// sweep must stop once a second chunk confirms it rather than narrowing its way +// through the whole repository. +func TestFetchAllReportsSystematicRejection(t *testing.T) { + const pointers = batchObjectLimit * 5 + files := make(map[string]string, pointers) + for index := range pointers { + _, pointerText := pointerFor([]byte(fmt.Sprintf("object %d", index))) + files[fmt.Sprintf("file-%04d.bin", index)] = pointerText + } + + lfsServer := newFakeLFSServer(t, nil) + // Discovery is answered, then every fetch request is refused. + lfsServer.batchStatus = http.StatusUnprocessableEntity + lfsServer.serveProbe = true + repositoryPath := newRepoWithLFS(t, files) + + err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "") + if err == nil || !strings.Contains(err.Error(), "rejected") { + t.Fatalf("FetchAll = %v, want the endpoint rejection reported", err) + } + if errors.Is(err, errObjectUnavailable) { + t.Errorf("err = %v, want an endpoint rejection rather than per-object unavailability", err) + } + if !strings.Contains(err.Error(), "422") { + t.Errorf("err = %v, want the endpoint's status", err) + } + + // Two chunks are narrowed — the first to establish the rejection and the + // second to confirm it is the endpoint's — and the rest are left alone. A + // per-pointer sweep of this repository would take five hundred requests. + if calls := lfsServer.batchCallCount(); calls > 450 { + t.Errorf("batch calls = %d, want the systematic rejection to stop the sweep", calls) + } +} + +// TestFetchAllNarrowsBothHalves covers a rejection that narrows onto a broken +// half: the sibling half must still be requested and downloaded, rather than +// being abandoned because the first half reported the endpoint's rejection. +func TestFetchAllNarrowsBothHalves(t *testing.T) { + bad := []byte("bad content") + badOID, badPointer := pointerFor(bad) + good := []byte("good content") + goodOID, goodPointer := pointerFor(good) + + lfsServer := newFakeLFSServer(t, map[string][]byte{ + badOID: bad, + goodOID: good, + }) + // Discovery is answered, so the fetch runs; every request naming the bad + // object is then refused, narrowing the chunk onto it. + lfsServer.serveProbe = true + lfsServer.rejectWhen = func(request batchRequest) int { + for _, object := range request.Objects { + if object.OID == badOID { + return http.StatusUnprocessableEntity + } + } + return http.StatusOK + } + repositoryPath := newRepoWithLFS(t, map[string]string{ + "bad.bin": badPointer, + "good.bin": goodPointer, + }) + + err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "") + if err == nil || !strings.Contains(err.Error(), "unavailable") { + t.Fatalf("FetchAll = %v, want the refused object reported", err) + } + if _, statErr := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects", + goodOID[0:2], goodOID[2:4], goodOID)); statErr != nil { + t.Errorf("the sibling half should be downloaded: %v", statErr) + } +} + +// TestFetchAllKeepsObjectsBesideAnAnsweredRejection covers a chunk whose halves +// disagree: one half's batch is refused down to a single object, while the other +// half's batch is answered with an object the endpoint will not serve. The +// answer proves the endpoint processes batches, so the refused object must not be +// mistaken for a broken endpoint — which would end the fetch and leave the +// answered half's siblings unattempted. +func TestFetchAllKeepsObjectsBesideAnAnsweredRejection(t *testing.T) { + held := []byte("held content") + heldOID, heldPointer := pointerFor(held) + phantom := []byte("phantom content") + phantomOID, phantomPointer := pointerFor(phantom) + refused := []byte("refused content") + refusedOID, refusedPointer := pointerFor(refused) + + lfsServer := newFakeLFSServer(t, map[string][]byte{ + heldOID: held, + phantomOID: phantom, + refusedOID: refused, + }) + lfsServer.refusedOIDs = []string{phantomOID} + lfsServer.serveProbe = true + lfsServer.rejectWhen = func(request batchRequest) int { + for _, object := range request.Objects { + if object.OID == refusedOID { + return http.StatusUnprocessableEntity + } + } + return http.StatusOK + } + repositoryPath := newRepoWithLFS(t, map[string]string{ + "held.bin": heldPointer, + "phantom.bin": phantomPointer, + "refused.bin": refusedPointer, + }) + + err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "") + if err == nil || !strings.Contains(err.Error(), "unavailable") { + t.Fatalf("FetchAll = %v, want the two unfetchable objects reported", err) + } + + for _, oid := range []string{heldOID} { + if _, statErr := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects", + oid[0:2], oid[2:4], oid)); statErr != nil { + t.Errorf("object %s should be mirrored: %v", shortOID(oid), statErr) + } + } +} + +// TestFetchAllKeepsSweepingAfterARefusedChunk covers a chunk the endpoint +// answers nothing for — every object refused, which alone looks like an endpoint +// rejecting every request — followed by a chunk it serves. The refused chunk +// must not end the sweep, or a chunk of stale pointers would cost the repository +// everything after it. +func TestFetchAllKeepsSweepingAfterARefusedChunk(t *testing.T) { + const staleInFirstChunk = 3 + files := make(map[string]string, batchObjectLimit*2) + data := make(map[string][]byte, batchObjectLimit*2) + var staleOIDs []string + for index := range batchObjectLimit * 2 { + content := []byte(fmt.Sprintf("object %d", index)) + oid, pointerText := pointerFor(content) + data[oid] = content + files[fmt.Sprintf("file-%04d.bin", index)] = pointerText + if index < staleInFirstChunk { + staleOIDs = append(staleOIDs, oid) + } + } + + lfsServer := newFakeLFSServer(t, data) + lfsServer.serveProbe = true + lfsServer.rejectWhen = func(request batchRequest) int { + for _, object := range request.Objects { + for _, stale := range staleOIDs { + if object.OID == stale { + return http.StatusUnprocessableEntity + } + } + } + return http.StatusOK + } + repositoryPath := newRepoWithLFS(t, files) + + err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "") + if err == nil || !strings.Contains(err.Error(), "could not be fetched") { + t.Fatalf("FetchAll = %v, want the refused objects summarised", err) + } + + // The chunk after the refused one is still fetched. + last := fmt.Sprintf("object %d", batchObjectLimit*2-1) + lastOID, _ := pointerFor([]byte(last)) + if _, statErr := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects", + lastOID[0:2], lastOID[2:4], lastOID)); statErr != nil { + t.Errorf("the chunk after a refused one should still be fetched: %v", statErr) + } + for _, oid := range staleOIDs { + if _, statErr := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects", + oid[0:2], oid[2:4], oid)); statErr == nil { + t.Errorf("refused object %s must not be stored", shortOID(oid)) + } + } +} + +// TestFetchAllKeepsObjectsBesideARefusedHalf covers a chunk whose first half is +// refused in full while its sibling is servable: the first half answers nothing +// at all, which looks like an endpoint-wide rejection, and concluding that from +// one half alone would abandon the sibling — and, one chunk later, the rest of +// the repository. +func TestFetchAllKeepsObjectsBesideARefusedHalf(t *testing.T) { + staleA := []byte("stale a") + staleAOID, staleAPointer := pointerFor(staleA) + staleB := []byte("stale b") + staleBOID, staleBPointer := pointerFor(staleB) + goodC := []byte("good c") + goodCOID, goodCPointer := pointerFor(goodC) + + lfsServer := newFakeLFSServer(t, map[string][]byte{ + staleAOID: staleA, + staleBOID: staleB, + goodCOID: goodC, + }) + // Requests naming either stale object are refused, so a half holding both + // narrows to rejections that the endpoint never answers. + lfsServer.rejectWhen = func(request batchRequest) int { + for _, object := range request.Objects { + if object.OID == staleAOID || object.OID == staleBOID { + return http.StatusUnprocessableEntity + } + } + return http.StatusOK + } + repositoryPath := newRepoWithLFS(t, map[string]string{ + "stale-a.bin": staleAPointer, + "stale-b.bin": staleBPointer, + "good-c.bin": goodCPointer, + }) + + err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "") + if err == nil || !strings.Contains(err.Error(), "unavailable") { + t.Fatalf("FetchAll = %v, want the refused objects reported", err) + } + if _, statErr := os.Stat(filepath.Join(repositoryPath, ".git", "lfs", "objects", + goodCOID[0:2], goodCOID[2:4], goodCOID)); statErr != nil { + t.Errorf("the servable sibling half should still be fetched: %v", statErr) + } +} + +// TestFetchAllReportsAnEndpointThatStopsPartway covers a service that answers +// discovery and then stops serving the fetch. Part of the repository is mirrored +// by then, so the failure must be reported rather than recorded as a clean skip: +// that would present a half-finished mirror as complete. +func TestFetchAllReportsAnEndpointThatStopsPartway(t *testing.T) { + const pointers = batchObjectLimit * 2 + files := make(map[string]string, pointers) + data := make(map[string][]byte, pointers) + for index := range pointers { + content := []byte(fmt.Sprintf("object %d", index)) + oid, pointerText := pointerFor(content) + data[oid] = content + files[fmt.Sprintf("file-%04d.bin", index)] = pointerText + } + + lfsServer := newFakeLFSServer(t, data) + // Discovery is answered; the fetch that follows looks like a host with + // nothing mounted at the path any more. + lfsServer.serveProbe = true + lfsServer.fetchStatus = http.StatusNotFound + repositoryPath := newRepoWithLFS(t, files) + + err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "") + if err == nil { + t.Fatal("an endpoint that stops serving the fetch should be reported") + } + // ErrDisabled is how the mirror layer recognises an expected skip, so this + // failure must not match it through the chain. + if errors.Is(err, ErrDisabled) || errors.Is(err, ErrNoEndpoint) { + t.Fatalf("err = %v, want a failure rather than a clean skip", err) + } +} + +func TestCollectPointersHandlesWindowsIllegalNames(t *testing.T) { + dir := t.TempDir() + repository, err := git.PlainInit(dir, false) + if err != nil { + t.Fatal(err) + } + worktree, err := repository.Worktree() + if err != nil { + t.Fatal(err) + } + + content := []byte("weights") + oid, pointerText := pointerFor(content) + if err := os.MkdirAll(filepath.Join(dir, "nested"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "nested", "weights.bin"), []byte(pointerText), 0o644); err != nil { + t.Fatal(err) + } + if _, err := worktree.Add("nested/weights.bin"); err != nil { + t.Fatal(err) + } + if _, err := worktree.Commit("add pointer", &git.CommitOptions{ + Author: &object.Signature{Name: "test", Email: "test@example.com", When: time.Now()}, + }); err != nil { + t.Fatal(err) + } + + head, err := repository.Head() + if err != nil { + t.Fatal(err) + } + commit, err := repository.CommitObject(head.Hash()) + if err != nil { + t.Fatal(err) + } + root, err := repository.TreeObject(commit.TreeHash) + if err != nil { + t.Fatal(err) + } + var nested object.TreeEntry + for _, entry := range root.Entries { + if entry.Name == "nested" { + nested = entry + } + } + if nested.Hash.IsZero() { + t.Fatal("nested subtree not found") + } + + // An empty subtree the hostile tree names twice, so the deduplication is + // exercised without competing with the commit's own trees for the visit. + empty := &object.Tree{} + emptyObject := repository.Storer.NewEncodedObject() + if err := empty.Encode(emptyObject); err != nil { + t.Fatal(err) + } + emptyHash, err := repository.Storer.SetEncodedObject(emptyObject) + if err != nil { + t.Fatal(err) + } + + hostile := &object.Tree{Entries: []object.TreeEntry{ + {Name: "empty-again", Hash: emptyHash, Mode: filemode.Dir}, + {Name: "empty", Hash: emptyHash, Mode: filemode.Dir}, + {Name: `src\windows.cpp`, Hash: plumbing.NewHash(strings.Repeat("a", 40)), Mode: filemode.Regular}, + }} + encoded := repository.Storer.NewEncodedObject() + if err := hostile.Encode(encoded); err != nil { + t.Fatal(err) + } + hostileHash, err := repository.Storer.SetEncodedObject(encoded) + if err != nil { + t.Fatal(err) + } + + // A ref points at a commit, and the scan reaches trees through commits, so + // the hostile tree is committed before the ref names it. + signature := object.Signature{Name: "test", Email: "test@example.com", When: time.Now()} + hostileCommit := &object.Commit{ + Author: signature, + Committer: signature, + Message: "hostile tree", + TreeHash: hostileHash, + } + commitObject := repository.Storer.NewEncodedObject() + if err := hostileCommit.Encode(commitObject); err != nil { + t.Fatal(err) + } + hostileCommitHash, err := repository.Storer.SetEncodedObject(commitObject) + if err != nil { + t.Fatal(err) + } + + headRef := plumbing.NewHashReference(plumbing.NewBranchReferenceName("hostile"), hostileCommitHash) + if err := repository.Storer.SetReference(headRef); err != nil { + t.Fatal(err) + } + + scan, err := collectPointers(context.Background(), repository) + if err != nil { + t.Fatalf("collectPointers failed on a tree with host-specific names: %v", err) + } + if len(scan.pointers) != 1 || scan.pointers[0].oid != oid { + t.Fatalf("pointers = %+v, want the single pointer %s", scan.pointers, oid) + } + // The hostile entry names a blob that was never stored, so the scan reports + // it rather than quietly reading a smaller set. The empty subtree behind the + // two directory entries is visited once, so it is not reported as skipped. + if len(scan.skipped) != 1 || scan.skipped[0].name != `src\windows.cpp` { + t.Errorf("skipped = %+v, want just the entry whose blob is missing", scan.skipped) + } +} + +// TestCollectPointersReportsUnreadableSubtrees covers a subtree whose object is +// missing — a partial fetch, say. Its pointers cannot be fetched, so the scan +// has to name them as skipped rather than reporting a smaller pointer set as if +// it were the whole truth. +func TestCollectPointersReportsUnreadableSubtrees(t *testing.T) { + const missingName = "missing" + repositoryPath := addMissingSubtreeCommit(t, missingName) + + repository, err := git.PlainOpen(repositoryPath) + if err != nil { + t.Fatal(err) + } + + scan, err := collectPointers(context.Background(), repository) + if err != nil { + t.Fatalf("a missing subtree should not fail the scan: %v", err) + } + if len(scan.skipped) != 1 || scan.skipped[0].name != missingName { + t.Fatalf("skipped = %+v, want the unreadable subtree named", scan.skipped) + } + if scan.skipped[0].err == nil { + t.Error("the skip should carry the reason it was skipped") + } +} + +// addMissingSubtreeCommit builds a repository whose history names a subtree +// object that was never stored, so the pointer scan has something it cannot +// read, and returns its path. +func addMissingSubtreeCommit(t *testing.T, name string) string { + t.Helper() + + repositoryPath := newRepoWithLFS(t, map[string]string{"README.md": "plain repo"}) + repository, err := git.PlainOpen(repositoryPath) + if err != nil { + t.Fatal(err) + } + + tree := &object.Tree{Entries: []object.TreeEntry{ + {Name: name, Hash: plumbing.NewHash(strings.Repeat("b", 40)), Mode: filemode.Dir}, + }} + treeObject := repository.Storer.NewEncodedObject() + if err := tree.Encode(treeObject); err != nil { + t.Fatal(err) + } + treeHash, err := repository.Storer.SetEncodedObject(treeObject) + if err != nil { + t.Fatal(err) + } + + signature := object.Signature{Name: "test", Email: "test@example.com", When: time.Now()} + commit := &object.Commit{ + Author: signature, + Committer: signature, + Message: "point at a missing subtree", + TreeHash: treeHash, + } + commitObject := repository.Storer.NewEncodedObject() + if err := commit.Encode(commitObject); err != nil { + t.Fatal(err) + } + commitHash, err := repository.Storer.SetEncodedObject(commitObject) + if err != nil { + t.Fatal(err) + } + if err := repository.Storer.SetReference(plumbing.NewHashReference(plumbing.NewBranchReferenceName("hostile"), commitHash)); err != nil { + t.Fatal(err) + } + return repositoryPath +} + +// TestFetchAllFailsWhenTheScanCouldNotReadEverything covers what the mirror +// layer sees when part of the repository cannot be read: the pointers behind it +// cannot be fetched either, so the fetch reports rather than returning success +// for a mirror that is quietly incomplete. +func TestFetchAllFailsWhenTheScanCouldNotReadEverything(t *testing.T) { + lfsServer := newFakeLFSServer(t, nil) + repositoryPath := addMissingSubtreeCommit(t, "missing") + + err := NewFetcher().FetchAll(context.Background(), repositoryPath, lfsServer.remoteURL(), "", "") + if err == nil || !strings.Contains(err.Error(), "could not be read") { + t.Fatalf("FetchAll = %v, want the unreadable entry reported", err) + } + if calls := lfsServer.batchCallCount(); calls != 0 { + t.Errorf("batch calls = %d, want no endpoint work for a scan that cannot complete", calls) + } +} + +func TestFetchAllRejectsUnsafeLFSConfigOverrides(t *testing.T) { + oid, pointerText := pointerFor([]byte("content")) + cases := []struct { + name string + lfsURL string + }{ + {"off-host override", "https://evil.example.com/lfs"}, + {"non-http scheme", "ssh://git@evil.example.com/repo.git/lfs"}, + {"relative value", "/custom/lfs"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + lfsServer := newFakeLFSServer(t, map[string][]byte{oid: []byte("content")}) repositoryPath := newRepoWithLFS(t, map[string]string{ ".lfsconfig": "[lfs]\n\turl = " + c.lfsURL + "\n", "file.bin": pointerText, @@ -379,7 +1830,74 @@ func TestCanonicalHostStripsSchemeDefaultPorts(t *testing.T) { } } -func TestResolveEndpointOverrideHostAndScheme(t *testing.T) { +// TestSuffixedEndpointAddsGitSuffix pins the endpoint a forge expects: GitHub +// and GitLab answer a suffix-less /info/lfs with 422, so a configured remote +// without .git still has to request the suffixed path. Everything else the +// remote URL carries has to survive the rewrite. +func TestSuffixedEndpointAddsGitSuffix(t *testing.T) { + cases := []struct { + remoteURL string + want string + }{ + {"https://github.com/owner/repo", "https://github.com/owner/repo.git/info/lfs"}, + {"https://github.com/owner/repo.git", "https://github.com/owner/repo.git/info/lfs"}, + {"https://github.com/owner/repo.GIT", "https://github.com/owner/repo.GIT/info/lfs"}, + {"https://github.com/owner/repo/", "https://github.com/owner/repo.git/info/lfs"}, + {"https://git.example.com:8443/group/sub/repo", "https://git.example.com:8443/group/sub/repo.git/info/lfs"}, + {"https://user:secret@git.example.com/owner/repo", "https://user:secret@git.example.com/owner/repo.git/info/lfs"}, + {"https://git.example.com", "https://git.example.com/info/lfs"}, + {"https://git.example.com/", "https://git.example.com/info/lfs"}, + {"https://git.example.com/owner/repo?", "https://git.example.com/owner/repo.git/info/lfs"}, + {"https://git.example.com/owner/repo#frag", "https://git.example.com/owner/repo.git/info/lfs"}, + {"https://git.example.com/owner/re%2Fpo", "https://git.example.com/owner/re%2Fpo.git/info/lfs"}, + {"https://git.example.com/owner/re po", "https://git.example.com/owner/re%20po.git/info/lfs"}, + } + + for _, c := range cases { + t.Run(redactedURL(c.remoteURL), func(t *testing.T) { + parsed, err := url.Parse(c.remoteURL) + if err != nil { + t.Fatal(err) + } + if got := suffixedEndpoint(parsed); got != c.want { + t.Errorf("suffixedEndpoint(%q) = %q, want %q", redactedURL(c.remoteURL), redactedURL(got), redactedURL(c.want)) + } + }) + } +} + +// TestEndpointCandidatesCollapseDuplicates covers remotes that reach the +// LFS API at one path whichever candidate is derived — a remote that already +// carries the repository suffix, and one with no path at all. Probing the same +// URL twice would repeat every request of a whole repository for nothing. +func TestEndpointCandidatesCollapseDuplicates(t *testing.T) { + for _, remoteURL := range []string{ + "https://git.example.com/owner/repo.git", + "https://git.example.com", + } { + t.Run(redactedURL(remoteURL), func(t *testing.T) { + repositoryPath := newRepoWithLFS(t, map[string]string{"README.md": "plain repo"}) + repository, err := git.PlainOpen(repositoryPath) + if err != nil { + t.Fatal(err) + } + + parsed, parseErr := parseRemoteURL(remoteURL) + if parseErr != nil { + t.Fatal(parseErr) + } + endpoints, err := endpointCandidates(repository, parsed) + if err != nil { + t.Fatalf("resolveEndpoints failed: %v", err) + } + if len(endpoints) != 1 { + t.Errorf("endpoints = %q, want one candidate for a remote that has only one path", redactedURL(strings.Join(candidateURLs(endpoints), ","))) + } + }) + } +} + +func TestEndpointCandidatesOverrideHostAndScheme(t *testing.T) { oid, pointerText := pointerFor([]byte("content")) cases := []struct { name string @@ -394,6 +1912,12 @@ func TestResolveEndpointOverrideHostAndScheme(t *testing.T) { remoteURL: "https://example.com/repo.git", wantAddress: "https://example.com/custom/lfs", }, + { + name: "endpoint rooted at the host", + lfsURL: "https://example.com:443/", + remoteURL: "https://example.com/repo.git", + wantAddress: "https://example.com", + }, { name: "http override downgrades the https remote", lfsURL: "http://example.com/custom/lfs", @@ -426,22 +1950,35 @@ func TestResolveEndpointOverrideHostAndScheme(t *testing.T) { t.Fatal(err) } - address, err := resolveEndpoint(repository, c.remoteURL) + parsed, parseErr := parseRemoteURL(c.remoteURL) + if parseErr != nil { + t.Fatal(parseErr) + } + endpoints, err := endpointCandidates(repository, parsed) if c.wantReject { if err == nil { - t.Fatalf("resolveEndpoint = %q, want rejection", address) + t.Fatalf("resolveEndpoints = %q, want rejection", redactedURL(strings.Join(candidateURLs(endpoints), ","))) } return } if err != nil { - t.Fatalf("resolveEndpoint failed: %v", err) + t.Fatalf("resolveEndpoints failed: %v", err) } - if address != c.wantAddress { - t.Errorf("endpoint = %q, want %q", address, c.wantAddress) + if len(endpoints) != 1 || endpoints[0].url != c.wantAddress { + t.Errorf("endpoints = %q, want just %q", redactedURL(strings.Join(candidateURLs(endpoints), ",")), redactedURL(c.wantAddress)) } if lfsServer.batchCallCount() != 0 { - t.Error("resolveEndpoint must not contact any endpoint") + t.Error("resolveEndpoints must not contact any endpoint") } }) } } + +// candidateURLs renders candidates for a test message. +func candidateURLs(candidates []endpointCandidate) []string { + urls := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + urls = append(urls, candidate.url) + } + return urls +} diff --git a/internal/lfs/scan.go b/internal/lfs/scan.go index d9dab76..e9da170 100644 --- a/internal/lfs/scan.go +++ b/internal/lfs/scan.go @@ -10,6 +10,7 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/filemode" "github.com/go-git/go-git/v5/plumbing/object" ) @@ -19,6 +20,21 @@ type pointer struct { size int64 } +// skippedSubtree records a subtree the scan could not read, so its pointers are +// known to be missing from the result rather than silently absent. +type skippedSubtree struct { + name string + hash plumbing.Hash + err error +} + +// scanResult is what one scan of a repository found: the pointers it collected +// and the subtrees it could not read. +type scanResult struct { + pointers []pointer + skipped []skippedSubtree +} + // pointerVersionValue is the version scheme URI on the first line of every // LFS pointer file. const pointerVersionValue = "https://git-lfs.github.com/spec/v1" @@ -31,12 +47,13 @@ const pointerMaxBytes = 4 * 1024 // inspected once. Commit history is traversed explicitly (rather than per-ref // iterators) so a repository with many refs costs one pass over its history, // not one per ref. -func collectPointers(ctx context.Context, repository *git.Repository) ([]pointer, error) { +func collectPointers(ctx context.Context, repository *git.Repository) (scanResult, error) { seenCommits := make(map[plumbing.Hash]struct{}) seenTrees := make(map[plumbing.Hash]bool) seenBlobs := make(map[plumbing.Hash]struct{}) var pointers []pointer + var skipped []skippedSubtree seenPointers := make(map[string]struct{}) enqueue := func(queue []plumbing.Hash, hash plumbing.Hash) []plumbing.Hash { @@ -52,7 +69,7 @@ func collectPointers(ctx context.Context, repository *git.Repository) ([]pointer queue := make([]plumbing.Hash, 0, 64) refs, err := repository.References() if err != nil { - return nil, err + return scanResult{}, err } err = refs.ForEach(func(ref *plumbing.Reference) error { if ref.Type() == plumbing.SymbolicReference || ref.Name().IsRemote() { @@ -62,12 +79,12 @@ func collectPointers(ctx context.Context, repository *git.Repository) ([]pointer return ctx.Err() }) if err != nil { - return nil, err + return scanResult{}, err } for len(queue) > 0 { if err := ctx.Err(); err != nil { - return nil, err + return scanResult{}, err } hash := queue[len(queue)-1] @@ -85,8 +102,8 @@ func collectPointers(ctx context.Context, repository *git.Repository) ([]pointer continue } - if err := scanTree(repository, commit.TreeHash, seenTrees, seenBlobs, seenPointers, &pointers); err != nil { - return nil, err + if err := scanTree(repository, commit.TreeHash, seenTrees, seenBlobs, seenPointers, &skipped, &pointers); err != nil { + return scanResult{}, err } for _, parent := range commit.ParentHashes { @@ -94,7 +111,7 @@ func collectPointers(ctx context.Context, repository *git.Repository) ([]pointer } } - return pointers, nil + return scanResult{pointers: pointers, skipped: skipped}, nil } // peelToCommitHash resolves a ref tip hash to the commit it designates, @@ -119,11 +136,24 @@ func peelToCommitHash(repository *git.Repository, hash plumbing.Hash) plumbing.H } } +// scanTree collects pointers from every blob under treeHash, descending into +// subtrees iteratively. +// +// Subtrees are walked from their tree objects directly rather than through +// object.TreeWalker: that walker validates each entry against the rules for +// materialising a working tree, so a repository containing a path that is +// illegal on the host — a backslash in an entry name on Windows, for instance — +// would abort the whole backup even though the mirror clones and uploads +// perfectly well. A tree object only ever holds entry names and subtree hashes, +// so enumerating them needs no path handling at all. seenTrees bounds the walk +// for repositories whose history repeats or self-references a tree, and makes +// the cross-ref scan visit each tree once (see collectPointers). func scanTree( repository *git.Repository, treeHash plumbing.Hash, seenTrees map[plumbing.Hash]bool, seenBlobs map[plumbing.Hash]struct{}, seenPointers map[string]struct{}, + skipped *[]skippedSubtree, pointers *[]pointer, ) error { tree, err := repository.TreeObject(treeHash) @@ -131,52 +161,72 @@ func scanTree( return fmt.Errorf("read tree: %w", err) } - walker := object.NewTreeWalker(tree, true, seenTrees) - defer walker.Close() + pending := []*object.Tree{tree} + seenTrees[treeHash] = true - for { - name, entry, err := walker.Next() - if err == io.EOF { - return nil - } - if err != nil { - return fmt.Errorf("walk tree: %w", err) - } - _ = name + for len(pending) > 0 { + current := pending[len(pending)-1] + pending = pending[:len(pending)-1] - if !entry.Mode.IsFile() { - continue - } - if _, seen := seenBlobs[entry.Hash]; seen { - continue - } - seenBlobs[entry.Hash] = struct{}{} + for _, entry := range current.Entries { + if entry.Mode == filemode.Dir { + if seenTrees[entry.Hash] { + continue + } + seenTrees[entry.Hash] = true - blob, err := repository.BlobObject(entry.Hash) - if err != nil { - continue - } - if blob.Size > pointerMaxBytes { - continue - } + subtree, err := repository.TreeObject(entry.Hash) + if err != nil { + // A subtree missing from a partial fetch costs its + // pointers but leaves the rest of the scan intact; record + // it so the loss is reported rather than silent. + *skipped = append(*skipped, skippedSubtree{name: entry.Name, hash: entry.Hash, err: err}) + continue + } + pending = append(pending, subtree) + continue + } + if !entry.Mode.IsFile() { + continue + } + if _, seen := seenBlobs[entry.Hash]; seen { + continue + } + seenBlobs[entry.Hash] = struct{}{} - reader, err := blob.Reader() - if err != nil { - continue - } - content, err := io.ReadAll(io.LimitReader(reader, pointerMaxBytes)) - _ = reader.Close() - if err != nil { - continue - } + blob, err := repository.BlobObject(entry.Hash) + if err != nil { + // A blob missing from a partial fetch costs whatever pointer it + // held; record it so the loss is reported rather than silent. + *skipped = append(*skipped, skippedSubtree{name: entry.Name, hash: entry.Hash, err: err}) + continue + } + if blob.Size > pointerMaxBytes { + continue + } - if parsed, ok := parsePointer(content); ok { - if _, duplicate := seenPointers[parsed.oid]; !duplicate { - seenPointers[parsed.oid] = struct{}{} - *pointers = append(*pointers, parsed) + reader, err := blob.Reader() + if err != nil { + *skipped = append(*skipped, skippedSubtree{name: entry.Name, hash: entry.Hash, err: err}) + continue + } + content, err := io.ReadAll(io.LimitReader(reader, pointerMaxBytes)) + _ = reader.Close() + if err != nil { + *skipped = append(*skipped, skippedSubtree{name: entry.Name, hash: entry.Hash, err: err}) + continue + } + + if parsed, ok := parsePointer(content); ok { + if _, duplicate := seenPointers[parsed.oid]; !duplicate { + seenPointers[parsed.oid] = struct{}{} + *pointers = append(*pointers, parsed) + } } } } + + return nil } // parsePointer parses an LFS pointer file: the version line followed by