fix: harden dag traversal and CAR responses

CHANGELOG.md

@@ -16,8 +16,12 @@ The following emojis are used to highlight certain changes:
 
 ### Added
 
+- `gateway`: added `WithMaxTraversalDepth`, bounding how deep `BlocksBackend` descends into a DAG while serving CAR responses. Traversal keeps per-level state, so its cost grows with depth. On by default at `DefaultMaxTraversalDepth` (1024), well above anything UnixFS produces: a file reaches terabytes by depth 4, and HAMT adds about 4 levels per million directory entries. Pass a positive value to set your own limit, or `WithMaxTraversalDepth(0)` to remove it entirely. [#1197](https://github.com/ipfs/boxo/pull/1197)
+
 ### Changed
 
+- `gateway`: a CAR response that fails partway through now ends with `[Gateway Error: CAR stream truncated, response is incomplete]`, the same approach `withRetrievalTimeout` already uses when it cuts a response short. `X-Stream-Error` is only set once the body is streaming, so it rarely reaches the client, and a truncated CAR was otherwise indistinguishable from a complete one. The marker makes the trailing bytes invalid CAR, so a reader stops with an error instead of accepting a short DAG. Mostly this is a quality of life improvement for operators: gateways usually sit behind reverse proxies and third-party CDNs, and when a response arrives short it is hard to tell which hop dropped it. Now the response says so itself. [#1197](https://github.com/ipfs/boxo/pull/1197)
+
 ### Removed
 
 ### Fixed

examples/go.mod

@@ -61,7 +61,7 @@ require (
 	github.com/ipfs/go-log/v2 v2.9.2 // indirect
 	github.com/ipfs/go-metrics-interface v0.3.0 // indirect
 	github.com/ipfs/go-peertaskqueue v0.8.3 // indirect
-	github.com/ipfs/go-unixfsnode v1.10.5 // indirect
+	github.com/ipfs/go-unixfsnode v1.10.6 // indirect
 	github.com/ipld/go-codec-dagpb v1.7.0 // indirect
 	github.com/ipld/go-ipld-prime v0.24.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect

examples/go.sum

@@ -141,8 +141,8 @@ github.com/ipfs/go-peertaskqueue v0.8.3 h1:tBPpGJy+A92RqtRFq5amJn0Uuj8Pw8tXi0X3e
 github.com/ipfs/go-peertaskqueue v0.8.3/go.mod h1:OqVync4kPOcXEGdj/LKvox9DCB5mkSBeXsPczCxLtYA=
 github.com/ipfs/go-test v0.4.1 h1:n6uNSakIgpTQIRorqNg2O02aMIFDLQk2z4rBfrlD3Uw=
 github.com/ipfs/go-test v0.4.1/go.mod h1:QmvVBf9kClNtRuFow4DASq03eFvjKla4Fy/UAkeeLO8=
-github.com/ipfs/go-unixfsnode v1.10.5 h1:V34JV7fM90y+2ZUef7ToShjmwAZ8oo1yP7zrpWzC5L4=
-github.com/ipfs/go-unixfsnode v1.10.5/go.mod h1:u/9Ukl+XYpfKTMu+NXQqxbzAJVTwSCoyTYBGgE+JdSE=
+github.com/ipfs/go-unixfsnode v1.10.6 h1:0rnKD4azJad4cT8t6wITqNl9Q0GTjB+YRBfMQ39C7Sw=
+github.com/ipfs/go-unixfsnode v1.10.6/go.mod h1:u/9Ukl+XYpfKTMu+NXQqxbzAJVTwSCoyTYBGgE+JdSE=
 github.com/ipld/go-car/v2 v2.17.0 h1:zgjSxf/lQNYcQPX08cvb5rSdEY8sv5OOnQIsZhZMPx4=
 github.com/ipld/go-car/v2 v2.17.0/go.mod h1:/4HY8tFZ1q42Mw54ILLPQfjkUqMJxFKqY1yMDKHlYko=
 github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0=

gateway/backend.go

@@ -25,6 +25,9 @@ type backendOptions struct {
 	// Only used by [BlocksBackend]:
 	r resolver.Resolver
 
+	// Only used by [BlocksBackend]:
+	maxTraversalDepth int
+
 	// Only used by [CarBackend]:
 	promRegistry    prometheus.Registerer
 	getBlockTimeout time.Duration
@@ -64,6 +67,26 @@ func WithPrometheusRegistry(reg prometheus.Registerer) BackendOption {
 	}
 }
 
+// DefaultMaxTraversalDepth is how deep a DAG traversal will descend before it
+// gives up. Traversal keeps per-level state, so its cost grows with depth. The
+// default is far above anything UnixFS produces: a file reaches terabytes by
+// depth 4, HAMT directories add about 4 levels per million entries, and
+// directory nesting follows the source tree.
+const DefaultMaxTraversalDepth = 1024
+
+// WithMaxTraversalDepth sets how deep [BlocksBackend] will descend into a DAG
+// before returning an error. By default, [DefaultMaxTraversalDepth] is used.
+// Pass 0 to remove the limit.
+func WithMaxTraversalDepth(depth int) BackendOption {
+	return func(opts *backendOptions) error {
+		if depth < 0 {
+			return fmt.Errorf("max traversal depth must not be negative; got %d", depth)
+		}
+		opts.maxTraversalDepth = depth
+		return nil
+	}
+}
+
 const DefaultGetBlockTimeout = time.Second * 60
 
 // WithGetBlockTimeout sets a custom timeout when getting blocks from the

gateway/backend_blocks.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"runtime/debug"
 	"strings"
 
 	"github.com/ipfs/boxo/blockservice"
@@ -47,17 +48,18 @@ import (
 // BlocksBackend is an [IPFSBackend] implementation based on a [blockservice.BlockService].
 type BlocksBackend struct {
 	baseBackend
-	blockStore   blockstore.Blockstore
-	blockService blockservice.BlockService
-	dagService   format.DAGService
-	resolver     resolver.Resolver
+	blockStore        blockstore.Blockstore
+	blockService      blockservice.BlockService
+	dagService        format.DAGService
+	resolver          resolver.Resolver
+	maxTraversalDepth int
 }
 
 var _ IPFSBackend = (*BlocksBackend)(nil)
 
 // NewBlocksBackend creates a new [BlocksBackend] backed by a [blockservice.BlockService].
 func NewBlocksBackend(blockService blockservice.BlockService, opts ...BackendOption) (*BlocksBackend, error) {
-	var compiledOptions backendOptions
+	compiledOptions := backendOptions{maxTraversalDepth: DefaultMaxTraversalDepth}
 	for _, o := range opts {
 		if err := o(&compiledOptions); err != nil {
 			return nil, err
@@ -84,11 +86,12 @@ func NewBlocksBackend(blockService blockservice.BlockService, opts ...BackendOpt
 	}
 
 	return &BlocksBackend{
-		baseBackend:  baseBackend,
-		blockStore:   blockService.Blockstore(),
-		blockService: blockService,
-		dagService:   dagService,
-		resolver:     r,
+		baseBackend:       baseBackend,
+		blockStore:        blockService.Blockstore(),
+		blockService:      blockService,
+		dagService:        dagService,
+		resolver:          r,
+		maxTraversalDepth: compiledOptions.maxTraversalDepth,
 	}, nil
 }
 
@@ -415,7 +418,7 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p path.ImmutablePath, param
 		}
 
 		// Setup the UnixFS resolver.
-		f := newNodeGetterFetcherSingleUseFactory(ctx, blockGetter)
+		f := newNodeGetterFetcherSingleUseFactory(ctx, blockGetter, bb.maxTraversalDepth)
 		pathResolver := resolver.NewBasicResolver(f)
 		_, _, err = pathResolver.ResolveToLastNode(ctx, p)
 
@@ -438,6 +441,18 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p path.ImmutablePath, param
 
 	r, w := io.Pipe()
 	go func() {
+		// Traversal decodes blocks with whatever codec their CID names, so it
+		// runs third-party code this package does not control. The goroutine is
+		// detached from the request, and a panic on it would end the process
+		// rather than the response, so keep it contained here.
+		defer func() {
+			if rec := recover(); rec != nil {
+				log.Errorf("recovered from panic during CAR traversal of %s: %v\n%s", p, rec, debug.Stack())
+				// io.PipeWriter.CloseWithError always returns nil.
+				_ = w.CloseWithError(errors.New("internal error during CAR traversal"))
+			}
+		}()
+
 		cw, err := storage.NewWritable(
 			w,
 			[]cid.Cid{pathMetadata.LastSegment.RootCid()},
@@ -458,12 +473,12 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p path.ImmutablePath, param
 		}
 
 		// Setup the UnixFS resolver.
-		f := newNodeGetterFetcherSingleUseFactory(ctx, blockGetter)
+		f := newNodeGetterFetcherSingleUseFactory(ctx, blockGetter, bb.maxTraversalDepth)
 		pathResolver := resolver.NewBasicResolver(f)
 
 		lsys := cidlink.DefaultLinkSystem()
 		unixfsnode.AddUnixFSReificationToLinkSystem(&lsys)
-		lsys.StorageReadOpener = blockOpener(ctx, blockGetter)
+		lsys.StorageReadOpener = blockOpener(ctx, blockGetter, bb.maxTraversalDepth)
 
 		// First resolve the path since we always need to.
 		lastCid, remainder, err := pathResolver.ResolveToLastNode(ctx, p)
@@ -489,8 +504,15 @@ func walkGatewaySimpleSelector(ctx context.Context, lastCid cid.Cid, terminalBlk
 	lctx := ipld.LinkContext{Ctx: ctx}
 	pathTerminalCidLink := cidlink.Link{Cid: lastCid}
 
-	// If the scope is the block, now we only need to retrieve the root block of the last element of the path.
+	// If the scope is the block, now we only need to retrieve the root block of
+	// the last element of the path. A caller that already resolved that block
+	// passes it in, and loading it again is not free: a link system reading a
+	// CAR stream in order has moved past it, and asking for it reports the
+	// stream as unexpectedly short even though the response is complete.
 	if params.Scope == DagScopeBlock {
+		if terminalBlk != nil {
+			return nil
+		}
 		_, err := lsys.LoadRaw(lctx, pathTerminalCidLink)
 		return err
 	}
@@ -871,10 +893,10 @@ type nodeGetterFetcherSingleUseFactory struct {
 	protoChooser traversal.LinkTargetNodePrototypeChooser
 }
 
-func newNodeGetterFetcherSingleUseFactory(ctx context.Context, ng format.NodeGetter) *nodeGetterFetcherSingleUseFactory {
+func newNodeGetterFetcherSingleUseFactory(ctx context.Context, ng format.NodeGetter, maxDepth int) *nodeGetterFetcherSingleUseFactory {
 	ls := cidlink.DefaultLinkSystem()
 	ls.TrustedStorage = true
-	ls.StorageReadOpener = blockOpener(ctx, ng)
+	ls.StorageReadOpener = blockOpener(ctx, ng, maxDepth)
 	ls.NodeReifier = unixfsnode.Reify
 
 	pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
@@ -943,8 +965,26 @@ func (n *nodeGetterFetcherSingleUseFactory) blankProgress(ctx context.Context) t
 	}
 }
 
-func blockOpener(ctx context.Context, ng format.NodeGetter) ipld.BlockReadOpener {
-	return func(_ ipld.LinkContext, lnk ipld.Link) (io.Reader, error) {
+// ErrTraversalTooDeep is returned when a DAG is nested deeper than the
+// backend's configured limit. See [WithMaxTraversalDepth].
+var ErrTraversalTooDeep = errors.New("dag traversal exceeded maximum depth")
+
+// blockOpener loads blocks for a traversal, refusing to descend past maxDepth.
+// A maxDepth of 0 means no limit.
+func blockOpener(ctx context.Context, ng format.NodeGetter, maxDepth int) ipld.BlockReadOpener {
+	return func(lctx ipld.LinkContext, lnk ipld.Link) (io.Reader, error) {
+		// LinkPath is the traversal path this link was reached by, so its
+		// length is the current depth.
+		if maxDepth > 0 && lctx.LinkPath.Len() > maxDepth {
+			// Deliberately without the path: at this depth it is thousands of
+			// segments long, and callers embed this error in their own output.
+			err := fmt.Errorf("%w of %d", ErrTraversalTooDeep, maxDepth)
+			// The response is already streaming by now, so this is the only
+			// place an operator can see why it was cut short.
+			log.Errorw("dag traversal stopped at depth limit", "limit", maxDepth, "link", lnk.String(), "err", err)
+			return nil, err
+		}
+
 		cidLink, ok := lnk.(cidlink.Link)
 		if !ok {
 			return nil, fmt.Errorf("invalid link type for loading: %v", lnk)

gateway/handler_car.go

@@ -26,6 +26,17 @@ const (
 	carOrderKey               = "car-order"
 )
 
+// carTruncationMessage is appended to a CAR response that failed partway
+// through, the same way withRetrievalTimeout marks a response it cuts short.
+// CAR has no in-band way to say "this stream is incomplete", so the bytes
+// themselves are made invalid where the next block header would start.
+//
+// It says nothing about why. The client already knows which DAG it asked for,
+// and the underlying error names the path it stopped at, which can run to
+// thousands of segments and means nothing to the caller. Operators get the
+// detail from the gateway log and the X-Stream-Error trailer.
+const carTruncationMessage = "\n\n[Gateway Error: CAR stream truncated, response is incomplete]"
+
 // serveCAR returns a CAR stream for specific DAG+selector
 func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.Request, rq *requestData) bool {
 	ctx, span := spanTrace(ctx, "Handler.ServeCAR", trace.WithAttributes(attribute.String("path", rq.immutablePath.String())))
@@ -118,6 +129,14 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R
 		// Due to this, we suggest client always verify that
 		// the received CAR stream response is matching requested DAG selector
 		w.Header().Set("X-Stream-Error", streamErr.Error())
+
+		// The status line and headers are long gone, so the trailer above is
+		// the only in-band signal and most clients never see it, and neither do
+		// reverse proxies or CDNs in front of this gateway. Append a marker so
+		// the stream stops being a well-formed CAR: a reader hits it where it
+		// expects the next block header and fails, instead of accepting a short
+		// DAG as complete. Same approach withRetrievalTimeout already uses.
+		fmt.Fprint(w, carTruncationMessage)
 		return false
 	}
 

go.mod

@@ -29,7 +29,7 @@ require (
 	github.com/ipfs/go-metrics-interface v0.3.0
 	github.com/ipfs/go-peertaskqueue v0.8.3
 	github.com/ipfs/go-test v0.4.1
-	github.com/ipfs/go-unixfsnode v1.10.5
+	github.com/ipfs/go-unixfsnode v1.10.6
 	github.com/ipld/go-car/v2 v2.17.0
 	github.com/ipld/go-codec-dagpb v1.7.0
 	github.com/ipld/go-ipld-prime v0.24.0

go.sum

@@ -142,8 +142,8 @@ github.com/ipfs/go-peertaskqueue v0.8.3 h1:tBPpGJy+A92RqtRFq5amJn0Uuj8Pw8tXi0X3e
 github.com/ipfs/go-peertaskqueue v0.8.3/go.mod h1:OqVync4kPOcXEGdj/LKvox9DCB5mkSBeXsPczCxLtYA=
 github.com/ipfs/go-test v0.4.1 h1:n6uNSakIgpTQIRorqNg2O02aMIFDLQk2z4rBfrlD3Uw=
 github.com/ipfs/go-test v0.4.1/go.mod h1:QmvVBf9kClNtRuFow4DASq03eFvjKla4Fy/UAkeeLO8=
-github.com/ipfs/go-unixfsnode v1.10.5 h1:V34JV7fM90y+2ZUef7ToShjmwAZ8oo1yP7zrpWzC5L4=
-github.com/ipfs/go-unixfsnode v1.10.5/go.mod h1:u/9Ukl+XYpfKTMu+NXQqxbzAJVTwSCoyTYBGgE+JdSE=
+github.com/ipfs/go-unixfsnode v1.10.6 h1:0rnKD4azJad4cT8t6wITqNl9Q0GTjB+YRBfMQ39C7Sw=
+github.com/ipfs/go-unixfsnode v1.10.6/go.mod h1:u/9Ukl+XYpfKTMu+NXQqxbzAJVTwSCoyTYBGgE+JdSE=
 github.com/ipld/go-car/v2 v2.17.0 h1:zgjSxf/lQNYcQPX08cvb5rSdEY8sv5OOnQIsZhZMPx4=
 github.com/ipld/go-car/v2 v2.17.0/go.mod h1:/4HY8tFZ1q42Mw54ILLPQfjkUqMJxFKqY1yMDKHlYko=
 github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0=

ipld/unixfs/hamt/hamt.go

@@ -176,6 +176,12 @@ func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*Shard, error) {
 		return nil, err
 	}
 
+	// A bitmap wider than the fanout would address children the shard cannot
+	// hold, and makeChilder copies it in without room to spare.
+	if len(fsn.Data()) > len(ds.childer.bitfield) {
+		return nil, fmt.Errorf("hamt bitfield of %d bytes exceeds fanout of %d", len(fsn.Data()), size)
+	}
+
 	ds.childer.makeChilder(fsn.Data(), pbnd.Links())
 
 	ds.hashFunc = fsn.HashType()