diff --git a/CHANGELOG.md b/CHANGELOG.md index a6594e8ba..b5838fef9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,8 +16,12 @@ The following emojis are used to highlight certain changes: ### Added +- `gateway`: added `WithMaxTraversalDepth`, bounding how deep `BlocksBackend` descends into a DAG while serving CAR responses. Traversal keeps per-level state, so its cost grows with depth. On by default at `DefaultMaxTraversalDepth` (1024), well above anything UnixFS produces: a file reaches terabytes by depth 4, and HAMT adds about 4 levels per million directory entries. Pass a positive value to set your own limit, or `WithMaxTraversalDepth(0)` to remove it entirely. [#1197](https://github.com/ipfs/boxo/pull/1197) + ### Changed +- `gateway`: a CAR response that fails partway through now ends with `[Gateway Error: CAR stream truncated, response is incomplete]`, the same approach `withRetrievalTimeout` already uses when it cuts a response short. `X-Stream-Error` is only set once the body is streaming, so it rarely reaches the client, and a truncated CAR was otherwise indistinguishable from a complete one. The marker makes the trailing bytes invalid CAR, so a reader stops with an error instead of accepting a short DAG. Mostly this is a quality of life improvement for operators: gateways usually sit behind reverse proxies and third-party CDNs, and when a response arrives short it is hard to tell which hop dropped it. Now the response says so itself. [#1197](https://github.com/ipfs/boxo/pull/1197) + ### Removed ### Fixed diff --git a/examples/go.mod b/examples/go.mod index 1528ecadb..b9dc3b91e 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -61,7 +61,7 @@ require ( github.com/ipfs/go-log/v2 v2.9.2 // indirect github.com/ipfs/go-metrics-interface v0.3.0 // indirect github.com/ipfs/go-peertaskqueue v0.8.3 // indirect - github.com/ipfs/go-unixfsnode v1.10.5 // indirect + github.com/ipfs/go-unixfsnode v1.10.6 // indirect github.com/ipld/go-codec-dagpb v1.7.0 // indirect github.com/ipld/go-ipld-prime v0.24.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect diff --git a/examples/go.sum b/examples/go.sum index cee9534f0..25d6bbaa5 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -141,8 +141,8 @@ github.com/ipfs/go-peertaskqueue v0.8.3 h1:tBPpGJy+A92RqtRFq5amJn0Uuj8Pw8tXi0X3e github.com/ipfs/go-peertaskqueue v0.8.3/go.mod h1:OqVync4kPOcXEGdj/LKvox9DCB5mkSBeXsPczCxLtYA= github.com/ipfs/go-test v0.4.1 h1:n6uNSakIgpTQIRorqNg2O02aMIFDLQk2z4rBfrlD3Uw= github.com/ipfs/go-test v0.4.1/go.mod h1:QmvVBf9kClNtRuFow4DASq03eFvjKla4Fy/UAkeeLO8= -github.com/ipfs/go-unixfsnode v1.10.5 h1:V34JV7fM90y+2ZUef7ToShjmwAZ8oo1yP7zrpWzC5L4= -github.com/ipfs/go-unixfsnode v1.10.5/go.mod h1:u/9Ukl+XYpfKTMu+NXQqxbzAJVTwSCoyTYBGgE+JdSE= +github.com/ipfs/go-unixfsnode v1.10.6 h1:0rnKD4azJad4cT8t6wITqNl9Q0GTjB+YRBfMQ39C7Sw= +github.com/ipfs/go-unixfsnode v1.10.6/go.mod h1:u/9Ukl+XYpfKTMu+NXQqxbzAJVTwSCoyTYBGgE+JdSE= github.com/ipld/go-car/v2 v2.17.0 h1:zgjSxf/lQNYcQPX08cvb5rSdEY8sv5OOnQIsZhZMPx4= github.com/ipld/go-car/v2 v2.17.0/go.mod h1:/4HY8tFZ1q42Mw54ILLPQfjkUqMJxFKqY1yMDKHlYko= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= diff --git a/gateway/backend.go b/gateway/backend.go index 8abbae04c..d126b7320 100644 --- a/gateway/backend.go +++ b/gateway/backend.go @@ -25,6 +25,9 @@ type backendOptions struct { // Only used by [BlocksBackend]: r resolver.Resolver + // Only used by [BlocksBackend]: + maxTraversalDepth int + // Only used by [CarBackend]: promRegistry prometheus.Registerer getBlockTimeout time.Duration @@ -64,6 +67,26 @@ func WithPrometheusRegistry(reg prometheus.Registerer) BackendOption { } } +// DefaultMaxTraversalDepth is how deep a DAG traversal will descend before it +// gives up. Traversal keeps per-level state, so its cost grows with depth. The +// default is far above anything UnixFS produces: a file reaches terabytes by +// depth 4, HAMT directories add about 4 levels per million entries, and +// directory nesting follows the source tree. +const DefaultMaxTraversalDepth = 1024 + +// WithMaxTraversalDepth sets how deep [BlocksBackend] will descend into a DAG +// before returning an error. By default, [DefaultMaxTraversalDepth] is used. +// Pass 0 to remove the limit. +func WithMaxTraversalDepth(depth int) BackendOption { + return func(opts *backendOptions) error { + if depth < 0 { + return fmt.Errorf("max traversal depth must not be negative; got %d", depth) + } + opts.maxTraversalDepth = depth + return nil + } +} + const DefaultGetBlockTimeout = time.Second * 60 // WithGetBlockTimeout sets a custom timeout when getting blocks from the diff --git a/gateway/backend_blocks.go b/gateway/backend_blocks.go index 6bb1bfae7..b7170ab38 100644 --- a/gateway/backend_blocks.go +++ b/gateway/backend_blocks.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "runtime/debug" "strings" "github.com/ipfs/boxo/blockservice" @@ -47,17 +48,18 @@ import ( // BlocksBackend is an [IPFSBackend] implementation based on a [blockservice.BlockService]. type BlocksBackend struct { baseBackend - blockStore blockstore.Blockstore - blockService blockservice.BlockService - dagService format.DAGService - resolver resolver.Resolver + blockStore blockstore.Blockstore + blockService blockservice.BlockService + dagService format.DAGService + resolver resolver.Resolver + maxTraversalDepth int } var _ IPFSBackend = (*BlocksBackend)(nil) // NewBlocksBackend creates a new [BlocksBackend] backed by a [blockservice.BlockService]. func NewBlocksBackend(blockService blockservice.BlockService, opts ...BackendOption) (*BlocksBackend, error) { - var compiledOptions backendOptions + compiledOptions := backendOptions{maxTraversalDepth: DefaultMaxTraversalDepth} for _, o := range opts { if err := o(&compiledOptions); err != nil { return nil, err @@ -84,11 +86,12 @@ func NewBlocksBackend(blockService blockservice.BlockService, opts ...BackendOpt } return &BlocksBackend{ - baseBackend: baseBackend, - blockStore: blockService.Blockstore(), - blockService: blockService, - dagService: dagService, - resolver: r, + baseBackend: baseBackend, + blockStore: blockService.Blockstore(), + blockService: blockService, + dagService: dagService, + resolver: r, + maxTraversalDepth: compiledOptions.maxTraversalDepth, }, nil } @@ -415,7 +418,7 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p path.ImmutablePath, param } // Setup the UnixFS resolver. - f := newNodeGetterFetcherSingleUseFactory(ctx, blockGetter) + f := newNodeGetterFetcherSingleUseFactory(ctx, blockGetter, bb.maxTraversalDepth) pathResolver := resolver.NewBasicResolver(f) _, _, err = pathResolver.ResolveToLastNode(ctx, p) @@ -438,6 +441,18 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p path.ImmutablePath, param r, w := io.Pipe() go func() { + // Traversal decodes blocks with whatever codec their CID names, so it + // runs third-party code this package does not control. The goroutine is + // detached from the request, and a panic on it would end the process + // rather than the response, so keep it contained here. + defer func() { + if rec := recover(); rec != nil { + log.Errorf("recovered from panic during CAR traversal of %s: %v\n%s", p, rec, debug.Stack()) + // io.PipeWriter.CloseWithError always returns nil. + _ = w.CloseWithError(errors.New("internal error during CAR traversal")) + } + }() + cw, err := storage.NewWritable( w, []cid.Cid{pathMetadata.LastSegment.RootCid()}, @@ -458,12 +473,12 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p path.ImmutablePath, param } // Setup the UnixFS resolver. - f := newNodeGetterFetcherSingleUseFactory(ctx, blockGetter) + f := newNodeGetterFetcherSingleUseFactory(ctx, blockGetter, bb.maxTraversalDepth) pathResolver := resolver.NewBasicResolver(f) lsys := cidlink.DefaultLinkSystem() unixfsnode.AddUnixFSReificationToLinkSystem(&lsys) - lsys.StorageReadOpener = blockOpener(ctx, blockGetter) + lsys.StorageReadOpener = blockOpener(ctx, blockGetter, bb.maxTraversalDepth) // First resolve the path since we always need to. lastCid, remainder, err := pathResolver.ResolveToLastNode(ctx, p) @@ -489,8 +504,15 @@ func walkGatewaySimpleSelector(ctx context.Context, lastCid cid.Cid, terminalBlk lctx := ipld.LinkContext{Ctx: ctx} pathTerminalCidLink := cidlink.Link{Cid: lastCid} - // If the scope is the block, now we only need to retrieve the root block of the last element of the path. + // If the scope is the block, now we only need to retrieve the root block of + // the last element of the path. A caller that already resolved that block + // passes it in, and loading it again is not free: a link system reading a + // CAR stream in order has moved past it, and asking for it reports the + // stream as unexpectedly short even though the response is complete. if params.Scope == DagScopeBlock { + if terminalBlk != nil { + return nil + } _, err := lsys.LoadRaw(lctx, pathTerminalCidLink) return err } @@ -871,10 +893,10 @@ type nodeGetterFetcherSingleUseFactory struct { protoChooser traversal.LinkTargetNodePrototypeChooser } -func newNodeGetterFetcherSingleUseFactory(ctx context.Context, ng format.NodeGetter) *nodeGetterFetcherSingleUseFactory { +func newNodeGetterFetcherSingleUseFactory(ctx context.Context, ng format.NodeGetter, maxDepth int) *nodeGetterFetcherSingleUseFactory { ls := cidlink.DefaultLinkSystem() ls.TrustedStorage = true - ls.StorageReadOpener = blockOpener(ctx, ng) + ls.StorageReadOpener = blockOpener(ctx, ng, maxDepth) ls.NodeReifier = unixfsnode.Reify pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { @@ -943,8 +965,26 @@ func (n *nodeGetterFetcherSingleUseFactory) blankProgress(ctx context.Context) t } } -func blockOpener(ctx context.Context, ng format.NodeGetter) ipld.BlockReadOpener { - return func(_ ipld.LinkContext, lnk ipld.Link) (io.Reader, error) { +// ErrTraversalTooDeep is returned when a DAG is nested deeper than the +// backend's configured limit. See [WithMaxTraversalDepth]. +var ErrTraversalTooDeep = errors.New("dag traversal exceeded maximum depth") + +// blockOpener loads blocks for a traversal, refusing to descend past maxDepth. +// A maxDepth of 0 means no limit. +func blockOpener(ctx context.Context, ng format.NodeGetter, maxDepth int) ipld.BlockReadOpener { + return func(lctx ipld.LinkContext, lnk ipld.Link) (io.Reader, error) { + // LinkPath is the traversal path this link was reached by, so its + // length is the current depth. + if maxDepth > 0 && lctx.LinkPath.Len() > maxDepth { + // Deliberately without the path: at this depth it is thousands of + // segments long, and callers embed this error in their own output. + err := fmt.Errorf("%w of %d", ErrTraversalTooDeep, maxDepth) + // The response is already streaming by now, so this is the only + // place an operator can see why it was cut short. + log.Errorw("dag traversal stopped at depth limit", "limit", maxDepth, "link", lnk.String(), "err", err) + return nil, err + } + cidLink, ok := lnk.(cidlink.Link) if !ok { return nil, fmt.Errorf("invalid link type for loading: %v", lnk) diff --git a/gateway/handler_car.go b/gateway/handler_car.go index 9e0687c2f..74205bd74 100644 --- a/gateway/handler_car.go +++ b/gateway/handler_car.go @@ -26,6 +26,17 @@ const ( carOrderKey = "car-order" ) +// carTruncationMessage is appended to a CAR response that failed partway +// through, the same way withRetrievalTimeout marks a response it cuts short. +// CAR has no in-band way to say "this stream is incomplete", so the bytes +// themselves are made invalid where the next block header would start. +// +// It says nothing about why. The client already knows which DAG it asked for, +// and the underlying error names the path it stopped at, which can run to +// thousands of segments and means nothing to the caller. Operators get the +// detail from the gateway log and the X-Stream-Error trailer. +const carTruncationMessage = "\n\n[Gateway Error: CAR stream truncated, response is incomplete]" + // serveCAR returns a CAR stream for specific DAG+selector func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.Request, rq *requestData) bool { ctx, span := spanTrace(ctx, "Handler.ServeCAR", trace.WithAttributes(attribute.String("path", rq.immutablePath.String()))) @@ -118,6 +129,14 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R // Due to this, we suggest client always verify that // the received CAR stream response is matching requested DAG selector w.Header().Set("X-Stream-Error", streamErr.Error()) + + // The status line and headers are long gone, so the trailer above is + // the only in-band signal and most clients never see it, and neither do + // reverse proxies or CDNs in front of this gateway. Append a marker so + // the stream stops being a well-formed CAR: a reader hits it where it + // expects the next block header and fails, instead of accepting a short + // DAG as complete. Same approach withRetrievalTimeout already uses. + fmt.Fprint(w, carTruncationMessage) return false } diff --git a/go.mod b/go.mod index 7b44625cf..f39621c34 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/ipfs/go-metrics-interface v0.3.0 github.com/ipfs/go-peertaskqueue v0.8.3 github.com/ipfs/go-test v0.4.1 - github.com/ipfs/go-unixfsnode v1.10.5 + github.com/ipfs/go-unixfsnode v1.10.6 github.com/ipld/go-car/v2 v2.17.0 github.com/ipld/go-codec-dagpb v1.7.0 github.com/ipld/go-ipld-prime v0.24.0 diff --git a/go.sum b/go.sum index 340c3d4d0..43f45a54b 100644 --- a/go.sum +++ b/go.sum @@ -142,8 +142,8 @@ github.com/ipfs/go-peertaskqueue v0.8.3 h1:tBPpGJy+A92RqtRFq5amJn0Uuj8Pw8tXi0X3e github.com/ipfs/go-peertaskqueue v0.8.3/go.mod h1:OqVync4kPOcXEGdj/LKvox9DCB5mkSBeXsPczCxLtYA= github.com/ipfs/go-test v0.4.1 h1:n6uNSakIgpTQIRorqNg2O02aMIFDLQk2z4rBfrlD3Uw= github.com/ipfs/go-test v0.4.1/go.mod h1:QmvVBf9kClNtRuFow4DASq03eFvjKla4Fy/UAkeeLO8= -github.com/ipfs/go-unixfsnode v1.10.5 h1:V34JV7fM90y+2ZUef7ToShjmwAZ8oo1yP7zrpWzC5L4= -github.com/ipfs/go-unixfsnode v1.10.5/go.mod h1:u/9Ukl+XYpfKTMu+NXQqxbzAJVTwSCoyTYBGgE+JdSE= +github.com/ipfs/go-unixfsnode v1.10.6 h1:0rnKD4azJad4cT8t6wITqNl9Q0GTjB+YRBfMQ39C7Sw= +github.com/ipfs/go-unixfsnode v1.10.6/go.mod h1:u/9Ukl+XYpfKTMu+NXQqxbzAJVTwSCoyTYBGgE+JdSE= github.com/ipld/go-car/v2 v2.17.0 h1:zgjSxf/lQNYcQPX08cvb5rSdEY8sv5OOnQIsZhZMPx4= github.com/ipld/go-car/v2 v2.17.0/go.mod h1:/4HY8tFZ1q42Mw54ILLPQfjkUqMJxFKqY1yMDKHlYko= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= diff --git a/ipld/unixfs/hamt/hamt.go b/ipld/unixfs/hamt/hamt.go index 7b136785f..0976adb17 100644 --- a/ipld/unixfs/hamt/hamt.go +++ b/ipld/unixfs/hamt/hamt.go @@ -176,6 +176,12 @@ func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*Shard, error) { return nil, err } + // A bitmap wider than the fanout would address children the shard cannot + // hold, and makeChilder copies it in without room to spare. + if len(fsn.Data()) > len(ds.childer.bitfield) { + return nil, fmt.Errorf("hamt bitfield of %d bytes exceeds fanout of %d", len(fsn.Data()), size) + } + ds.childer.makeChilder(fsn.Data(), pbnd.Links()) ds.hashFunc = fsn.HashType()