diff --git a/pkg/sentry/mm/io.go b/pkg/sentry/mm/io.go index e7651b4adaa..c784b840d48 100644 --- a/pkg/sentry/mm/io.go +++ b/pkg/sentry/mm/io.go @@ -543,7 +543,7 @@ func (mm *MemoryManager) handleASIOFault(ctx context.Context, addr hostarch.Addr // Ensure that we have usable pmas. mm.activeMu.Lock() - pseg, pend, err := mm.getPMAsLocked(ctx, vseg, ar, at, true /* callerIndirectCommit */) + pseg, pend, err := mm.getPMAsLocked(ctx, vseg, ar, at, true /* callerIndirectCommit */, false /* forPin */) mm.mappingMu.RUnlock() if pendaddr := pend.Start(); pendaddr < ar.End { if pendaddr <= ar.Start { @@ -597,7 +597,7 @@ func (mm *MemoryManager) withInternalMappings(ctx context.Context, ar hostarch.A // Ensure that we have usable pmas. mm.activeMu.Lock() - pseg, pend, perr := mm.getPMAsLocked(ctx, vseg, ar, at, true /* callerIndirectCommit */) + pseg, pend, perr := mm.getPMAsLocked(ctx, vseg, ar, at, true /* callerIndirectCommit */, false /* forPin */) mm.mappingMu.RUnlock() if pendaddr := pend.Start(); pendaddr < ar.End { if pendaddr <= ar.Start { diff --git a/pkg/sentry/mm/lifecycle.go b/pkg/sentry/mm/lifecycle.go index feb4c03fcdb..ef8e1c9ee42 100644 --- a/pkg/sentry/mm/lifecycle.go +++ b/pkg/sentry/mm/lifecycle.go @@ -19,12 +19,16 @@ import ( "gvisor.dev/gvisor/pkg/atomicbitops" "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/hostarch" + "gvisor.dev/gvisor/pkg/safecopy" + "gvisor.dev/gvisor/pkg/safemem" "gvisor.dev/gvisor/pkg/sentry/arch" "gvisor.dev/gvisor/pkg/sentry/limits" "gvisor.dev/gvisor/pkg/sentry/memmap" "gvisor.dev/gvisor/pkg/sentry/pgalloc" "gvisor.dev/gvisor/pkg/sentry/platform" + "gvisor.dev/gvisor/pkg/sentry/usage" ) // NewMemoryManager returns a new MemoryManager with no mappings and 1 user. @@ -151,12 +155,17 @@ func (mm *MemoryManager) Fork(ctx context.Context) (*MemoryManager, error) { defer mm.activeMu.Unlock() mm2.activeMu.NestedLock(activeLockForked) defer mm2.activeMu.NestedUnlock(activeLockForked) - if dontforks { + if dontforks || mm.hasPinned { defer mm.pmas.MergeInsideRange(mm.applicationAddrRange()) } srcvseg := mm.vmas.FirstSegment() dstpgap := mm2.pmas.FirstGap() var unmapAR hostarch.AddrRange + defer func() { + if unmapAR.Length() != 0 { + mm.unmapASLocked(unmapAR) + } + }() memCgID := pgalloc.MemoryCgroupIDFromContext(ctx) for srcpseg := mm.pmas.FirstSegment(); srcpseg.Ok(); srcpseg = srcpseg.NextSegment() { pma := srcpseg.ValuePtr() @@ -184,6 +193,50 @@ func (mm *MemoryManager) Fork(ctx context.Context) (*MemoryManager, error) { pma = srcpseg.ValuePtr() } + if mm.hasPinned && !pma.needCOW { + // Pinned pages must not be made copy-on-write: breaking + // copy-on-write moves the writing process to a new copy of the + // page, while any DMA registered against the original page + // continues to target it, causing the two to diverge. Instead, + // give the child a copy of possibly-pinned pages immediately, + // leaving the parent's mappings unchanged; compare Linux's + // mm/memory.c:copy_present_ptes() => folio_needs_cow_for_dma(). + // Since Pin() breaks copy-on-write on the pinned range, + // possibly-pinned pages are exactly those in private + // non-copy-on-write pmas with more than one reference. + if sfr, ok := mm.mf.FirstSharedRange(srcpseg.fileRange()); ok { + sar := hostarch.AddrRange{ + Start: srcpseg.Start() + hostarch.Addr(sfr.Start-pma.off), + End: srcpseg.Start() + hostarch.Addr(sfr.End-pma.off), + } + if sar.Start > srcpseg.Start() { + // Isolate the pages preceding sar and fall through to + // make only those copy-on-write; the remainder of the + // pma is revisited on the next iteration. + srcpseg = mm.pmas.Isolate(srcpseg, hostarch.AddrRange{srcpseg.Start(), sar.Start}) + pma = srcpseg.ValuePtr() + } else { + // Copy the possibly-pinned pages for the child now. + srcpseg = mm.pmas.Isolate(srcpseg, sar) + var err error + dstpgap, err = mm.forkCopyPMALocked(mm2, srcpseg, dstpgap, memCgID) + if err != nil { + // Fork fails, as when Linux's copy_page_range() + // fails. Release the references and mappings already + // established for mm2. + for pseg := mm2.pmas.FirstSegment(); pseg.Ok(); pseg = pseg.NextSegment() { + pseg.ValuePtr().file.DecRef(pseg.fileRange()) + } + mm2.pmas.RemoveAll() + _, droppedIDs = mm2.removeVMAsLocked(ctx, mm2.applicationAddrRange(), droppedIDs) + as.Release() + return nil, err + } + continue + } + } + } + if !pma.needCOW { pma.needCOW = true if pma.effectivePerms.Write { @@ -212,9 +265,6 @@ func (mm *MemoryManager) Fork(ctx context.Context) (*MemoryManager, error) { mm2.addRSSLocked(addrRange) dstpgap = mm2.pmas.Insert(dstpgap, addrRange, *pma).NextGap() } - if unmapAR.Length() != 0 { - mm.unmapASLocked(unmapAR) - } // Between when we call memmap.Mappable.AddMapping while copying vmas and // when we lock mm2.activeMu to copy pmas, calls to mm2.Invalidate() are @@ -248,6 +298,50 @@ func (mm *MemoryManager) Fork(ctx context.Context) (*MemoryManager, error) { return mm2, nil } +// forkCopyPMALocked copies the contents of the private pma represented by +// srcpseg in mm into newly-allocated memory, and inserts a pma mapping that +// memory into mm2 at the same address range, as Fork requires for pages that +// may be pinned for DMA. It returns the gap after the inserted pma. +// +// Preconditions: +// - mm.activeMu must be locked for writing. +// - mm2.activeMu must be locked for writing. +// - srcpseg.ValuePtr().private == true. +// - dstpgap must be the gap in mm2.pmas at which the new pma should be +// inserted. +func (mm *MemoryManager) forkCopyPMALocked(mm2 *MemoryManager, srcpseg pmaIterator, dstpgap pmaGapIterator, memCgID uint32) (pmaGapIterator, error) { + if err := srcpseg.getInternalMappingsLocked(); err != nil { + return dstpgap, err + } + copyAR := srcpseg.Range() + reader := safemem.BlockSeqReader{Blocks: mm.internalMappingsLocked(srcpseg, copyAR)} + huge := mm.mf.HugepagesEnabled() && copyAR.IsHugePageAligned() + fr, err := mm.mf.Allocate(uint64(copyAR.Length()), pgalloc.AllocOpts{ + Kind: usage.Anonymous, + MemCgID: memCgID, + Mode: pgalloc.AllocateAndWritePopulate, + Huge: huge, + ReaderFunc: reader.ReadToBlocks, + }) + if err != nil { + if _, ok := err.(safecopy.BusError); ok { + // Compare Linux's mm/memory.c:copy_present_page() => + // copy_mc_user_highpage(). + err = linuxerr.EHWPOISON + } + if fr.Length() != 0 { + mm.mf.DecRef(fr) + } + return dstpgap, err + } + newpma := srcpseg.Value() + newpma.off = fr.Start + newpma.huge = huge + newpma.internalMappings = safemem.BlockSeq{} + mm2.addRSSLocked(copyAR) + return mm2.pmas.Insert(dstpgap, copyAR, newpma).NextGap(), nil +} + // IncUsers increments mm's user count and returns true. If the user count is // already 0, IncUsers does nothing and returns false. func (mm *MemoryManager) IncUsers() bool { diff --git a/pkg/sentry/mm/mm.go b/pkg/sentry/mm/mm.go index b2e1d82792c..2f36adabdba 100644 --- a/pkg/sentry/mm/mm.go +++ b/pkg/sentry/mm/mm.go @@ -158,6 +158,13 @@ type MemoryManager struct { // maxRSS is protected by activeMu. maxRSS uint64 + // hasPinned is true if pages in this MemoryManager have ever been pinned + // by Pin. It is never cleared, even if all pinned pages are unpinned; + // compare Linux's MMF_HAS_PINNED. + // + // hasPinned is protected by activeMu. + hasPinned bool + // as is the platform.AddressSpace that pmas are mapped into. as is immutable // until users becomes 0, at which point as becomes nil. as platform.AddressSpace `state:"nosave"` diff --git a/pkg/sentry/mm/mm_test.go b/pkg/sentry/mm/mm_test.go index 645e24d9bd8..8250b4fb64a 100644 --- a/pkg/sentry/mm/mm_test.go +++ b/pkg/sentry/mm/mm_test.go @@ -15,11 +15,13 @@ package mm import ( + "bytes" "testing" "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/hostarch" + "gvisor.dev/gvisor/pkg/safemem" "gvisor.dev/gvisor/pkg/sentry/arch" "gvisor.dev/gvisor/pkg/sentry/contexttest" "gvisor.dev/gvisor/pkg/sentry/limits" @@ -343,3 +345,191 @@ func TestGetAllocationDirection(t *testing.T) { }) } } + +// readPinnedRange returns the current contents of the pinned range pr. +func readPinnedRange(t *testing.T, pr PinnedRange) []byte { + t.Helper() + ims, err := pr.File.MapInternal(pr.FileRange(), hostarch.Read) + if err != nil { + t.Fatalf("MapInternal got err %v want nil", err) + } + buf := make([]byte, pr.Source.Length()) + if _, err := safemem.CopySeq(safemem.BlockSeqOf(safemem.BlockFromSafeSlice(buf)), ims); err != nil { + t.Fatalf("CopySeq got err %v want nil", err) + } + return buf +} + +// TestPinnedPagesCopiedOnFork tests that Fork copies pinned pages to the +// child immediately instead of making them copy-on-write, so that the +// parent's mappings never diverge from pages registered for DMA. +func TestPinnedPagesCopiedOnFork(t *testing.T) { + ctx := contexttest.Context(t) + mm := testMemoryManager(ctx, t) + defer mm.DecUsers(ctx) + + // Map 3 pages, but pin only the middle page, so that Fork must handle + // both the pinned page and the copy-on-write pages surrounding it. + const npages = 3 + addr, err := mm.MMap(ctx, memmap.MMapOpts{ + Length: npages * hostarch.PageSize, + Private: true, + Perms: hostarch.ReadWrite, + MaxPerms: hostarch.AnyAccess, + }) + if err != nil { + t.Fatalf("MMap got err %v want nil", err) + } + preFork := bytes.Repeat([]byte{'A'}, npages*hostarch.PageSize) + if _, err := mm.CopyOut(ctx, addr, preFork, usermem.IOOpts{}); err != nil { + t.Fatalf("CopyOut got err %v want nil", err) + } + + pinAR := hostarch.AddrRange{addr + hostarch.PageSize, addr + 2*hostarch.PageSize} + prs, err := mm.Pin(ctx, pinAR, hostarch.ReadWrite, false /* ignorePermissions */) + if err != nil { + t.Fatalf("Pin got err %v want nil", err) + } + defer Unpin(prs) + + // Fork twice so that both the first fork of a pinned pma, and the fork + // of the resulting parent pma state, are tested. + for i := 0; i < 2; i++ { + mm2, err := mm.Fork(ctx) + if err != nil { + t.Fatalf("Fork got err %v want nil", err) + } + defer mm2.DecUsers(ctx) + + // The parent's writes to the pinned page must be visible through the + // pinned range, i.e. the parent must not have been moved to a new + // copy of the page by copy-on-write. + postFork := bytes.Repeat([]byte{'B' + byte(i)}, npages*hostarch.PageSize) + if _, err := mm.CopyOut(ctx, addr, postFork, usermem.IOOpts{}); err != nil { + t.Fatalf("CopyOut got err %v want nil", err) + } + if got, want := readPinnedRange(t, prs[0]), postFork[:hostarch.PageSize]; !bytes.Equal(got, want) { + t.Errorf("pinned page contains %q..., want %q...; parent's mapping of the pinned page diverged", got[:4], want[:4]) + } + + // The child must see the pre-fork contents of all pages. + childBuf := make([]byte, npages*hostarch.PageSize) + if _, err := mm2.CopyIn(ctx, addr, childBuf, usermem.IOOpts{}); err != nil { + t.Fatalf("CopyIn got err %v want nil", err) + } + if !bytes.Equal(childBuf, preFork) { + t.Errorf("child read %q..., want %q...", childBuf[:4], preFork[:4]) + } + + // The child's writes must not be visible through the pinned range. + if _, err := mm2.CopyOut(ctx, addr, bytes.Repeat([]byte{'z'}, npages*hostarch.PageSize), usermem.IOOpts{}); err != nil { + t.Fatalf("CopyOut got err %v want nil", err) + } + if got, want := readPinnedRange(t, prs[0]), postFork[:hostarch.PageSize]; !bytes.Equal(got, want) { + t.Errorf("pinned page contains %q..., want %q...; child's writes are visible through the pinned range", got[:4], want[:4]) + } + + preFork = postFork + } +} + +// TestPinUnsharesCopyOnWritePages tests that pinning copy-on-write pages +// breaks copy-on-write, so that the pinned pages are those mapped by the +// pinning process, even if the pin does not require write access. +func TestPinUnsharesCopyOnWritePages(t *testing.T) { + ctx := contexttest.Context(t) + mm := testMemoryManager(ctx, t) + defer mm.DecUsers(ctx) + + addr, err := mm.MMap(ctx, memmap.MMapOpts{ + Length: hostarch.PageSize, + Private: true, + Perms: hostarch.ReadWrite, + MaxPerms: hostarch.AnyAccess, + }) + if err != nil { + t.Fatalf("MMap got err %v want nil", err) + } + preFork := bytes.Repeat([]byte{'A'}, hostarch.PageSize) + if _, err := mm.CopyOut(ctx, addr, preFork, usermem.IOOpts{}); err != nil { + t.Fatalf("CopyOut got err %v want nil", err) + } + + // Make the page copy-on-write by forking, then pin it for reading only. + mm2, err := mm.Fork(ctx) + if err != nil { + t.Fatalf("Fork got err %v want nil", err) + } + defer mm2.DecUsers(ctx) + ar := hostarch.AddrRange{addr, addr + hostarch.PageSize} + prs, err := mm.Pin(ctx, ar, hostarch.Read, false /* ignorePermissions */) + if err != nil { + t.Fatalf("Pin got err %v want nil", err) + } + defer Unpin(prs) + + // The parent's writes must be visible through the pinned range, and must + // not be visible to the child. + postFork := bytes.Repeat([]byte{'B'}, hostarch.PageSize) + if _, err := mm.CopyOut(ctx, addr, postFork, usermem.IOOpts{}); err != nil { + t.Fatalf("CopyOut got err %v want nil", err) + } + if got := readPinnedRange(t, prs[0]); !bytes.Equal(got, postFork) { + t.Errorf("pinned page contains %q..., want %q...; pin did not unshare the copy-on-write page", got[:4], postFork[:4]) + } + childBuf := make([]byte, hostarch.PageSize) + if _, err := mm2.CopyIn(ctx, addr, childBuf, usermem.IOOpts{}); err != nil { + t.Fatalf("CopyIn got err %v want nil", err) + } + if !bytes.Equal(childBuf, preFork) { + t.Errorf("child read %q..., want %q...", childBuf[:4], preFork[:4]) + } +} + +// TestUnpinnedPagesCopyOnWriteAfterFork tests that pages that were pinned but +// have been unpinned are once again made copy-on-write by Fork. +func TestUnpinnedPagesCopyOnWriteAfterFork(t *testing.T) { + ctx := contexttest.Context(t) + mm := testMemoryManager(ctx, t) + defer mm.DecUsers(ctx) + + addr, err := mm.MMap(ctx, memmap.MMapOpts{ + Length: hostarch.PageSize, + Private: true, + Perms: hostarch.ReadWrite, + MaxPerms: hostarch.AnyAccess, + }) + if err != nil { + t.Fatalf("MMap got err %v want nil", err) + } + if _, err := mm.CopyOut(ctx, addr, bytes.Repeat([]byte{'A'}, hostarch.PageSize), usermem.IOOpts{}); err != nil { + t.Fatalf("CopyOut got err %v want nil", err) + } + + ar := hostarch.AddrRange{addr, addr + hostarch.PageSize} + prs, err := mm.Pin(ctx, ar, hostarch.ReadWrite, false /* ignorePermissions */) + if err != nil { + t.Fatalf("Pin got err %v want nil", err) + } + Unpin(prs) + + mm2, err := mm.Fork(ctx) + if err != nil { + t.Fatalf("Fork got err %v want nil", err) + } + defer mm2.DecUsers(ctx) + + // Since no pages remain pinned, all pages should be copy-on-write, not + // copied for the child. + mm.activeMu.RLock() + pseg := mm.pmas.FindSegment(addr) + if !pseg.Ok() { + mm.activeMu.RUnlock() + t.Fatalf("no pma for addr %#x", addr) + } + needCOW := pseg.ValuePtr().needCOW + mm.activeMu.RUnlock() + if !needCOW { + t.Errorf("pma is not copy-on-write after fork of unpinned page") + } +} diff --git a/pkg/sentry/mm/pma.go b/pkg/sentry/mm/pma.go index 661c7fa667c..19e196e2470 100644 --- a/pkg/sentry/mm/pma.go +++ b/pkg/sentry/mm/pma.go @@ -99,6 +99,11 @@ func (mm *MemoryManager) existingVecPMAsLocked(ars hostarch.AddrRangeSeq, at hos // commit all pages in ar without using the caller's page tables, in the same // sense as pgalloc.AllocateCallerIndirectCommit. // +// If forPin is true, the caller is Pin, and copy-on-write pmas will be broken +// even if at.Write is false, so that pinned pages remain coherent with the +// application's mappings; compare Linux's +// mm/internal.h:gup_must_unshare(FOLL_PIN|FOLL_LONGTERM). +// // Preconditions: // - mm.mappingMu must be locked. // - mm.activeMu must be locked for writing. @@ -106,7 +111,7 @@ func (mm *MemoryManager) existingVecPMAsLocked(ars hostarch.AddrRangeSeq, at hos // - vseg.Range().Contains(ar.Start). // - vmas must exist for all addresses in ar, and support accesses of type at // (i.e. permission checks must have been performed against vmas). -func (mm *MemoryManager) getPMAsLocked(ctx context.Context, vseg vmaIterator, ar hostarch.AddrRange, at hostarch.AccessType, callerIndirectCommit bool) (pmaIterator, pmaGapIterator, error) { +func (mm *MemoryManager) getPMAsLocked(ctx context.Context, vseg vmaIterator, ar hostarch.AddrRange, at hostarch.AccessType, callerIndirectCommit, forPin bool) (pmaIterator, pmaGapIterator, error) { if checkInvariants { if !ar.WellFormed() || ar.Length() == 0 { panic(fmt.Sprintf("invalid ar: %v", ar)) @@ -128,7 +133,7 @@ func (mm *MemoryManager) getPMAsLocked(ctx context.Context, vseg vmaIterator, ar } ar = hostarch.AddrRange{ar.Start.RoundDown(), end} - pstart, pend, perr := mm.getPMAsInternalLocked(ctx, vseg, ar, at, callerIndirectCommit) + pstart, pend, perr := mm.getPMAsInternalLocked(ctx, vseg, ar, at, callerIndirectCommit, forPin) if pend.Start() <= ar.Start { return pmaIterator{}, pend, perr } @@ -174,7 +179,7 @@ func (mm *MemoryManager) getVecPMAsLocked(ctx context.Context, ars hostarch.Addr } ar = hostarch.AddrRange{ar.Start.RoundDown(), end} - _, pend, perr := mm.getPMAsInternalLocked(ctx, mm.vmas.FindSegment(ar.Start), ar, at, callerIndirectCommit) + _, pend, perr := mm.getPMAsInternalLocked(ctx, mm.vmas.FindSegment(ar.Start), ar, at, callerIndirectCommit, false /* forPin */) if perr != nil { return truncatedAddrRangeSeq(ars, arsit, pend.Start()), perr } @@ -225,7 +230,7 @@ func (mm *MemoryManager) getAllocationDirection(ar hostarch.AddrRange, vma *vma) // - getPMAsInternalLocked additionally requires that ar is page-aligned. // getPMAsInternalLocked is an implementation helper for getPMAsLocked and // getVecPMAsLocked; other clients should call one of those instead. -func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIterator, ar hostarch.AddrRange, at hostarch.AccessType, callerIndirectCommit bool) (pmaIterator, pmaGapIterator, error) { +func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIterator, ar hostarch.AddrRange, at hostarch.AccessType, callerIndirectCommit, forPin bool) (pmaIterator, pmaGapIterator, error) { if checkInvariants { if !ar.WellFormed() || ar.Length() == 0 || !ar.IsPageAligned() { panic(fmt.Sprintf("invalid ar: %v", ar)) @@ -412,7 +417,7 @@ func (mm *MemoryManager) getPMAsInternalLocked(ctx context.Context, vseg vmaIter case pseg.Ok() && pseg.Start() < vsegAR.End: oldpma := pseg.ValuePtr() - if at.Write && mm.isPMACopyOnWriteLocked(vseg, pseg) { + if (at.Write || forPin) && mm.isPMACopyOnWriteLocked(vseg, pseg) { // Break copy-on-write by copying. if checkInvariants { if !oldpma.maxPerms.Read { @@ -731,8 +736,11 @@ func (mm *MemoryManager) invalidateLocked(ar hostarch.AddrRange, invalidatePriva // PinnedRanges and a non-nil error. // // Pin does not prevent mapped ranges from changing, making it unsuitable for -// most I/O. It should only be used in contexts that would use get_user_pages() -// in the Linux kernel. +// most I/O. It should only be used in contexts that would use +// pin_user_pages(FOLL_LONGTERM) in the Linux kernel. Like Linux, Pin breaks +// copy-on-write on the pinned range (even if at.Write is false). This also +// yields the invariant that pinned pages in private mappings live in non-CoW +// pmas with >1 reference. // // Preconditions: // - ar.Length() != 0. @@ -757,7 +765,8 @@ func (mm *MemoryManager) Pin(ctx context.Context, ar hostarch.AddrRange, at host // Ensure that we have usable pmas. mm.activeMu.Lock() - pseg, pend, perr := mm.getPMAsLocked(ctx, vseg, ar, at, false /* callerIndirectCommit */) + mm.hasPinned = true + pseg, pend, perr := mm.getPMAsLocked(ctx, vseg, ar, at, false /* callerIndirectCommit */, true /* forPin */) mm.mappingMu.RUnlock() if pendaddr := pend.Start(); pendaddr < ar.End { if pendaddr <= ar.Start { diff --git a/pkg/sentry/mm/syscalls.go b/pkg/sentry/mm/syscalls.go index 4dbe6dec756..992b8ecea8a 100644 --- a/pkg/sentry/mm/syscalls.go +++ b/pkg/sentry/mm/syscalls.go @@ -54,7 +54,7 @@ func (mm *MemoryManager) HandleUserFault(ctx context.Context, addr hostarch.Addr // Ensure that we have a usable pma. mm.activeMu.Lock() - pseg, _, err := mm.getPMAsLocked(ctx, vseg, ar, at, true /* callerIndirectCommit */) + pseg, _, err := mm.getPMAsLocked(ctx, vseg, ar, at, true /* callerIndirectCommit */, false /* forPin */) mm.mappingMu.RUnlock() if err != nil { mm.activeMu.Unlock() @@ -192,7 +192,7 @@ func (mm *MemoryManager) populateVMA(ctx context.Context, vseg vmaIterator, ar h } // Ensure that we have usable pmas. - pseg, _, err := mm.getPMAsLocked(ctx, vseg, ar, hostarch.NoAccess, platformEffect == memmap.PlatformEffectCommit) + pseg, _, err := mm.getPMAsLocked(ctx, vseg, ar, hostarch.NoAccess, platformEffect == memmap.PlatformEffectCommit, false /* forPin */) if err != nil { mm.activeMu.Unlock() return err @@ -238,7 +238,7 @@ func (mm *MemoryManager) populateVMAAndUnlock(ctx context.Context, vseg vmaItera // mm.mappingMu doesn't need to be write-locked for getPMAsLocked, and it // isn't needed at all for mapASLocked. mm.mappingMu.DowngradeLock() - pseg, _, err := mm.getPMAsLocked(ctx, vseg, ar, hostarch.NoAccess, platformEffect == memmap.PlatformEffectCommit) + pseg, _, err := mm.getPMAsLocked(ctx, vseg, ar, hostarch.NoAccess, platformEffect == memmap.PlatformEffectCommit, false /* forPin */) mm.mappingMu.RUnlock() if err != nil { // mm/util.c:vm_mmap_pgoff() ignores the error, if any, from @@ -911,7 +911,7 @@ func (mm *MemoryManager) MLock(ctx context.Context, addr hostarch.Addr, length u mm.mappingMu.RUnlock() return linuxerr.ENOMEM } - _, _, err := mm.getPMAsLocked(ctx, vseg, vseg.Range().Intersect(ar), hostarch.NoAccess, true /* callerIndirectCommit */) + _, _, err := mm.getPMAsLocked(ctx, vseg, vseg.Range().Intersect(ar), hostarch.NoAccess, true /* callerIndirectCommit */, false /* forPin */) if err != nil { mm.activeMu.Unlock() mm.mappingMu.RUnlock() @@ -1006,7 +1006,7 @@ func (mm *MemoryManager) MLockAll(ctx context.Context, opts MLockAllOpts) error mm.mappingMu.DowngradeLock() for vseg := mm.vmas.FirstSegment(); vseg.Ok(); vseg = vseg.NextSegment() { if vseg.ValuePtr().effectivePerms.Any() { - mm.getPMAsLocked(ctx, vseg, vseg.Range(), hostarch.NoAccess, true /* callerIndirectCommit */) + mm.getPMAsLocked(ctx, vseg, vseg.Range(), hostarch.NoAccess, true /* callerIndirectCommit */, false /* forPin */) } } diff --git a/pkg/sentry/pgalloc/pgalloc.go b/pkg/sentry/pgalloc/pgalloc.go index 7cdad4c27da..6c2eae32a8c 100644 --- a/pkg/sentry/pgalloc/pgalloc.go +++ b/pkg/sentry/pgalloc/pgalloc.go @@ -1171,6 +1171,45 @@ func (f *MemoryFile) HasUniqueRef(fr memmap.FileRange) bool { return hasUniqueRef } +// FirstSharedRange returns the first subrange of fr on which more than one +// reference is held, and true if such a subrange exists. As for HasUniqueRef, +// if the caller holds a reference on the given range and is preventing other +// goroutines from copying it, then subranges reported as unshared (i.e. not +// contained in the returned range) are not racy. +// +// Preconditions: At least one reference must be held on all pages in fr. +func (f *MemoryFile) FirstSharedRange(fr memmap.FileRange) (memmap.FileRange, bool) { + var sr memmap.FileRange + f.mu.Lock() + defer f.mu.Unlock() + f.forEachChunk(fr, func(chunk *chunkInfo, chunkFR memmap.FileRange) bool { + unfree := &f.unfreeSmall + if chunk.huge { + unfree = &f.unfreeHuge + } + cont := true + unfree.VisitFullRange(chunkFR, func(ufseg unfreeIterator) bool { + if ufseg.ValuePtr().refs > 1 { + r := ufseg.Range().Intersect(chunkFR) + if sr.Length() == 0 { + sr = r + } else if sr.End == r.Start { + sr.End = r.End + } else { + cont = false + } + } else if sr.Length() != 0 { + cont = false + } + return cont + }) + // Continue to the next chunk only if no shared range has been found, + // or if the one found may extend into the next chunk. + return cont && (sr.Length() == 0 || sr.End == chunkFR.End) + }) + return sr, sr.Length() != 0 +} + // IncRef implements memmap.File.IncRef. func (f *MemoryFile) IncRef(fr memmap.FileRange, memCgID uint32) { if !fr.WellFormed() || fr.Length() == 0 || !hostarch.IsPageAligned(fr.Start) || !hostarch.IsPageAligned(fr.End) {