diff --git a/Makefile b/Makefile index 179a3ef1..4608b40f 100644 --- a/Makefile +++ b/Makefile @@ -148,7 +148,11 @@ OBJECTS := \ firmware.o \ gxf.o gxf_asm.o \ heapblock.o \ - hv.o hv_vm.o hv_exc.o hv_vuart.o hv_wdt.o hv_asm.o hv_aic.o hv_t8140.o hv_virtio.o \ + hv.o hv_vm.o hv_exc.o hv_sptm.o \ + hv_sptm_boot.o hv_sptm_core.o hv_sptm_dart.o hv_sptm_dart_power.o \ + hv_sptm_frame.o hv_sptm_init.o hv_sptm_nvme.o hv_sptm_pmap.o \ + hv_sptm_sart.o hv_sptm_txm.o hv_sptm_uat.o hv_sptm_xnu.o \ + hv_vuart.o hv_wdt.o hv_asm.o hv_aic.o hv_power_register_guard.o hv_virtio.o \ i2c.o \ iodev.o \ iova.o \ @@ -167,6 +171,7 @@ OBJECTS := \ proxy.o \ ringbuffer.o \ rtkit.o \ + sapt.o \ sart.o \ sep.o \ sio.o \ diff --git a/proxyclient/m1n1/hv/__init__.py b/proxyclient/m1n1/hv/__init__.py index 784fdc99..b8fe6b24 100644 --- a/proxyclient/m1n1/hv/__init__.py +++ b/proxyclient/m1n1/hv/__init__.py @@ -7,7 +7,7 @@ from ..proxy import IODEV, START, EVENT, EXC, EXC_RET, ExcInfo from ..utils import * from ..sysreg import * -from ..macho import MachO +from ..macho import MachO, MachOLoadCmdType from ..adt import load_adt from .. import xnutools, shell @@ -21,13 +21,17 @@ class HV(Reloadable): PAC_MASK = 0xfffff00000000000 + # Apple AMCC read-only region bounds, encoded as trapped sysreg tuples. + AMCC_RORGN_BEGIN = (3, 0, 11, 1, 2) + AMCC_RORGN_END = (3, 0, 11, 1, 3) + PTE_VALID = 1 << 0 PTE_MEMATTR_UNCHANGED = 0b1111 << 2 PTE_S2AP_RW = 0b11 << 6 - PTE_SH_NS = 0b11 << 8 + PTE_SH_IS = 0b11 << 8 PTE_ACCESS = 1 << 10 - PTE_ATTRIBUTES = PTE_ACCESS | PTE_SH_NS | PTE_S2AP_RW | PTE_MEMATTR_UNCHANGED + PTE_ATTRIBUTES = PTE_ACCESS | PTE_SH_IS | PTE_S2AP_RW | PTE_MEMATTR_UNCHANGED SPTE_TRACE_READ = 1 << 63 SPTE_TRACE_WRITE = 1 << 62 @@ -74,10 +78,12 @@ class HV(Reloadable): AIC_EVT_TYPE_HW = 1 IRQTRACE_IRQ = 1 - def __init__(self, iface, proxy, utils): + def __init__(self, iface, proxy, utils, *, verbose=False): self.iface = iface self.p = proxy self.u = utils + self.verbose = verbose + self.verbose_print = print if verbose else lambda *args, **kwargs: None self.pac_mask = self.PAC_MASK self.user_pac_mask = self.PAC_MASK self.vbar_el1 = None @@ -91,6 +97,7 @@ def __init__(self, iface, proxy, utils): self.symbols = [] self.symbol_dict = {} self.sysreg = {} + self.sysreg_ro_overrides = {} self.novm = False self._in_handler = False self._sigint_pending = False @@ -118,6 +125,8 @@ def __init__(self, iface, proxy, utils): self.switching_context = False self.show_timestamps = False self.virtio_devs = {} + self.sptm_symbols = None + self.sptm_panic_state_pa = 0 def _reloadme(self): super()._reloadme() @@ -140,18 +149,18 @@ def _update_shell_locals(self): self.shell_locals["ctx"] = self.context - def log(self, s, *args, show_cpu=True, **kwargs): + def log(self, s, *args, show_cpu=True, console=True, **kwargs): + prefix = "" if self.ctx is not None and show_cpu: ts="" if self.show_timestamps: ts = f"[{self.u.mrs(CNTPCT_EL0):#x}]" - print(ts+f"[cpu{self.ctx.cpu_id}] " + s, *args, **kwargs) - if self.print_tracer.log_file: - print(f"# {ts}[cpu{self.ctx.cpu_id}] " + s, *args, file=self.print_tracer.log_file, **kwargs) - else: - print(s, *args, **kwargs) - if self.print_tracer.log_file: - print("# " + s, *args, file=self.print_tracer.log_file, **kwargs) + prefix = ts + f"[cpu{self.ctx.cpu_id}] " + if console: + print(prefix + s, *args, **kwargs) + if self.print_tracer.log_file: + print("# " + prefix + s, *args, + file=self.print_tracer.log_file, **kwargs) def unmap(self, ipa, size): assert self.p.hv_map(ipa, 0, size, 0) >= 0 @@ -300,6 +309,8 @@ def pt_update(self): self.mmio_maps.compact() top = 0 + mapped_regions = 0 + unmapped_regions = 0 for zone in self.dirty_maps: if zone.stop <= top: @@ -311,7 +322,9 @@ def pt_update(self): continue if top < mzone.start: self.unmap(top, mzone.start - top) - self.log(f"PT[{top:09x}:{mzone.start:09x}] -> *UNMAPPED*") + unmapped_regions += 1 + self.log(f"PT[{top:09x}:{mzone.start:09x}] -> *UNMAPPED*", + console=self.verbose) top = mzone.stop if not maps: @@ -323,7 +336,12 @@ def pt_update(self): need_write = any(m[3] for m in maps) if mode == TraceMode.RESERVED: - self.log(f"PT[{mzone.start:09x}:{mzone.stop:09x}] -> RESERVED {ident}") + mapped_regions += 1 + self.log( + f"PT[{mzone.start:09x}:{mzone.stop:09x}] -> " + f"RESERVED {ident}", + console=self.verbose, + ) continue elif mode in (TraceMode.HOOK, TraceMode.SYNC): self.map_hook_idx(mzone.start, mzone.stop - mzone.start, 0, @@ -347,7 +365,12 @@ def pt_update(self): self.map_sw(mzone.start, pa, mzone.stop - mzone.start) elif mode == TraceMode.OFF: self.map_hw(mzone.start, mzone.start, mzone.stop - mzone.start) - self.log(f"PT[{mzone.start:09x}:{mzone.stop:09x}] -> HW:{ident}") + mapped_regions += 1 + self.log( + f"PT[{mzone.start:09x}:{mzone.stop:09x}] -> " + f"HW:{ident}", + console=self.verbose, + ) continue rest = [m[1] for m in maps[1:] if m[0] != TraceMode.OFF] @@ -356,14 +379,27 @@ def pt_update(self): else: rest = "" - self.log(f"PT[{mzone.start:09x}:{mzone.stop:09x}] -> {mode.name}.{'R' if read else ''}{'W' if read else ''} {ident}{rest}") + mapped_regions += 1 + self.log( + f"PT[{mzone.start:09x}:{mzone.stop:09x}] -> " + f"{mode.name}.{'R' if read else ''}" + f"{'W' if read else ''} {ident}{rest}", + console=self.verbose, + ) if top < zone.stop: self.unmap(top, zone.stop - top) - self.log(f"PT[{top:09x}:{zone.stop:09x}] -> *UNMAPPED*") + unmapped_regions += 1 + self.log(f"PT[{top:09x}:{zone.stop:09x}] -> *UNMAPPED*", + console=self.verbose) self.u.inst(0xd50c83df) # tlbi vmalls12e1is self.dirty_maps.clear() + self.log( + f"Updated page tables: {mapped_regions + unmapped_regions} " + f"regions ({mapped_regions} mapped/reserved, " + f"{unmapped_regions} unmapped)" + ) def shellwrap(self, func, description, update=None, needs_ret=False): @@ -684,6 +720,16 @@ def handle_msr(self, ctx, iss=None): self.log("Guest is shutting down CPU") self.p.hv_exit_cpu() del self.started_cpus[self.ctx.cpu_id] + elif enc in self.sysreg_ro_overrides: + value = self.sysreg_ro_overrides[enc] + if iss.DIR == MSR_DIR.READ: + self.log(f"Virtual: mrs x{iss.Rt}, {name} = {value:x}") + if iss.Rt != 31: + ctx.regs[iss.Rt] = value + else: + if iss.Rt != 31: + value = ctx.regs[iss.Rt] + self.log(f"Skip: msr {name}, x{iss.Rt} = {value:x}") elif enc in shadow: if iss.DIR == MSR_DIR.READ: value = self.sysreg[self.ctx.cpu_id].setdefault(enc, 0) @@ -745,6 +791,10 @@ def handle_impdef(self, ctx): def handle_hvc(self, ctx): idx = ctx.esr.ISS + if self.sptm_symbols is not None and ( + idx <= 4 or 0x1c0 <= idx < 0x280 or 0x400 <= idx < 0xac0 + ): + return False if idx == 0: return False @@ -780,15 +830,7 @@ def handle_hvc(self, ctx): if elr_phys: self.u.disassemble_at(elr_phys - 4 * 4, 9 * 4, elr - 4 * 4, elr, sym=self.get_sym) if self.sym(elr)[1] == "com.apple.kernel:_panic_trap_to_debugger": - self.log("Panic! Trying to decode panic...") - try: - self.decode_panic_call() - except: - self.log("Error decoding panic.") - try: - self.bt() - except: - pass + self.report_xnu_panic() return False if esr.EC == ESR_EC.UNKNOWN: instr = self.p.read32(elr_phys) @@ -963,6 +1005,7 @@ def handle_exception(self, reason, code, info): handled = False user_interrupt = False + xnu_panic = False try: if reason == START.EXCEPTION_LOWER: @@ -979,6 +1022,18 @@ def handle_exception(self, reason, code, info): elif code == HV_EVENT.USER_INTERRUPT: handled = True user_interrupt = True + elif code == HV_EVENT.XNU_PANIC: + handled = True + xnu_panic = True + owner = domain = None + if self.sptm_panic_state_pa: + state = self.iface.readmem(self.sptm_panic_state_pa, 16) + owner = struct.unpack_from("= IODEV.USB0 and iodev != self.iodev: - print(f"Disable iodev {iodev!s}") + self.verbose_print(f"Disable iodev {iodev!s}") self.p.iodev_set_usage(iodev, 0) print("Initializing hypervisor over iodev %s" % self.iodev) @@ -1392,12 +1500,18 @@ def init(self): self.iface.set_handler(START.HV, HV_EVENT.CPU_SWITCH, self.handle_exception) self.iface.set_handler(START.HV, HV_EVENT.VIRTIO, self.handle_virtio) self.iface.set_handler(START.HV, HV_EVENT.PANIC, self.handle_bark) + self.iface.set_handler(START.HV, HV_EVENT.XNU_PANIC, self.handle_exception) self.iface.set_event_handler(EVENT.MMIOTRACE, self.handle_mmiotrace) self.iface.set_event_handler(EVENT.IRQTRACE, self.handle_irqtrace) # Map MMIO ranges as HW by default - for r in self.adt["/arm-io"].ranges: - print(f"Mapping MMIO range: {r.parent_addr:#x} .. {r.parent_addr + r.size:#x}") + mmio_ranges = list(self.adt["/arm-io"].ranges) + print(f"Registering {len(mmio_ranges)} MMIO ranges") + for r in mmio_ranges: + self.verbose_print( + f"Mapping MMIO range: {r.parent_addr:#x} .. " + f"{r.parent_addr + r.size:#x}" + ) self.add_tracer(irange(r.parent_addr, r.size), "HW", TraceMode.OFF) hcr = HCR(self.u.mrs(HCR_EL2)) @@ -1406,7 +1520,6 @@ def init(self): hcr.AMO = 0 else: hcr.TACR = 1 - hcr.TIDCP = 0 hcr.TVM = 0 hcr.FMO = 1 hcr.IMO = 0 @@ -1418,7 +1531,9 @@ def init(self): if not self.novm: #hacr.TRAP_CPU_EXT = 1 #hacr.TRAP_SPRR = 1 - #hacr.TRAP_GXF = 1 + # Route raw-mode GXF save/restore accesses to the target-side shadows. + if not self.u.cpu_features.apple_sysregs_unlocked: + hacr.TRAP_GXF = 1 hacr.TRAP_CTRR = 1 hacr.TRAP_EHID = 1 hacr.TRAP_HID = 1 @@ -1637,6 +1752,9 @@ def setup_adt(self): # to re-enable XNU serial output for newer macOS versions self.adt["defaults"].serial_device = getattr(self.adt["/arm-io/uart0"], "AAPL,phandle") + adt_candidates = 0 + adt_removed = 0 + if self.iodev >= IODEV.USB0: idx = self.iodev - IODEV.USB0 for prefix in ("/arm-io/dart-usb%d", @@ -1659,28 +1777,43 @@ def setup_adt(self): "/arm-io/atc-phy%d", ): name = prefix % idx - print(f"Removing ADT node {name}") + adt_candidates += 1 + self.verbose_print(f"Removing ADT node {name}") try: del self.adt[name] except KeyError: pass + else: + adt_removed += 1 if self.wdt_cpu is not None: name = f"/cpus/cpu{self.wdt_cpu}" - print(f"Removing ADT node {name}") + adt_candidates += 1 + self.verbose_print(f"Removing ADT node {name}") try: del self.adt[name] except KeyError: pass + else: + adt_removed += 1 if not self.smp: for cpu in list(self.adt["cpus"]): if cpu.state != "running": - print(f"Removing ADT node {cpu._path}") + adt_candidates += 1 + self.verbose_print(f"Removing ADT node {cpu._path}") try: del self.adt["cpus"][cpu.name] except KeyError: pass + else: + adt_removed += 1 + + if adt_candidates: + print( + f"ADT pruning: removed {adt_removed} of " + f"{adt_candidates} candidate nodes" + ) def set_bootargs(self, boot_args): if "-v" in boot_args.split(): @@ -1743,7 +1876,7 @@ def load_raw(self, image, entryoffset=0x800, use_xnu_symbols=False, vmin=0): print(f"Physical memory: 0x{phys_base:x} .. 0x{mem_top:x}") print(f"Guest region start: 0x{guest_base:x}") - + self.entry = guest_base + entryoffset print(f"Mapping guest physical memory...") @@ -1834,6 +1967,8 @@ def load_macho(self, data, symfile=None): self.xnu_mode = True self.macho = macho = MachO(data) + if "com.apple.kernel" in macho.subfiles: + macho.add_fileset_symbols("com.apple.kernel") if symfile is not None: if isinstance(symfile, str): symfile = open(symfile, "rb") @@ -1933,71 +2068,169 @@ def load_hook_m4(data, segname, size, fileoff, dest): p += 1 continue opcode = a[p // 4] - if 0x00201420 <= opcode <= 0x00201424: # genter #imm (imm 0..4) + if opcode == 0x00201420: + # The dispatch ABI is entirely carried in x16. a[p // 4] = self.hvc(0) + elif opcode == 0x00201424: + # genter #SPTM_GENTER_XNU_PANIC_BEGIN + a[p // 4] = self.hvc(4) p += 4 - # sysreg neutralization: NOP writes to guarded impdef regs and - # SME control register + hvc_sysregs = {} + + def sysreg_opcode(read, op0, op1, crn, crm, op2): + opcode = 0xd5300000 if read else 0xd5100000 + return ( + opcode + | (op0 << 19) + | (op1 << 16) + | (crn << 12) + | (crm << 8) + | (op2 << 5) + ) + + # Route locked PMCR0 accesses through the target-side model. + if not self.u.cpu_features.apple_sysregs_unlocked: + hvc_sysregs.update({ + 0xd519f000: 0x380, + 0xd539f000: 0x3c0, + }) + # These CLPC sysregs are undefined at EL1 when Apple sysregs are locked. + # Lower counter reads to CNTPCT and route other accesses to EL2. + clpc_regs = [ + (3, 7, 15, 0, 3), # CORE_PERF0 + (3, 7, 15, 1, 3), # CORE_PERF_ENABLE0 + (3, 7, 15, 2, 3), # CORE_PERF1 + (3, 7, 15, 3, 3), # CORE_PERF_ENABLE1 + (3, 7, 15, 4, 3), # CORE_PERF2 + (3, 7, 15, 5, 3), # CORE_PERF_ENABLE2 + (3, 7, 15, 6, 3), # CORE_PERF3 + (3, 7, 15, 7, 3), # CORE_PERF_ENABLE3 + (3, 7, 15, 8, 3), # CORE_PERF4 + (3, 7, 15, 9, 3), # CORE_PERF_ENABLE4 + (3, 7, 15, 10, 3), # CORE_PERF5 + (3, 7, 15, 11, 3), # CORE_PERF_ENABLE5 + (3, 7, 15, 12, 3), # CORE_PERF6 + (3, 7, 15, 13, 3), # CORE_PERF_ENABLE6 + (3, 7, 15, 14, 3), # CORE_PERF7 + (3, 7, 15, 15, 3), # CORE_PERF_ENABLE7 + (3, 7, 15, 5, 0), # CORE_PERF_CTRL2 + (3, 1, 15, 8, 3), # CORE_ACC0 + (3, 1, 15, 9, 3), # CORE_ACC1 + (3, 1, 15, 10, 3), # CORE_ACC2 + (3, 1, 15, 11, 3), # CORE_ACC3 + (3, 1, 15, 12, 3), # CORE_ACC4 + (3, 1, 15, 0, 3), # MEM_STALL_CFG0 + (3, 1, 15, 1, 3), # MEM_STALL_CFG1 + (3, 1, 15, 2, 3), # MEM_STALL_CFG2 + ] + clpc_counter_sysregs = set() + for reg_index, encoding in enumerate(clpc_regs): + for is_read in (False, True): + is_counter = ( + (reg_index < 16 and not (reg_index & 1)) or + 17 <= reg_index <= 21 + ) + if is_read and is_counter: + clpc_counter_sysregs.add( + sysreg_opcode(True, *encoding) + ) + continue + operation = reg_index * 2 + int(is_read) + hvc_sysregs[sysreg_opcode(is_read, *encoding)] = ( + 0x400 + (operation << 5) + ) + + hvc_sysregs.update({ + 0xd53ef1a0: 0x1c0, # mrs SPRR_PERM_EL0 + 0xd51ef1a0: 0x1e0, # msr SPRR_PERM_EL0 + 0xd53ef1c0: 0x200, # mrs SPRR_PERM_EL1 + 0xd51ef1c0: 0x220, # msr SPRR_PERM_EL1 + }) + + # Route undefined ObjC breakpoint registers through the C shadows. + hvc_sysregs.update({ + 0xd51ef080: 0xa40, # msr S3_6_C15_C0_4 + 0xd53ef080: 0xa60, # mrs S3_6_C15_C0_4 + 0xd51ef0a0: 0xa80, # msr S3_6_C15_C0_5 + 0xd53ef0a0: 0xaa0, # mrs S3_6_C15_C0_5 + }) + + # NOP writes to guarded impdef and SME control registers. nop_msr = [ 0xd51cf100, # msr KERNKEYLO_EL1 (s3_4_c15_c1_0) 0xd51cf120, # msr KERNKEYHI_EL1 (s3_4_c15_c1_1) 0xd51cfdc0, # msr s3_4_c15_c13_6 (Apple PAC key) 0xd51cfde0, # msr s3_4_c15_c13_7 (Apple PAC key) + 0xd519f1a0, # msr AGTCNTRDIR_EL1 (s3_1_c15_c1_5) + 0xd51cfec0, # msr AGTCNTRDIR_EL12 (s3_4_c15_c14_6) + 0xd51cf180, # msr AMX_CTL_EL1 (s3_4_c15_c1_4) + 0xd519f740, # msr alternate PMCR1 (s3_1_c15_c7_2) 0xd51c12a0, # msr SMPRIMAP_EL2 (s3_4_c1_c2_5) + 0xd51cfc00, # msr guarded c12 state (s3_4_c15_c12_0) + 0xd519fd80, # msr Apple physical timer ctl (s3_1_c15_c13_4) + 0xd51df2c0, # msr WATCHDOGDIAG0 (s3_5_c15_c2_6) + 0xd51cffc0, # msr JCTL_EL0 (s3_4_c15_c15_6) + 0xd51ef300, # msr SPRR_UMPRR_EL1 (s3_6_c15_c3_0) ] - p = 0 - while (p := data.find(b"\x1c\xd5", p)) != -1: - if (p & 3) != 2: - p += 1 + for word_index, instruction in enumerate(a): + opcode = instruction & ~0x1f + if opcode in clpc_counter_sysregs: + rt = instruction & 0x1f + a[word_index] = ( + sysreg_opcode(True, 3, 3, 14, 0, 1) | rt + ) + continue + if opcode == 0xd53ef300: + # Avoid an EL2 round trip for each page-policy read. + rt = instruction & 0x1f + a[word_index] = 0xaa1f03e0 | rt # mov xRt, xzr + continue + if opcode in hvc_sysregs: + rt = instruction & 0x1f + a[word_index] = self.hvc(hvc_sysregs[opcode] | rt) continue - opcode = a[p // 4] & ~0x1f if opcode in nop_msr: - if opcode == 0xd51c12a0: - print(f" 0x{seg_vmaddr + (p & ~3):x}: SMPRIMAP_EL2 -> noop") - a[p // 4] = nop - p += 4 - - # commpage: force XNU to publish VM-safe userspace feature policy - # bytes. The Apple timebase path and HW-TPRO/SPRR path use - # userspace-inaccessible Apple registers under this EL1 guest. - # - # hard coded kernelcache addresses is a hack we should find a way - # to avoid - off = 0xfffffe000b5a3330 - seg_vmaddr - if 0 <= off < size: - if a[off // 4] != 0x52800068: - raise RuntimeError("_COMM_PAGE_USER_TIMEBASE patch target mismatch") - a[off // 4] = 0x52800008 - print(f" 0xfffffe000b5a3330: _COMM_PAGE_USER_TIMEBASE=0") - - off = 0xfffffe000b5a3388 - seg_vmaddr - if 0 <= off < size: - if a[off // 4] != 0x52800028: - raise RuntimeError("_COMM_PAGE_CONT_HWCLOCK patch target mismatch") - a[off // 4] = 0x52800008 - print(f" 0xfffffe000b5a3388: _COMM_PAGE_CONT_HWCLOCK=0") - - off = 0xfffffe000b5a358c - seg_vmaddr - if 0 <= off < size: - if a[off // 4] != 0x39043128: - raise RuntimeError("_COMM_PAGE_HW_TPRO patch target mismatch") - a[off // 4] = 0x3904313f - print(f" 0xfffffe000b5a358c: _COMM_PAGE_HW_TPRO=0") + a[word_index] = nop print("Done.") return a.tobytes() chip_id = self.u.adt["/chosen"].chip_id if chip_id in (0x8122, 0x6030, 0x6031, 0x6032, 0x6034): - image = macho.prepare_image(load_hook_m3) - elif chip_id in (0x8132, 0x8140, 0x6040, 0x6041): - # M4 / A18 Pro class: T8132, T8140, T6040 (M4 Pro), T6041 (M4 Max) - image = macho.prepare_image(load_hook_m4) + image = macho.prepare_image(load_hook_m3, verbose=self.verbose) + elif not self.u.cpu_features.apple_sysregs_unlocked: + image = macho.prepare_image(load_hook_m4, verbose=self.verbose) else: - image = macho.prepare_image() + image = macho.prepare_image(verbose=self.verbose) self.load_raw(image, entryoffset=(macho.entry - macho.vmin), use_xnu_symbols=self.xnu_mode and symfile is not None, vmin=macho.vmin) + if not self.u.cpu_features.apple_sysregs_unlocked: + writable_segments = [ + cmd.args for cmd in macho.get_cmds(MachOLoadCmdType.SEGMENT_64) + if cmd.args.initprot.PROT_WRITE + ] + if not writable_segments: + raise RuntimeError("Mach-O has no writable segment after its protected prefix") + + writable_start = min(seg.vmaddr for seg in writable_segments) + ro_start = self.guest_base + ro_end = self.guest_base + writable_start - macho.vmin + if ro_end <= ro_start or (ro_start & 0xfff) or (ro_end & 0xfff): + raise RuntimeError( + f"Guest AMCC read-only range is not page-aligned: " + f"0x{ro_start:x}..0x{ro_end:x}" + ) + + self.sysreg_ro_overrides.update({ + self.AMCC_RORGN_BEGIN: ro_start, + self.AMCC_RORGN_END: ro_end - 0x1000, + }) + print( + f"Guest AMCC read-only range: 0x{ro_start:x}..0x{ro_end:x} " + "(virtualized)" + ) + def update_pac_mask(self): tcr = TCR(self.u.mrs(TCR_EL12)) valid_bits = (1 << (64 - tcr.T1SZ)) - 1 @@ -2053,11 +2286,14 @@ def run_code(self, code): exec(code, self.shell_locals) def start(self): - print("Disabling other iodevs...") - for iodev in IODEV: - if iodev != self.iodev: - print(f" - {iodev!s}") - self.p.iodev_set_usage(iodev, 0) + other_iodevs = [iodev for iodev in IODEV if iodev != self.iodev] + if self.verbose: + print("Disabling other iodevs...") + else: + print(f"Disabling {len(other_iodevs)} other iodevs...") + for iodev in other_iodevs: + self.verbose_print(f" - {iodev!s}") + self.p.iodev_set_usage(iodev, 0) print("Doing essential MMIO remaps...") self.map_essential() @@ -2070,6 +2306,11 @@ def start(self): assert len(adt_blob) <= align(self.u.ba.devtree_size) self.iface.writemem(self.adt_base, adt_blob) + if self.sptm_symbols is not None: + self.sptm_panic_state_pa = self.p.hv_sptm_init( + self.adt_base, *self.sptm_symbols + ) + print("Improving logo...") self.p.fb_improve_logo() diff --git a/proxyclient/m1n1/hv/types.py b/proxyclient/m1n1/hv/types.py index 4de7d374..605b432e 100644 --- a/proxyclient/m1n1/hv/types.py +++ b/proxyclient/m1n1/hv/types.py @@ -39,6 +39,7 @@ class HV_EVENT(IntEnum): CPU_SWITCH = 5 VIRTIO = 6 PANIC = 7 + XNU_PANIC = 8 VMProxyHookData = Struct( "flags" / RegAdapter(MMIOTraceFlags), diff --git a/proxyclient/m1n1/macho.py b/proxyclient/m1n1/macho.py index bcb603b6..82b02075 100644 --- a/proxyclient/m1n1/macho.py +++ b/proxyclient/m1n1/macho.py @@ -146,16 +146,26 @@ def load_info(self): elif cmd.cmd == MachOLoadCmdType.UNIXTHREAD: self.entry = cmd.args[0].data.pc - def prepare_image(self, load_hook=None): + def prepare_image(self, load_hook=None, *, verbose=False): memory_size = self.vmax - self.vmin + verbose_print = print if verbose else lambda *args, **kwargs: None image = bytearray(memory_size) + segment_count = 0 + loaded_size = 0 + zeroed_size = 0 + skipped_size = 0 for cmd in self.get_cmds(MachOLoadCmdType.SEGMENT_64): + segment_count += 1 dest = cmd.args.vmaddr - self.vmin end = min(self.size, cmd.args.fileoff + cmd.args.filesize) size = end - cmd.args.fileoff - print(f"LOAD: {cmd.args.segname} {size} bytes from {cmd.args.fileoff:x} to {dest:x}") + loaded_size += size + verbose_print( + f"LOAD: {cmd.args.segname} {size} bytes from " + f"{cmd.args.fileoff:x} to {dest:x}" + ) self.io.seek(self.off + cmd.args.fileoff) data = self.io.read(size) if load_hook is not None: @@ -164,13 +174,30 @@ def prepare_image(self, load_hook=None): if cmd.args.vmsize > size: clearsize = cmd.args.vmsize - size if cmd.args.segname == "PYLD": - print("SKIP: %d bytes from 0x%x to 0x%x" % (clearsize, dest + size, dest + size + clearsize)) + skipped_size += clearsize + verbose_print( + "SKIP: %d bytes from 0x%x to 0x%x" % + (clearsize, dest + size, dest + size + clearsize) + ) memory_size -= clearsize - 4 # leave a payload end marker image = image[:memory_size] else: - print("ZERO: %d bytes from 0x%x to 0x%x" % (clearsize, dest + size, dest + size + clearsize)) + zeroed_size += clearsize + verbose_print( + "ZERO: %d bytes from 0x%x to 0x%x" % + (clearsize, dest + size, dest + size + clearsize) + ) image[dest + size:dest + cmd.args.vmsize] = bytes(clearsize) + summary = ( + f"Prepared Mach-O image: {segment_count} segments, " + f"0x{len(image):x} bytes (0x{loaded_size:x} loaded, " + f"0x{zeroed_size:x} zero-filled" + ) + if skipped_size: + summary += f", 0x{skipped_size:x} skipped" + print(summary + ")") + return image def get_cmds(self, cmdtype): @@ -223,14 +250,26 @@ def add_symbols(self, filename, syms, demangle=False): sname = f"{filename}:{sym}" self.symbols[sname] = addr - sym_seg.args.vmaddr + seg.args.vmaddr - def load_symbols(self, demangle=False): + def add_fileset_symbols(self, filename, demangle=False): + subfile = self.subfiles[filename] + if not list(subfile.get_cmds(MachOLoadCmdType.SYMTAB)): + return + + subfile.load_symbols(demangle=demangle, file_base=self.off) + for name, addr in subfile.symbols.items(): + self.symbols[f"{filename}:{name}"] = addr + + def load_symbols(self, demangle=False, file_base=None): self.symbols = {} + if file_base is None: + file_base = self.off + cmd = self.get_cmd(MachOLoadCmdType.SYMTAB) nsyms = cmd.args.nsyms length = NList.sizeof() * nsyms - self.io.seek(self.off + cmd.args.symoff) + self.io.seek(file_base + cmd.args.symoff) symdata = self.io.read(length) symbols = Array(nsyms, NList).parse(symdata) @@ -238,7 +277,7 @@ def load_symbols(self, demangle=False): symbols_dict = {} for i in symbols: off = cmd.args.stroff + i.n_strx - self.io.seek(self.off + off) + self.io.seek(file_base + off) name = self.io.read(1024).split(b"\x00")[0].decode("ascii") symbols_dict[name] = i.n_value diff --git a/proxyclient/m1n1/proxy.py b/proxyclient/m1n1/proxy.py index ef72f8ca..dca3b6dd 100644 --- a/proxyclient/m1n1/proxy.py +++ b/proxyclient/m1n1/proxy.py @@ -514,6 +514,8 @@ class GUARD(IntFlag): "actlr_el2" / bool_, "counter_redirect" / bool_, "sme_enabled" / bool_, + "sapt" / bool_, + Padding(3), ) # Uses UartInterface.proxyreq() to send requests to M1N1 and process @@ -661,6 +663,7 @@ class M1N1Proxy(Reloadable): P_VIRTIO_PUT_BUFFER = 0xc0e P_HV_EXIT_CPU = 0xc0f P_HV_ADD_TIME = 0xc10 + P_HV_SPTM_INIT = 0xc11 P_FB_INIT = 0xd00 P_FB_SHUTDOWN = 0xd01 @@ -1151,7 +1154,14 @@ def hv_exit_cpu(self, cpu=-1): return self.request(self.P_HV_EXIT_CPU, cpu) def hv_add_time(self, time): return self.request(self.P_HV_ADD_TIME, time) - + def hv_sptm_init(self, guest_adt, cons_ops, page_shift_const, xnu_text): + return self.request( + self.P_HV_SPTM_INIT, + guest_adt, + cons_ops, + page_shift_const, + xnu_text, + ) def fb_init(self): return self.request(self.P_FB_INIT) def fb_shutdown(self, restore_logo=True): diff --git a/proxyclient/m1n1/shell.py b/proxyclient/m1n1/shell.py index 277be68b..fd71b7ac 100644 --- a/proxyclient/m1n1/shell.py +++ b/proxyclient/m1n1/shell.py @@ -27,7 +27,7 @@ def init_history(self, histfile): readline.read_history_file(histfile) except FileNotFoundError: pass - except PermissionError as e: + except OSError as e: print(f"Failed reading history from {histfile}: {e}", file=sys.stderr) if sys.platform == "darwin": print(f" On macOS this may be caused by extended attributes. " diff --git a/proxyclient/m1n1/sysreg.py b/proxyclient/m1n1/sysreg.py index ed3a37a1..3cb3e48d 100644 --- a/proxyclient/m1n1/sysreg.py +++ b/proxyclient/m1n1/sysreg.py @@ -86,6 +86,7 @@ class ESR_EC(IntEnum): MSR = 0b011000 SVE = 0b011001 PAUTH_FAIL = 0b011100 + SME = 0b011101 IABORT_LOWER = 0b100000 IABORT = 0b100001 PC_ALIGN = 0b100010 diff --git a/proxyclient/tools/chainload.py b/proxyclient/tools/chainload.py index dee2fe66..5c708fc7 100755 --- a/proxyclient/tools/chainload.py +++ b/proxyclient/tools/chainload.py @@ -12,6 +12,8 @@ parser.add_argument('-r', '--raw', action="store_true", help="Image is raw") parser.add_argument('-E', '--entry-point', action="store", type=int, help="Entry point for the raw image", default=0x800) parser.add_argument('-x', '--xnu', action="store_true", help="Set up for chainloading XNU") +parser.add_argument('--verbose', action="store_true", + help="Show individual Mach-O segment entries") parser.add_argument('payload', type=pathlib.Path) parser.add_argument('boot_args', default=[], nargs="*") args = parser.parse_args() @@ -29,7 +31,7 @@ entry = new_base + args.entry_point else: macho = MachO(args.payload.read_bytes()) - image = macho.prepare_image() + image = macho.prepare_image(verbose=args.verbose) image += b"\x00\x00\x00\x00" entry = macho.entry entry -= macho.vmin @@ -55,9 +57,7 @@ bootargs_size = 0x4000 image_size += bootargs_size -# On M4/A18, iBoot adds a 48 KiB region ending at top_of_kernel_data -# that secondary CPUs can't write to. This hack avoids using that region -# by making the next m1n1's heap start above it +# Avoid the M4/A18 secondary-write-protected region below top_of_kernel_data. protected_start = u.ba.top_of_kernel_data - 3 * 0x4000 image_end = new_base + image_size if image_end > protected_start: diff --git a/proxyclient/tools/run_guest.py b/proxyclient/tools/run_guest.py index 2cd61108..ef25e38d 100755 --- a/proxyclient/tools/run_guest.py +++ b/proxyclient/tools/run_guest.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-or-later -import sys, pathlib, traceback +import argparse +import pathlib +import sys +import traceback + sys.path.append(str(pathlib.Path(__file__).resolve().parents[1])) -import argparse, pathlib from io import BytesIO def volumespec(s): @@ -20,8 +23,14 @@ def boot_arg_key(arg): "wdt=-1", # disables some internal XNU watchdog "sprr_tpro=0", # disable XNU's TPRO boot policy; the commpage bit is patched separately "sprr_tpro_pagers=0", # ... same, for pager mappings + "wfi=0", # raw M4 WFI can lose PE state without CYC_OVRD + "cluster_power=0", # EL2 owns live cluster state until warm power-up is emulated + "processor_exit=0", # keep every booted logical CPU available to the scheduler + "vm_compressor=0", # avoid WKDMC instruction that faults in raw mode "-v", # Optional: verbose boot f"msgbuf={1024 * 1024}", # Optional: enlarge the kernel msgbuf + "amfi_get_out_of_my_way=1", # Optional: allow guest tooling to run + "amfi_allow_only_tc_override=2", ] for arg in defaults: key = boot_arg_key(arg) @@ -37,6 +46,9 @@ def boot_arg_key(arg): parser.add_argument('-e', '--hook-exceptions', action="store_true") parser.add_argument('-d', '--debug-xnu', action="store_true") parser.add_argument('-l', '--logfile', type=pathlib.Path) +parser.add_argument('--verbose', action='store_true', + help='Show detailed launch progress, including individual ' + 'MMIO, ADT, page-table, and Mach-O segment entries.') parser.add_argument('-C', '--cpus', default=None) parser.add_argument('--strip-node', action="append", default=[], metavar='SUBSTR', help='Remove every ADT node whose name contains SUBSTR.') @@ -63,7 +75,7 @@ def boot_arg_key(arg): iface = UartInterface() p = M1N1Proxy(iface, debug=False) bootstrap_port(iface, p) -u = ProxyUtils(p, heap_size = 128 * 1024 * 1024) +u = ProxyUtils(p, heap_size=128 * 1024 * 1024) # Setup counter redirect / AHCR_EL2 as expected by macOS for macho payloads if not args.raw: @@ -72,7 +84,7 @@ def boot_arg_key(arg): u.msr(AGTCNTRDIR_EL1, 3) u.msr(AGTCNTRDIR_EL12, 3) -hv = HV(iface, p, u) +hv = HV(iface, p, u, verbose=args.verbose) hv.hook_exceptions = args.hook_exceptions @@ -105,17 +117,15 @@ def strip_nodes(node, path=""): if args.debug_xnu: hv.adt["chosen"].debug_enabled = 1 -# Exclaves are not yet supported -if not args.raw and u.adt["/chosen"].chip_id in (0x8132, 0x8140, 0x6040, 0x6041): +# Exclaves are not yet supported by the locked-Apple-sysreg guest path. +if not args.raw and not u.cpu_features.apple_sysregs_unlocked: for name in ("/arm-io/exdisplaypipe", "/arm-io/exdisplaypipe-s-proxy", "/arm-io/dcp-exclave-ioreporting", "/arm-io/dcp-exclave-mailbox"): try: del hv.adt[name] except KeyError: pass - # The internal DCP's iop-dcp-nub binds its RTKit transport to the (now-gone) - # dcp-exclave-mailbox via routes=206; clearing routes makes the DCP firmware - # fall back to the plain ASC mailbox like the routeless external dcpext. + # Clear the stale exclave route so DCP falls back to its ASC mailbox. try: nub = hv.adt["/arm-io/dcp/iop-dcp-nub"] if getattr(nub, "routes", None) is not None: @@ -130,8 +140,8 @@ def strip_nodes(node, path=""): if args.logfile: hv.set_logfile(args.logfile.open("w")) -# macOS-under-HV needs a specific boot-arg set on the M4/macOS chip_ids -if not args.raw and u.adt["/chosen"].chip_id in (0x8132, 0x8140, 0x6040, 0x6041): +# macOS-under-HV needs a specific boot-arg set when Apple sysregs are locked. +if not args.raw and not u.cpu_features.apple_sysregs_unlocked: hv.set_bootargs(sptm_hv_boot_args(args.boot_args)) elif len(args.boot_args) > 0: boot_args = " ".join(args.boot_args) @@ -156,6 +166,17 @@ def strip_nodes(node, path=""): else: hv.load_macho(payload, symfile=symfile) +if not args.raw and not u.cpu_features.apple_sysregs_unlocked: + sptm_symbols = tuple( + hv.symbol_dict[f"com.apple.kernel:{name}"] + for name in ( + "_cons_ops", + "_PAGE_SHIFT_CONST", + "__TEXT", + ) + ) + hv.sptm_symbols = sptm_symbols + PMU(u).reset_panic_counter() for i in args.script: diff --git a/src/arm_cpu_regs.h b/src/arm_cpu_regs.h index 5e2d8c0a..e535a0f7 100644 --- a/src/arm_cpu_regs.h +++ b/src/arm_cpu_regs.h @@ -25,6 +25,9 @@ #define CNTHCTL_EL0VCTEN BIT(1) #define CNTHCTL_EL0PCTEN BIT(0) +// governs EL0/EL1 counter access and event stream +#define SYS_CNTKCTL_EL12 sys_reg(3, 5, 14, 1, 0) + #define SYS_CNTV_CTL_EL0 sys_reg(3, 3, 14, 3, 1) #define SYS_CNTV_CTL_EL02 sys_reg(3, 5, 14, 3, 1) #define SYS_CNTP_CTL_EL0 sys_reg(3, 3, 14, 2, 1) @@ -68,6 +71,7 @@ #define ESR_EC_MSR 0b011000 #define ESR_EC_SVE 0b011001 #define ESR_EC_PAUTH_FAIL 0b011100 +#define ESR_EC_SME 0b011101 #define ESR_EC_IABORT_LOWER 0b100000 #define ESR_EC_IABORT 0b100001 #define ESR_EC_PC_ALIGN 0b100010 @@ -197,6 +201,10 @@ #define CPTR_EL2_ZEN GENMASK(17, 16) #define CPTR_EL2_ZEN_NONE (0x3UL << 16) +#define SYS_SMCR_EL2 sys_reg(3, 4, 1, 2, 6) +#define SMCR_EL2_EZT0 BIT(30) +#define SMCR_EL2_LEN GENMASK(3, 0) + #define SYS_HFGRTR_EL2 sys_reg(3, 4, 1, 1, 4) #define SYS_HFGWTR_EL2 sys_reg(3, 4, 1, 1, 5) #define HFGxTR_EL2_nTPIDR2_EL0 BIT(55) diff --git a/src/chickens.c b/src/chickens.c index 4360ed0b..60d2283e 100644 --- a/src/chickens.c +++ b/src/chickens.c @@ -114,6 +114,7 @@ const struct midr_part_features features_m4 = { .fast_ipi = true, .actlr_el2 = true, .sme_enabled = true, + .sapt = true, }; /* diff --git a/src/cpu_regs.h b/src/cpu_regs.h index 5ba266a1..a7f48619 100644 --- a/src/cpu_regs.h +++ b/src/cpu_regs.h @@ -27,6 +27,7 @@ #define SYS_IMP_APL_KERNEL_CNTV_TVAL_EL0 sys_reg(3, 1, 15, 15, 4) #define SYS_IMP_APL_KERNEL_CNTVCT_EL0 sys_reg(3, 4, 15, 11, 7) #define SYS_IMP_APL_KERNEL_CNTKCTL_EL1 sys_reg(3, 4, 15, 9, 6) +#define SYS_IMP_APL_S3_1_C15_C13_3 sys_reg(3, 1, 15, 13, 3) #define SYS_IMP_APL_JCTL_EL0 sys_reg(3, 4, 15, 15, 6) /* HID registers */ @@ -478,13 +479,23 @@ #define SYS_IMP_APL_S3_6_C15_C0_5 sys_reg(3, 6, 15, 0, 5) #define SYS_IMP_APL_S3_4_C15_C12_0 sys_reg(3, 4, 15, 12, 0) -/* T8140 CLPC sampler controls/counters. Live EL2 access faults in the SPTM path. */ +// CLPC sampler controls/counters #define SYS_IMP_APL_CLPC_CORE_PERF0 sys_reg(3, 7, 15, 0, 3) #define SYS_IMP_APL_CLPC_CORE_PERF1 sys_reg(3, 7, 15, 2, 3) #define SYS_IMP_APL_CLPC_CORE_PERF2 sys_reg(3, 7, 15, 4, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF3 sys_reg(3, 7, 15, 6, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF4 sys_reg(3, 7, 15, 8, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF5 sys_reg(3, 7, 15, 10, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF6 sys_reg(3, 7, 15, 12, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF7 sys_reg(3, 7, 15, 14, 3) #define SYS_IMP_APL_CLPC_CORE_PERF_ENABLE0 sys_reg(3, 7, 15, 1, 3) #define SYS_IMP_APL_CLPC_CORE_PERF_ENABLE1 sys_reg(3, 7, 15, 3, 3) #define SYS_IMP_APL_CLPC_CORE_PERF_ENABLE2 sys_reg(3, 7, 15, 5, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF_ENABLE3 sys_reg(3, 7, 15, 7, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF_ENABLE4 sys_reg(3, 7, 15, 9, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF_ENABLE5 sys_reg(3, 7, 15, 11, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF_ENABLE6 sys_reg(3, 7, 15, 13, 3) +#define SYS_IMP_APL_CLPC_CORE_PERF_ENABLE7 sys_reg(3, 7, 15, 15, 3) #define SYS_IMP_APL_CLPC_CORE_PERF_CTRL2 sys_reg(3, 7, 15, 5, 0) #define SYS_IMP_APL_CLPC_CORE_ACC0 sys_reg(3, 1, 15, 8, 3) #define SYS_IMP_APL_CLPC_CORE_ACC1 sys_reg(3, 1, 15, 9, 3) @@ -742,6 +753,7 @@ #define SYS_IMP_APL_GXF_ABORT_EL12 sys_reg(3, 6, 15, 15, 3) #define SYS_IMP_APL_GXF_ENTER_EL12 sys_reg(3, 6, 15, 15, 2) +#define SYS_IMP_APL_GXF_CONFIG_EL12 sys_reg(3, 6, 15, 15, 1) #define SYS_IMP_APL_SPRR_PERM_EL0 sys_reg(3, 6, 15, 1, 5) #define SYS_IMP_APL_SPRR_PERM_EL1 sys_reg(3, 6, 15, 1, 6) @@ -757,16 +769,17 @@ #define SYS_IMP_APL_ELR_GL1 sys_reg(3, 6, 15, 10, 6) #define SYS_IMP_APL_FAR_GL1 sys_reg(3, 6, 15, 10, 7) +#define SYS_IMP_APL_AFSR1_GL12 sys_reg(3, 6, 15, 0, 3) #define SYS_IMP_APL_VBAR_GL12 sys_reg(3, 6, 15, 9, 2) #define SYS_IMP_APL_SPSR_GL12 sys_reg(3, 6, 15, 9, 3) #define SYS_IMP_APL_ASPSR_GL12 sys_reg(3, 6, 15, 9, 4) #define SYS_IMP_APL_ESR_GL12 sys_reg(3, 6, 15, 9, 5) #define SYS_IMP_APL_ELR_GL12 sys_reg(3, 6, 15, 9, 6) +#define SYS_IMP_APL_FAR_GL12 sys_reg(3, 6, 15, 9, 7) #define SYS_IMP_APL_SP_GL12 sys_reg(3, 6, 15, 10, 0) #define SYS_IMP_APL_AFSR1_GL1 sys_reg(3, 6, 15, 0, 1) -/* PAuth registers */ #define SYS_IMP_APL_APVMKEYLO_EL2 sys_reg(3, 6, 15, 14, 4) #define SYS_IMP_APL_APVMKEYHI_EL2 sys_reg(3, 6, 15, 14, 5) #define SYS_IMP_APL_APSTS_EL12 sys_reg(3, 6, 15, 14, 7) diff --git a/src/hv.c b/src/hv.c index 09f2fac7..cb855e8f 100644 --- a/src/hv.c +++ b/src/hv.c @@ -5,6 +5,7 @@ #include "cpu_regs.h" #include "display.h" #include "gxf.h" +#include "hv_sptm.h" #include "memory.h" #include "pcie.h" #include "smp.h" @@ -103,6 +104,7 @@ static void hv_config_sme(bool verbose) reg_mask(SYS_CPTR_EL2, CPTR_EL2_SMEN | CPTR_EL2_FPEN | CPTR_EL2_ZEN, CPTR_EL2_SMEN_NONE | CPTR_EL2_FPEN_NONE | CPTR_EL2_ZEN_NONE); + reg_set(SYS_SMCR_EL2, SMCR_EL2_EZT0 | SMCR_EL2_LEN); if (FIELD_GET(ID_AA64MMFR0_FGT, mrs(ID_AA64MMFR0_EL1))) { reg_set(SYS_HFGRTR_EL2, HFGxTR_EL2_nTPIDR2_EL0 | HFGxTR_EL2_nSMPRI_EL1); @@ -138,13 +140,6 @@ void hv_init(void) HCR_AMO | // Trap SError exceptions HCR_VM; // Enable stage 2 translation - /* - * On guarded (M4-class) SoCs the Apple impdef sysregs are GL2-only and fault - * when executed from EL2, so trap the guest's EL1 accesses to us and soft- - * model/shadow them. Do NOT set TIDCP on unlocked SoCs: there the guest - * accesses these registers directly at EL1 and trapping them would regress - * existing (M1-M3) virtualization. - */ if (!cpu_features->apple_sysregs_unlocked) hcr |= HCR_TIDCP; @@ -189,19 +184,23 @@ static void hv_set_gxf_vbar(void) void hv_start(void *entry, u64 regs[4]) { + sptm_boot_prepare_start(&entry, regs, false); + if (boot_cpu_idx == -1) { printf("Boot CPU has not been found, can't start hypervisor\n"); return; } + // Give the guest this core's identity instead of stale firmware state. + msr(VMPIDR_EL2, mrs(MPIDR_EL1)); + memset(hv_should_exit, 0, sizeof(hv_should_exit)); memset(hv_started_cpus, 0, sizeof(hv_started_cpus)); hv_started_cpus[boot_cpu_idx] = true; - // Install the T8140 CPU/ACC/CPM write guard before the guest runs (no-op on - // other SoCs). Must be after the guest's device mappings are established. - hv_t8140_map_accumulators(); + // Install CPU/CPM power-state guards after guest device mappings exist. + hv_map_power_register_guard(); msr(VBAR_EL1, _hv_vectors_start); @@ -275,6 +274,9 @@ void hv_start(void *entry, u64 regs[4]) static void hv_init_secondary(struct hv_secondary_info_t *info) { + // Give the guest this core's identity instead of stale firmware state. + msr(VMPIDR_EL2, mrs(MPIDR_EL1)); + if (cpu_features->apple_sysregs_unlocked) gxf_init(); @@ -342,22 +344,17 @@ static void hv_enter_secondary(void *entry, u64 regs[4]) void hv_start_secondary(int cpu, void *entry, u64 regs[4]) { - printf("HV: Initializing secondary %d\n", cpu); - iodev_console_flush(); + sptm_boot_prepare_start(&entry, regs, true); if (!cpu_features->apple_sysregs_unlocked) hv_capture_guest_el12_state(&hv_secondary_info); + mmu_init_secondary(cpu); - iodev_console_flush(); smp_call4(cpu, hv_init_secondary, (u64)&hv_secondary_info, 0, 0, 0); smp_wait(cpu); - iodev_console_flush(); - printf("HV: Entering guest secondary %d at %p\n", cpu, entry); hv_started_cpus[cpu] = true; __atomic_or_fetch(&hv_cpus_in_guest, BIT(cpu), __ATOMIC_ACQUIRE); - - iodev_console_flush(); smp_call4(cpu, hv_enter_secondary, (u64)entry, (u64)regs, 0, 0); } @@ -481,6 +478,7 @@ void hv_arm_tick(bool secondary) else msr(CNTP_TVAL_EL0, hv_tick_interval); msr(CNTP_CTL_EL0, CNTx_CTL_ENABLE); + hv_rearm_soft_timer(); } void hv_maybe_exit(void) diff --git a/src/hv.h b/src/hv.h index 694ffd51..42fb839e 100644 --- a/src/hv.h +++ b/src/hv.h @@ -49,6 +49,7 @@ typedef enum _hv_entry_type { HV_CPU_SWITCH, HV_VIRTIO, HV_PANIC, + HV_XNU_PANIC, } hv_entry_type; /* VM */ @@ -60,8 +61,8 @@ int hv_map_hw_ro(u64 from, u64 to, u64 size); int hv_map_sw(u64 from, u64 to, u64 size); int hv_map_hook(u64 from, hv_hook_t *hook, u64 size); -/* T8140 CPU/ACC/CPM write guard (no-op on other SoCs); see hv_t8140.c. */ -int hv_t8140_map_accumulators(void); +/* Guard unsafe guest CPU/CPM writes on supported SoCs. */ +int hv_map_power_register_guard(void); bool hv_pt_is_ram(u64 ipa); int hv_pt_set_writable(u64 ipa, bool writable); @@ -84,6 +85,9 @@ void virtio_put_buffer(u64 base, int qu, u32 id, u32 len); /* Exceptions */ void hv_exc_proxy(struct exc_info *ctx, uartproxy_boot_reason_t reason, u32 type, void *extra); +bool hv_handle_pmc_hvc(struct exc_info *ctx, u32 immediate); +u64 *hv_sprr_shadow(u32 index); +bool hv_handle_objc_bp_hvc(struct exc_info *ctx, u32 immediate); void hv_set_time_stealing(bool enabled, bool reset); void hv_apply_time_stealing_offset(void); void hv_add_time(s64 time); @@ -124,6 +128,7 @@ void hv_rendezvous(void); bool hv_switch_cpu(int cpu); void hv_pin_cpu(int cpu); void hv_arm_tick(bool secondary); +void hv_rearm_soft_timer(void); void hv_rearm(void); void hv_maybe_exit(void); void hv_tick(struct exc_info *ctx); diff --git a/src/hv_asm.S b/src/hv_asm.S index 634eb097..747859ca 100644 --- a/src/hv_asm.S +++ b/src/hv_asm.S @@ -166,8 +166,9 @@ hv_enter_guest: str x5, [x6, x7, LSL #3] mrs x5, daif - mov x6, #5 - orr x5, x5, x6 // EL1h + // XNU requires all exceptions, including debug, masked on kernel entry. + mov x6, #0x3c5 // D | A | I | F | EL1h + orr x5, x5, x6 msr spsr_el2, x5 msr elr_el2, x4 diff --git a/src/hv_exc.c b/src/hv_exc.c index 782132c3..02e575ce 100644 --- a/src/hv_exc.c +++ b/src/hv_exc.c @@ -1,10 +1,12 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ #include "hv.h" +#include "hv_sptm.h" #include "assert.h" #include "cpu_regs.h" #include "exception.h" #include "smp.h" +#include "soc.h" #include "string.h" #include "uart.h" #include "uartproxy.h" @@ -44,13 +46,31 @@ struct hv_pcpu_data { u64 amx_ctx_el1; u64 amx_ctl_el1; u64 amx_state_t; + u64 gxf_config_el1; + u64 gxf_entry_el1; + u64 gxf_pabentry_el1; + u64 gxf_config_el12; + u64 gxf_entry_el12; + u64 gxf_pabentry_el12; + u64 gxf_tpidr_gl1; + u64 gxf_afsr1_gl12; + u64 gxf_vbar_gl12; + u64 gxf_spsr_gl12; + u64 gxf_aspsr_gl12; + u64 gxf_esr_gl12; + u64 gxf_elr_gl12; + u64 gxf_far_gl12; + u64 gxf_sp_gl12; } ALIGNED(64); struct hv_pcpu_data pcpu[MAX_CPUS]; static u64 hv_clpc_core_perf_ctrl2[MAX_CPUS]; +static u64 hv_clpc_core_perf_enable[MAX_CPUS][8]; static u64 hv_clpc_mem_stall_cfg[MAX_CPUS][3]; -static s64 hv_clpc_counter_bias[MAX_CPUS][16]; +static s64 hv_clpc_counter_bias[MAX_CPUS][8]; + +#define HV_PMCR0_READ_HVC 0x3c0 /* * SPRR permission-remap table, soft-cached on M4+ SoCs where the real register @@ -60,6 +80,15 @@ static u64 hv_sprr_perm_el0 = 0x2010002030100000; static u64 hv_sprr_perm_el1 = 0x2020a506f020f0e0; static u64 hv_sprr_umprr_el1; +// SPTM guests reach the same shadows through hypercalls: PERM_EL0, PERM_EL1, then UMPRR_EL1. +u64 *hv_sprr_shadow(u32 index) +{ + u64 *const shadows[] = {&hv_sprr_perm_el0, &hv_sprr_perm_el1, &hv_sprr_umprr_el1}; + return shadows[index]; +} + +#define APPLE_CNTV_FREQ 24000000ULL + /* * Track EL2/proxy time hidden from the guest. The architectural counter and * Apple CNTVCT alias run in different clock domains, so each needs its own @@ -68,20 +97,8 @@ static u64 hv_sprr_umprr_el1; static u64 stolen_time = 0; static u64 stolen_time_kernel_cntv = 0; -/* - * M4-class SoCs schedule based off an Apple impdef timer that is locked in raw - * boot mode. We soft-model it: keep the deadline in a per-CPU shadow, read - * "now" from a counter (KERNEL_CNTVCTSS == CNTVCT_ALIAS, readable and - * already passed through), and deliver the timer as a virtual FIQ from - * hv_update_fiq() once the soft deadline elapses. - */ static inline u64 hv_kernel_cntv_now(void) { - /* - * CNTVOFF_EL2 only offsets the architectural virtual counter. XNU's Apple - * kernel deadline timer reads CNTVCT_ALIAS instead, so track stolen time for - * that timer separately in CNTVCT_ALIAS units and subtract it here. - */ return mrs(SYS_IMP_APL_CNTVCT_ALIAS_EL0) - stolen_time_kernel_cntv; } @@ -90,6 +107,33 @@ static inline s64 hv_kernel_cntv_delta(void) return (s64)(PERCPU(kernel_cntv_cval) - hv_kernel_cntv_now()); } +void hv_rearm_soft_timer(void) +{ + if (cpu_features->apple_sysregs_unlocked || !PERCPU(kernel_cntv_valid)) + return; + + u64 ctl = PERCPU(kernel_cntv_ctl) & (CNTx_CTL_ENABLE | CNTx_CTL_IMASK); + if (ctl != CNTx_CTL_ENABLE) + return; + + s64 apple_delta = hv_kernel_cntv_delta(); + if (apple_delta <= 0) + return; + + u64 frequency = mrs(CNTFRQ_EL0); + u64 arch_delta = (u64)(((__uint128_t)(u64)apple_delta * frequency + + APPLE_CNTV_FREQ - 1) / + APPLE_CNTV_FREQ); + u64 deadline = mrs(CNTPCT_EL0) + arch_delta; + u64 host_ctl = mrs(CNTP_CTL_EL0); + + if (!(host_ctl & CNTx_CTL_ENABLE) || + (s64)(deadline - mrs(CNTP_CVAL_EL0)) < 0) { + msr(CNTP_CVAL_EL0, deadline); + msr(CNTP_CTL_EL0, CNTx_CTL_ENABLE); + } +} + static void hv_kernel_cntv_init(void) { if (PERCPU(kernel_cntv_valid)) @@ -101,6 +145,7 @@ static void hv_kernel_cntv_init(void) mrs(SYS_IMP_APL_KERNEL_CNTV_CTL_EL0) & (CNTx_CTL_ENABLE | CNTx_CTL_IMASK); PERCPU(kernel_cntv_cval) = hv_kernel_cntv_now() + (s64)(s32)tval; PERCPU(kernel_cntv_valid) = true; + hv_rearm_soft_timer(); } static u64 hv_kernel_cntv_read_ctl(void) @@ -123,12 +168,14 @@ static void hv_kernel_cntv_write_ctl(u64 val) { hv_kernel_cntv_init(); PERCPU(kernel_cntv_ctl) = val & (CNTx_CTL_ENABLE | CNTx_CTL_IMASK); + hv_rearm_soft_timer(); } static void hv_kernel_cntv_write_tval(u64 val) { hv_kernel_cntv_init(); PERCPU(kernel_cntv_cval) = hv_kernel_cntv_now() + (s64)(s32)(u32)val; + hv_rearm_soft_timer(); } static bool hv_kernel_cntv_pending(void) @@ -141,7 +188,7 @@ static bool hv_kernel_cntv_pending(void) (CNTx_CTL_ISTATUS | CNTx_CTL_ENABLE); } -#define KERNEL_CNTKCTL_CNTHCTL_MASK \ +#define KERNEL_CNTKCTL_MASK \ (CNTHCTL_EL0PCTEN | CNTHCTL_EL0VCTEN | CNTHCTL_EVNTI | CNTHCTL_EVNTDIR | CNTHCTL_EVNTEN) static bool hv_handle_kernel_cntkctl(struct exc_info *ctx, u64 rt, bool is_read) @@ -155,18 +202,58 @@ static bool hv_handle_kernel_cntkctl(struct exc_info *ctx, u64 rt, bool is_read) u64 val = rt < 31 ? ctx->regs[rt] : 0; PERCPU(kernel_cntkctl_el1) = val; - /* - * XNU programs WFE timeout events and EL0 timebase access through this - * Apple alias. On guarded SoCs, direct EL2 writes to the Apple register can - * fault, so mirror only the architectural CNTHCTL_EL2 bits that correspond - * to the guest-visible Apple control state. - */ - msr(SYS_CNTHCTL_EL2, (mrs(SYS_CNTHCTL_EL2) & ~KERNEL_CNTKCTL_CNTHCTL_MASK) | - (val & KERNEL_CNTKCTL_CNTHCTL_MASK)); + // Rescale the event bit from XNU's 24 MHz counter to CNTFRQ_EL0. + u64 cntkctl = val & KERNEL_CNTKCTL_MASK; + if (val & CNTHCTL_EVNTEN) { + u32 apple_index = FIELD_GET(CNTHCTL_EVNTI, val); + u64 apple_period = 1ULL << (apple_index + 1); + u64 arch_period = + (apple_period * mrs(CNTFRQ_EL0) + APPLE_CNTV_FREQ / 2) / APPLE_CNTV_FREQ; + u32 arch_index = 0; + u64 candidate_period = 2; + + while (arch_index < 15 && candidate_period < arch_period) { + candidate_period <<= 1; + arch_index++; + } + if (arch_index && candidate_period >= arch_period && + arch_period - (candidate_period >> 1) < candidate_period - arch_period) + arch_index--; + + cntkctl &= ~CNTHCTL_EVNTI; + cntkctl |= FIELD_PREP(CNTHCTL_EVNTI, arch_index); + } + + msr(SYS_CNTKCTL_EL12, (mrs(SYS_CNTKCTL_EL12) & ~KERNEL_CNTKCTL_MASK) | cntkctl); sysop("isb"); return true; } +static bool hv_handle_pmcr0(struct exc_info *ctx, u32 rt, bool is_read) +{ + u64 value = rt < 31 ? ctx->regs[rt] : 0; + + if (is_read) { + value = (mrs(SYS_IMP_APL_PMCR0) & ~PMCR0_IMODE_MASK) | + PERCPU(pmc_irq_mode); + if (PERCPU(pmc_pending)) + value |= PMCR0_IACT; + if (rt < 31) + ctx->regs[rt] = value; + } else { + PERCPU(pmc_pending) = !!(value & PMCR0_IACT); + PERCPU(pmc_irq_mode) = value & PMCR0_IMODE_MASK; + msr(SYS_IMP_APL_PMCR0, value); + } + + return true; +} + +bool hv_handle_pmc_hvc(struct exc_info *ctx, u32 immediate) +{ + return hv_handle_pmcr0(ctx, immediate & 0x1f, immediate >= HV_PMCR0_READ_HVC); +} + void hv_exit_guest(void) __attribute__((noreturn)); static u64 exc_entry_time; @@ -289,18 +376,24 @@ void hv_set_time_stealing(bool enabled, bool reset) void hv_apply_time_stealing_offset(void) { msr(CNTVOFF_EL2, stolen_time); + if (!cpu_features->apple_sysregs_unlocked) + hv_rearm_soft_timer(); } void hv_add_time(s64 time) { u64 ticks = time < 0 ? (u64)-time : (u64)time; - u64 kernel_ticks = (u64)((__uint128_t)ticks * 24000000ULL / mrs(CNTFRQ_EL0)); + u64 kernel_ticks = + (u64)((__uint128_t)ticks * APPLE_CNTV_FREQ / mrs(CNTFRQ_EL0)); stolen_time -= (u64)time; if (time < 0) stolen_time_kernel_cntv += kernel_ticks; else stolen_time_kernel_cntv -= kernel_ticks; + + if (!cpu_features->apple_sysregs_unlocked) + hv_rearm_soft_timer(); } static void hv_update_fiq(void) @@ -332,6 +425,7 @@ static void hv_update_fiq(void) } else if (!(hcr & HCR_VF) && fiq_pending) { hv_write_hcr(hcr | HCR_VF); } + } #define SYSREG_MAP(sr, to) \ @@ -372,9 +466,6 @@ static bool hv_handle_clpc_counter(struct exc_info *ctx, u64 rt, bool is_read, u * is unsafe in the current T8140/SPTM context, so provide coherent * monotonic guest-visible counters. */ - if (counter >= 16) - return false; - u64 cpu = mrs(TPIDR_EL2); u64 base = mrs(CNTPCT_EL0) + (counter * 0x1000000ULL); @@ -397,9 +488,6 @@ static bool hv_handle_clpc_mem_stall_config(struct exc_info *ctx, u64 rt, bool i * read/modify/write. Keep readback coherent, but do not touch live EL2 * hardware registers. */ - if (regno >= 3) - return false; - u64 cpu = mrs(TPIDR_EL2); u64 *cache = &hv_clpc_mem_stall_cfg[cpu][regno]; @@ -431,6 +519,144 @@ static bool hv_handle_clpc_core_perf_ctrl2(struct exc_info *ctx, u64 rt, bool is return true; } +static bool hv_handle_clpc_core_perf_enable(struct exc_info *ctx, u64 rt, bool is_read, + unsigned int channel) +{ + u64 cpu = mrs(TPIDR_EL2); + u64 *cache = &hv_clpc_core_perf_enable[cpu][channel]; + + if (is_read) { + if (rt < 31) + ctx->regs[rt] = *cache; + } else { + *cache = rt < 31 ? ctx->regs[rt] : 0; + } + + return true; +} + +static bool hv_handle_gxf_shadow(struct exc_info *ctx, u64 reg, u64 rt, bool is_read) +{ + u64 *value; + + switch (reg) { + case SYSREG_ISS(SYS_IMP_APL_GXF_STATUS_EL1): + if (is_read && rt < 31) + ctx->regs[rt] = 0; + return is_read; + case SYSREG_ISS(SYS_IMP_APL_GXF_CONFIG_EL1): + value = &PERCPU(gxf_config_el1); + break; + case SYSREG_ISS(SYS_IMP_APL_GXF_ENTER_EL1): + value = &PERCPU(gxf_entry_el1); + break; + case SYSREG_ISS(SYS_IMP_APL_GXF_ABORT_EL1): + value = &PERCPU(gxf_pabentry_el1); + break; + case SYSREG_ISS(SYS_IMP_APL_GXF_CONFIG_EL12): + value = &PERCPU(gxf_config_el12); + break; + case SYSREG_ISS(SYS_IMP_APL_GXF_ENTER_EL12): + value = &PERCPU(gxf_entry_el12); + break; + case SYSREG_ISS(SYS_IMP_APL_GXF_ABORT_EL12): + value = &PERCPU(gxf_pabentry_el12); + break; + case SYSREG_ISS(SYS_IMP_APL_TPIDR_GL1): + value = &PERCPU(gxf_tpidr_gl1); + break; + case SYSREG_ISS(SYS_IMP_APL_AFSR1_GL1): + case SYSREG_ISS(SYS_IMP_APL_AFSR1_GL12): + value = &PERCPU(gxf_afsr1_gl12); + break; + case SYSREG_ISS(SYS_IMP_APL_VBAR_GL1): + case SYSREG_ISS(SYS_IMP_APL_VBAR_GL12): + value = &PERCPU(gxf_vbar_gl12); + break; + case SYSREG_ISS(SYS_IMP_APL_SPSR_GL1): + case SYSREG_ISS(SYS_IMP_APL_SPSR_GL12): + value = &PERCPU(gxf_spsr_gl12); + break; + case SYSREG_ISS(SYS_IMP_APL_ASPSR_GL1): + case SYSREG_ISS(SYS_IMP_APL_ASPSR_GL12): + value = &PERCPU(gxf_aspsr_gl12); + break; + case SYSREG_ISS(SYS_IMP_APL_ESR_GL1): + case SYSREG_ISS(SYS_IMP_APL_ESR_GL12): + value = &PERCPU(gxf_esr_gl12); + break; + case SYSREG_ISS(SYS_IMP_APL_ELR_GL1): + case SYSREG_ISS(SYS_IMP_APL_ELR_GL12): + value = &PERCPU(gxf_elr_gl12); + break; + case SYSREG_ISS(SYS_IMP_APL_FAR_GL1): + case SYSREG_ISS(SYS_IMP_APL_FAR_GL12): + value = &PERCPU(gxf_far_gl12); + break; + case SYSREG_ISS(SYS_IMP_APL_SP_GL12): + value = &PERCPU(gxf_sp_gl12); + break; + default: + return false; + } + + if (is_read) { + if (rt < 31) + ctx->regs[rt] = *value; + } else { + *value = rt < 31 ? ctx->regs[rt] : 0; + } + return true; +} + +bool hv_handle_clpc_hvc(struct exc_info *ctx, u32 immediate) +{ + u32 operation = (immediate - 0x400) >> 5; + u32 rt = immediate & 0x1f; + bool is_read = operation & 1; + u32 reg = operation >> 1; + + if (reg < 16) { + if (reg & 1) + return hv_handle_clpc_core_perf_enable(ctx, rt, is_read, reg >> 1); + return hv_handle_clpc_counter(ctx, rt, is_read, reg >> 1); + } + + switch (reg) { + case 16: + return hv_handle_clpc_core_perf_ctrl2(ctx, rt, is_read); + case 17: + case 18: + case 19: + case 20: + case 21: + return hv_handle_clpc_counter(ctx, rt, is_read, reg - 17); + case 22: + case 23: + case 24: + return hv_handle_clpc_mem_stall_config(ctx, rt, is_read, reg - 22); + default: + __builtin_unreachable(); + } +} + +bool hv_handle_objc_bp_hvc(struct exc_info *ctx, u32 immediate) +{ + u32 operation = (immediate - 0xa40) >> 5; + u32 rt = immediate & 0x1f; + bool is_read = operation & 1; + u64 *value = operation < 2 ? &PERCPU(impdef_c15_c0_4) : &PERCPU(impdef_c15_c0_5); + + if (is_read) { + if (rt < 31) + ctx->regs[rt] = *value; + } else { + *value = rt < 31 ? ctx->regs[rt] : 0; + } + + return true; +} + static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) { u64 reg = iss & (ESR_ISS_MSR_OP0 | ESR_ISS_MSR_OP2 | ESR_ISS_MSR_OP1 | ESR_ISS_MSR_CRn | @@ -439,12 +665,22 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) bool is_read = iss & ESR_ISS_MSR_DIR; u64 *regs = ctx->regs; - regs[31] = 0; + if (!cpu_features->apple_sysregs_unlocked && + hv_handle_gxf_shadow(ctx, reg, rt, is_read)) + return true; + switch (reg) { SYSREG_PASS(SYS_IMP_APL_CORE_NRG_ACC_DAT); SYSREG_PASS(SYS_IMP_APL_CORE_SRM_NRG_ACC_DAT); + case SYSREG_ISS(SYS_IMP_APL_S3_1_C15_C13_3): + // PPM polls this register frequently. Match Python's hardware + // pass-through locally instead of stopping all CPUs for each read. + if (!is_read) + return false; + regs[rt] = mrs(SYS_IMP_APL_S3_1_C15_C13_3); + return true; /* Architectural timer, for ECV */ SYSREG_MAP(SYS_CNTV_CTL_EL0, SYS_CNTV_CTL_EL02) SYSREG_MAP(SYS_CNTV_CVAL_EL0, SYS_CNTV_CVAL_EL02) @@ -452,8 +688,7 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) SYSREG_MAP(SYS_CNTP_CTL_EL0, SYS_CNTP_CTL_EL02) SYSREG_MAP(SYS_CNTP_CVAL_EL0, SYS_CNTP_CVAL_EL02) SYSREG_MAP(SYS_CNTP_TVAL_EL0, SYS_CNTP_TVAL_EL02) - /* Apple kernel deadline timer that we shadow soft-modeled, CTL/TVAL - * are write-locked at EL2; the counters read through. */ + /* Software-model the write-locked Apple deadline timer. */ case SYSREG_ISS(SYS_IMP_APL_KERNEL_CNTV_CTL_EL0): if (is_read) regs[rt] = hv_kernel_cntv_read_ctl(); @@ -537,6 +772,16 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) return hv_handle_clpc_counter(ctx, rt, is_read, 1); case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF2): return hv_handle_clpc_counter(ctx, rt, is_read, 2); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF3): + return hv_handle_clpc_counter(ctx, rt, is_read, 3); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF4): + return hv_handle_clpc_counter(ctx, rt, is_read, 4); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF5): + return hv_handle_clpc_counter(ctx, rt, is_read, 5); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF6): + return hv_handle_clpc_counter(ctx, rt, is_read, 6); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF7): + return hv_handle_clpc_counter(ctx, rt, is_read, 7); case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_ACC0): return hv_handle_clpc_counter(ctx, rt, is_read, 0); case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_ACC1): @@ -548,11 +793,21 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_ACC4): return hv_handle_clpc_counter(ctx, rt, is_read, 4); case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF_ENABLE0): + return hv_handle_clpc_core_perf_enable(ctx, rt, is_read, 0); case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF_ENABLE1): + return hv_handle_clpc_core_perf_enable(ctx, rt, is_read, 1); case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF_ENABLE2): - if (is_read) - regs[rt] = 0; - return true; + return hv_handle_clpc_core_perf_enable(ctx, rt, is_read, 2); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF_ENABLE3): + return hv_handle_clpc_core_perf_enable(ctx, rt, is_read, 3); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF_ENABLE4): + return hv_handle_clpc_core_perf_enable(ctx, rt, is_read, 4); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF_ENABLE5): + return hv_handle_clpc_core_perf_enable(ctx, rt, is_read, 5); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF_ENABLE6): + return hv_handle_clpc_core_perf_enable(ctx, rt, is_read, 6); + case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF_ENABLE7): + return hv_handle_clpc_core_perf_enable(ctx, rt, is_read, 7); case SYSREG_ISS(SYS_IMP_APL_CLPC_CORE_PERF_CTRL2): return hv_handle_clpc_core_perf_ctrl2(ctx, rt, is_read); case SYSREG_ISS(SYS_IMP_APL_CLPC_MEM_STALL_CFG0): @@ -562,14 +817,8 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) case SYSREG_ISS(SYS_IMP_APL_CLPC_MEM_STALL_CFG2): return hv_handle_clpc_mem_stall_config(ctx, rt, is_read, 2); SYSREG_MAP(SYS_IMP_APL_APCTL_EL1, SYS_IMP_APL_APCTL_EL12) - /* - * AMX. On M4 userspace uses SME, so AMX is inert, but XNU's per-CPU - * init and context switch still touch these regs, which fault at EL2 on - * guarded SoCs. Advertise a valid version (AMXIDR/AIDR2, else boot - * panics) and soft-cache the control/state on guarded SoCs so XNU's - * read-after-write is coherent; the real AMX unit is never used. Unlocked - * SoCs keep the original map/pass behavior. - */ + // Raw boot locks us out of AMX, fake it enough to boot XNU + // We'll tell userspace we don't have it once booted case SYSREG_ISS(SYS_IMP_APL_AMXIDR_EL1): if (!is_read) return false; @@ -653,15 +902,7 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss) /* shadow the interrupt mode and state flag */ case SYSREG_ISS(SYS_IMP_APL_PMCR0): - if (is_read) { - u64 val = (mrs(SYS_IMP_APL_PMCR0) & ~PMCR0_IMODE_MASK) | PERCPU(pmc_irq_mode); - regs[rt] = val | (PERCPU(pmc_pending) ? PMCR0_IACT : 0); - } else { - PERCPU(pmc_pending) = !!(regs[rt] & PMCR0_IACT); - PERCPU(pmc_irq_mode) = regs[rt] & PMCR0_IMODE_MASK; - msr(SYS_IMP_APL_PMCR0, regs[rt]); - } - return true; + return hv_handle_pmcr0(ctx, rt, is_read); /* * Handle this one here because m1n1/Linux (will) use it for explicit cpuidle. @@ -797,6 +1038,8 @@ void hv_exc_sync(struct exc_info *ctx) bool handled = false; u32 ec = FIELD_GET(ESR_EC, ctx->esr); + bool advance = true; + switch (ec) { case ESR_EC_MSR: hv_wdt_breadcrumb('m'); @@ -810,11 +1053,17 @@ void hv_exc_sync(struct exc_info *ctx) break; } break; + case ESR_EC_HVC: + handled = hv_sptm_handle_hvc_unlocked(ctx, FIELD_GET(ESR_ISS, ctx->esr) & 0xffff); + // HVC saves the address after the call, unlike a trapped MSR/MRS. + advance = false; + break; } if (handled) { hv_wdt_breadcrumb('#'); - ctx->elr += 4; + if (advance) + ctx->elr += 4; hv_set_elr(ctx->elr); hv_update_fiq(); hv_wdt_breadcrumb('s'); @@ -840,11 +1089,17 @@ void hv_exc_sync(struct exc_info *ctx) break; } break; + case ESR_EC_HVC: + hv_wdt_breadcrumb('H'); + handled = hv_sptm_handle_hvc(ctx, FIELD_GET(ESR_ISS, ctx->esr) & 0xffff); + advance = false; + break; } if (handled) { hv_wdt_breadcrumb('+'); - ctx->elr += 4; + if (advance) + ctx->elr += 4; } else { hv_wdt_breadcrumb('-'); // VM code can forward a nested SError exception here diff --git a/src/hv_power_register_guard.c b/src/hv_power_register_guard.c new file mode 100644 index 00000000..db3e266b --- /dev/null +++ b/src/hv_power_register_guard.c @@ -0,0 +1,159 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include "hv.h" +#include "adt.h" +#include "smp.h" +#include "soc.h" +#include "string.h" +#include "utils.h" + +#define POWER_GUARD_PAGE_SIZE 0x4000UL +#define POWER_GUARD_CPU_CONTROL_OFFSET 0x8000UL +#define POWER_GUARD_CPM_CONTROL0_OFFSET 0x4000UL +#define POWER_GUARD_CPM_CONTROL1_OFFSET 0x8000UL +#define POWER_GUARD_MAX_PAGES (MAX_CPUS * 3) +#define POWER_GUARD_SHADOW_ENTRIES 256 + +struct power_guard_shadow { + u64 ipa; + u64 value; + u8 width; + bool valid; +}; + +static u64 power_guard_pages[POWER_GUARD_MAX_PAGES]; +static u32 power_guard_page_count; +static struct power_guard_shadow power_guard_shadow[POWER_GUARD_SHADOW_ENTRIES]; + +static int power_guard_page_index(u64 ipa) +{ + u64 page = ipa & ~(POWER_GUARD_PAGE_SIZE - 1); + + for (u32 i = 0; i < power_guard_page_count; i++) + if (page == power_guard_pages[i]) + return i; + + return -1; +} + +static int power_guard_add_page(u64 addr) +{ + addr &= ~(POWER_GUARD_PAGE_SIZE - 1); + if (power_guard_page_index(addr) >= 0) + return 0; + if (power_guard_page_count >= ARRAY_SIZE(power_guard_pages)) + return -1; + + power_guard_pages[power_guard_page_count++] = addr; + return 0; +} + +static bool power_guard_shadow_read(u64 ipa, u64 *value, int width) +{ + for (u32 i = 0; i < ARRAY_SIZE(power_guard_shadow); i++) { + struct power_guard_shadow *entry = &power_guard_shadow[i]; + if (entry->valid && entry->ipa == ipa && entry->width == width) { + *value = entry->value; + return true; + } + } + return false; +} + +static void power_guard_shadow_write(u64 ipa, u64 value, int width) +{ + int free = -1; + + for (u32 i = 0; i < ARRAY_SIZE(power_guard_shadow); i++) { + struct power_guard_shadow *entry = &power_guard_shadow[i]; + if (entry->valid) { + if (entry->ipa == ipa && entry->width == width) { + entry->value = value; + return; + } + } else if (free < 0) { + free = i; + } + } + + if (free >= 0) + power_guard_shadow[free] = (struct power_guard_shadow){ + .ipa = ipa, + .value = value, + .width = width, + .valid = true, + }; +} + +static bool power_guard_hook(struct exc_info *ctx, u64 ipa, u64 *value, bool write, int width) +{ + /* EL2 owns the live CPU/CPM state; XNU sees its writes through a shadow. */ + if (write) { + power_guard_shadow_write(ipa, *value, width); + return true; + } + + if (power_guard_shadow_read(ipa, value, width)) + return true; + return hv_pa_rw(ctx, ipa, value, false, width); +} + +int hv_map_power_register_guard(void) +{ + if (chip_id != T8132 && chip_id != T8140 && chip_id != T6040 && chip_id != T6041) + return 0; + + int cpus = adt_path_offset(adt, "/cpus"); + if (cpus < 0) { + printf("HV: CPU power register guard: /cpus is missing\n"); + return -1; + } + + power_guard_page_count = 0; + memset(power_guard_shadow, 0, sizeof(power_guard_shadow)); + int failures = 0; + + int node = cpus; + ADT_FOREACH_CHILD(adt, node) + { + u64 cpu_reg[2]; + u64 cpm_reg[2]; + if (ADT_GETPROP_ARRAY(adt, node, "cpu-impl-reg", cpu_reg) < 0 || + ADT_GETPROP_ARRAY(adt, node, "cpm-impl-reg", cpm_reg) < 0) { + printf("HV: CPU power register guard: incomplete CPU node %s\n", + adt_get_name(adt, node)); + failures++; + continue; + } + + if (cpu_reg[1] <= POWER_GUARD_CPU_CONTROL_OFFSET || + power_guard_add_page(cpu_reg[0] + POWER_GUARD_CPU_CONTROL_OFFSET) < 0) { + printf("HV: CPU power register guard: invalid CPU range for %s\n", + adt_get_name(adt, node)); + failures++; + } + + const u64 cpm_offsets[] = { + POWER_GUARD_CPM_CONTROL0_OFFSET, + POWER_GUARD_CPM_CONTROL1_OFFSET, + }; + for (u32 i = 0; i < ARRAY_SIZE(cpm_offsets); i++) { + if (cpm_reg[1] <= cpm_offsets[i]) + continue; + if (power_guard_add_page(cpm_reg[0] + cpm_offsets[i]) < 0) { + printf("HV: CPU power register guard: invalid CPM range for %s\n", + adt_get_name(adt, node)); + failures++; + } + } + } + + for (u32 i = 0; i < power_guard_page_count; i++) + failures += hv_map_hook(power_guard_pages[i], power_guard_hook, POWER_GUARD_PAGE_SIZE) != 0; + + sysop("dsb ishst"); + sysop("tlbi vmalls12e1is"); + sysop("dsb ish"); + sysop("isb"); + return failures ? -failures : 0; +} diff --git a/src/hv_sptm.c b/src/hv_sptm.c new file mode 100644 index 00000000..f7950b33 --- /dev/null +++ b/src/hv_sptm.c @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include "hv.h" + +#include "hv_sptm_internal.h" + +#define SPTM_GENTER_XNU_PANIC_BEGIN 4 +#define SPTM_PANIC_CPU_NONE 0xffff +#define SPTM_PANIC_DOMAIN_XNU 1 +#define SPTM_PANIC_FLAG_OFFSET 0 +#define SPTM_PANIC_CPU_OFFSET 8 +#define SPTM_PANIC_DOMAIN_OFFSET 12 + +static bool sptm_handle_xnu_panic_begin(struct exc_info *ctx) +{ + if (ctx->cpu_id >= sptm.max_cpus) + return false; + + struct sptm_cpu *cpu = sptm_find_cpu(sptm.hv_phys_ids[ctx->cpu_id]); + if (!cpu) + return false; + + u16 *owner = (u16 *)(sptm.panic_state_pa + SPTM_PANIC_CPU_OFFSET); + u16 expected = SPTM_PANIC_CPU_NONE; + bool first = __atomic_compare_exchange_n(owner, &expected, cpu->logical_id, false, + __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE); + if (first) { + __atomic_store_n((u8 *)(sptm.panic_state_pa + SPTM_PANIC_DOMAIN_OFFSET), + SPTM_PANIC_DOMAIN_XNU, __ATOMIC_RELEASE); + printf("HV: SPTM XNU panic owner CPU %u (HV CPU %lu)\n", cpu->logical_id, ctx->cpu_id); + } + + /* Publish the irreversible panic latch after its owner metadata. */ + __atomic_store_n((u8 *)(sptm.panic_state_pa + SPTM_PANIC_FLAG_OFFSET), true, __ATOMIC_RELEASE); + + if (first) + hv_exc_proxy(ctx, START_HV, HV_XNU_PANIC, NULL); + + return true; +} + +static bool sptm_handle_dispatch(struct exc_info *ctx) +{ + u64 dispatch = ctx->regs[16]; + u32 domain = (dispatch >> 48) & 0xff; + u32 table = (dispatch >> 32) & 0xff; + u32 endpoint = dispatch; + bool handled = false; + + if (!(dispatch & 0xff00ff0000000000ULL)) { + if (domain == 2 && table == 0) { + handled = sptm_handle_txm(ctx, endpoint); + } else if (domain == 0 && table == 0) { + spin_lock(&sptm.service_lock); + handled = sptm_handle_xnu_bootstrap(ctx, endpoint); + spin_unlock(&sptm.service_lock); + } else if (domain == 0 && table == 3 && endpoint <= 16) { + handled = sptm_handle_dart(ctx, endpoint); + } else if (domain == 0 && table == 4 && endpoint <= 3) { + handled = sptm_handle_dart(ctx, endpoint); + } else if (domain == 0 && table == 5 && endpoint <= 2) { + handled = sptm_handle_sart(ctx, endpoint); + } else if (domain == 0 && table == 6 && endpoint <= 8) { + handled = sptm_handle_nvme(ctx, endpoint); + } else if (domain == 0 && table == 7 && endpoint <= 12) { + handled = sptm_handle_uat(ctx, endpoint); + } else if (domain == 0 && table == 9 && endpoint <= 12) { + ctx->regs[0] = SPTM_STATUS_SUCCESS; + handled = true; + } + } + + return handled; +} + +static bool sptm_handle_shadow_hvc(struct exc_info *ctx, u32 immediate) +{ + u32 rt = immediate & 0x1f; + u64 value = rt == 31 ? 0 : ctx->regs[rt]; + + if (immediate >= 0x1c0 && immediate < 0x280) { + u32 operation = (immediate - 0x1c0) >> 5; + u64 *shadow = hv_sprr_shadow(operation >> 1); + if (operation & 1) + *shadow = value; + else if (rt != 31) + ctx->regs[rt] = *shadow; + return true; + } + + if ((immediate >= 0x380 && immediate < 0x3a0) || (immediate >= 0x3c0 && immediate < 0x3e0)) + return hv_handle_pmc_hvc(ctx, immediate); + + if (immediate >= 0x400 && immediate < 0xa40) { + return hv_handle_clpc_hvc(ctx, immediate); + } + + if (immediate >= 0xa40 && immediate < 0xac0) + return hv_handle_objc_bp_hvc(ctx, immediate); + + return false; +} + +bool hv_sptm_handle_hvc_unlocked(struct exc_info *ctx, u32 immediate) +{ + if (!sptm.enabled) + return false; + + // CLPC shadows and counter biases belong to the current CPU. They do not + // need the global HV lock used by page-table and device services. + if (immediate >= 0x400 && immediate < 0xa40) + return hv_handle_clpc_hvc(ctx, immediate); + + return false; +} + +bool hv_sptm_handle_hvc(struct exc_info *ctx, u32 immediate) +{ + if (!sptm.enabled) + return false; + + if (immediate == 0) + return sptm_handle_dispatch(ctx); + + if (immediate == SPTM_GENTER_XNU_PANIC_BEGIN) + return sptm_handle_xnu_panic_begin(ctx); + + return sptm_handle_shadow_hvc(ctx, immediate); +} diff --git a/src/hv_sptm.h b/src/hv_sptm.h new file mode 100644 index 00000000..8425ede7 --- /dev/null +++ b/src/hv_sptm.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef HV_SPTM_H +#define HV_SPTM_H + +#include "exception.h" +#include "types.h" + +u64 hv_sptm_init(u64 guest_adt, u64 cons_ops, u64 page_shift_const, u64 xnu_text); +bool hv_sptm_handle_hvc(struct exc_info *ctx, u32 immediate); +bool hv_sptm_handle_hvc_unlocked(struct exc_info *ctx, u32 immediate); +void sptm_boot_prepare_start(void **entry, u64 regs[4], bool secondary); +bool hv_handle_clpc_hvc(struct exc_info *ctx, u32 immediate); + +#endif diff --git a/src/hv_sptm_boot.c b/src/hv_sptm_boot.c new file mode 100644 index 00000000..acac3cce --- /dev/null +++ b/src/hv_sptm_boot.c @@ -0,0 +1,710 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include "hv_sptm_internal.h" + +#include "adt.h" +#include "xnuboot.h" + +// Headroom for fixed boot objects and ADT-derived platform configuration. +#define SPTM_AUX_FIXED_SIZE (64 * SZ_1M) +#define SPTM_CPU_STACK_WINDOW_SIZE SZ_1M +// TXM indexes a compile-time channel array (16 slots on T6041), even when +// fewer CPU IDs are present in the ADT. Back every supported slot. +#define SPTM_TXM_BOOT_CHANNELS MAX_CPUS +#define SPTM_BOOTSTRAP_SIZE 0x340 +#define SPTM_FRAME_ENTRY_SIZE 16 +#define SPTM_FRAME_PARAM_SIZE 0x90 +#define SPTM_VM_MAX_KERNEL_ADDRESS 0xfffffecfffffffffULL + +#define SPTM_TCR_BOOT 0x10800336511a511ULL +#define SPTM_MAIR_BOOT 0x0c0d00ff01a040ffULL +#define SPTM_SCTLR_BOOT 0x100010103414593dULL + +#define SPTM_LEAF_CODE 0x603 +#define SPTM_LEAF_DATA 0x60000000000703ULL +#define SPTM_LEAF_DATA_OUTER 0x60000000000603ULL +#define SPTM_LEAF_DATA_NC 0x60000000000407ULL +#define SPTM_TABLE_DESC 3 + +#define MACHO_MAGIC_64 0xfeedfacf +#define MACHO_LC_SEGMENT_64 0x19 +#define MACHO_LC_UNIXTHREAD 0x05 +#define MACHO_VM_PROT_WRITE 2 +#define MACHO_VM_PROT_EXEC 4 + +struct macho_header_64 { + u32 magic; + u32 cpu_type; + u32 cpu_subtype; + u32 file_type; + u32 command_count; + u32 command_size; + u32 flags; + u32 reserved; +} PACKED; + +struct macho_command { + u32 type; + u32 size; +} PACKED; + +struct macho_segment_64 { + struct macho_command command; + char name[16]; + u64 vm_address; + u64 vm_size; + u64 file_offset; + u64 file_size; + u32 maximum_protection; + u32 initial_protection; + u32 section_count; + u32 flags; +} PACKED; + +struct macho_thread_64 { + struct macho_command command; + u32 flavor; + u32 count; + u64 x[29]; + u64 fp; + u64 lr; + u64 sp; + u64 pc; + u32 cpsr; + u32 flags; +} PACKED; + +struct sptm_boot_pt_page { + u64 pa; + u8 level; +}; + +struct sptm_boot_io_range { + u64 address; + u64 size; + u32 flags; + u32 signature; +} PACKED; + +struct sptm_boot_io_filter { + u32 signature; + u16 offset; + u16 length; +} PACKED; + +struct sptm_boot_papt { + u64 pa; + u64 va; + u32 count; + u32 reserved; +} PACKED; + +struct sptm_boot_context { + struct sptm_allocator allocator; + u64 managed_start; + u64 managed_end; + u64 physmap_base; + u64 physmap_end; + u64 kernel_pa; + u64 kernel_vmin; + u64 kernel_vmax; + u64 kernel_entry; + u64 ttbr0_pa; + u64 ttbr1_pa; + u64 bootstrap_pa; + u64 panic_state_pa; + u64 scratch_pa; + u64 frame_table_pa; + u64 uat_global_state_pa; + u64 uat_global_root_pa; + u64 uat_l2_pa; + u64 uat_l2_va; + u64 txm_info_va; + u8 *frame_table; + u8 *frame_priority; +}; + +static struct { + bool ready; + u64 entry; + u64 bootargs_va; + u64 bootstrap_va; + struct sptm_boot_pt_page *pages; + size_t page_capacity; + u32 page_count; +} sptm_boot; + +void *sptm_alloc(struct sptm_allocator *allocator, size_t size, size_t alignment) +{ + u64 address = ALIGN_UP(allocator->next, alignment); + assert(address <= allocator->end && size <= allocator->end - address); + allocator->next = address + size; + return (void *)address; +} + +void *sptm_alloc_zero(struct sptm_allocator *allocator, size_t size, size_t alignment) +{ + void *result = sptm_alloc(allocator, size, alignment); + memset(result, 0, size); + return result; +} + +u64 sptm_adt_integer(int node, const char *name, u64 fallback) +{ + u32 size; + const void *value = adt_getprop(adt, node, name, &size); + if (!value) + return fallback; + u64 result = 0; + memcpy(&result, value, min(size, (u32)sizeof(result))); + return result; +} + +static bool sptm_boot_adt_bool(int node, const char *name, bool fallback) +{ + u32 size; + const void *value = adt_getprop(adt, node, name, &size); + if (!value) + return fallback; + if (!size) + return true; + u64 result = 0; + memcpy(&result, value, min(size, (u32)sizeof(result))); + return result != 0; +} + +static void sptm_boot_adt_region(int node, const char *name, u64 *address, u64 *size) +{ + u64 region[2]; + if (ADT_GETPROP_ARRAY(adt, node, name, region) < 0) + panic("SPTM: memory map has no %s region\n", name); + *address = region[0]; + *size = region[1]; +} + +static u64 sptm_boot_property(const int *nodes, size_t count, const char *name, u64 fallback) +{ + for (size_t index = 0; index < count; index++) { + if (adt_get_property(adt, nodes[index], name)) + return sptm_adt_integer(nodes[index], name, fallback); + } + return fallback; +} + +static bool sptm_boot_bool_property(const int *nodes, size_t count, const char *name, bool fallback) +{ + for (size_t index = 0; index < count; index++) { + if (adt_get_property(adt, nodes[index], name)) + return sptm_boot_adt_bool(nodes[index], name, fallback); + } + return fallback; +} + +static u64 sptm_boot_alloc(struct sptm_boot_context *context, size_t size, size_t alignment) +{ + return (u64)sptm_alloc_zero(&context->allocator, size, alignment); +} + +static u64 sptm_boot_alloc_page(struct sptm_boot_context *context) +{ + return sptm_boot_alloc(context, SPTM_PAGE_SIZE, SPTM_PAGE_SIZE); +} + +static u64 sptm_boot_va(const struct sptm_boot_context *context, u64 pa) +{ + return context->physmap_base + pa - context->managed_start; +} + +static u64 sptm_boot_alloc_table(struct sptm_boot_context *context, u8 level) +{ + u64 pa = sptm_boot_alloc_page(context); + assert(sptm_boot.page_count < sptm_boot.page_capacity); + sptm_boot.pages[sptm_boot.page_count++] = (struct sptm_boot_pt_page){pa, level}; + return pa; +} + +static u64 sptm_boot_ensure_l3(struct sptm_boot_context *context, u64 root, u64 va) +{ + u64 table = root; + for (u32 level = 1, shift = 36; level <= 2; level++, shift -= 11) { + u64 *slot = (u64 *)table + ((va >> shift) & 0x7ff); + if (!(*slot & 1)) + *slot = sptm_boot_alloc_table(context, level + 1) | SPTM_TABLE_DESC; + table = *slot & SPTM_DESC_PA_MASK_16K; + } + return table; +} + +static void sptm_boot_map_page(struct sptm_boot_context *context, u64 root, u64 va, u64 pa, + u64 attributes) +{ + u64 table = sptm_boot_ensure_l3(context, root, va); + ((u64 *)table)[(va >> SPTM_PAGE_SHIFT) & 0x7ff] = (pa & SPTM_DESC_PA_MASK_16K) | attributes; +} + +static void sptm_boot_map_range(struct sptm_boot_context *context, u64 root, u64 va, u64 pa, + u64 size, u64 attributes) +{ + for (u64 offset = 0; offset < size; offset += SPTM_PAGE_SIZE) + sptm_boot_map_page(context, root, va + offset, pa + offset, attributes); +} + +static void sptm_boot_preallocate_l3(struct sptm_boot_context *context, u64 root, u64 start, + u64 end) +{ + for (u64 va = ALIGN_DOWN(start, SZ_32M); va < ALIGN_UP(end, SZ_32M); va += SZ_32M) + sptm_boot_ensure_l3(context, root, va); +} + +// Upper bound for the L2 and L3 tables touched by a half-open VA range. +// Count ranges independently: overlapping ranges only make the bound safer. +static size_t sptm_boot_range_tables(u64 start, u64 end) +{ + if (start == end) + return 0; + assert(start < end); + return ((end - 1) >> 25) - (start >> 25) + 1 + + ((end - 1) >> 36) - (start >> 36) + 1; +} + +static u16 sptm_boot_valid_count(u64 table) +{ + u16 count = 0; + for (size_t index = 0; index < SPTM_PAGE_SIZE / sizeof(u64); index++) + count += (((u64 *)table)[index] & 1) != 0; + return count; +} + +static void sptm_boot_set_frame(struct sptm_boot_context *context, u64 pa, u8 type, u8 level, + u16 valid) +{ + u8 *entry = context->frame_table + + ((pa - context->managed_start) >> SPTM_PAGE_SHIFT) * SPTM_FRAME_ENTRY_SIZE; + entry[2] = type; + if (sptm_is_table_type(type)) { + entry[4] = level; + write16((u64)entry + 8, valid); + } +} + +static void sptm_boot_macho_info(struct sptm_boot_context *context) +{ + const struct macho_header_64 *header = (void *)context->kernel_pa; + assert(header->magic == MACHO_MAGIC_64); + const struct macho_command *command = (const void *)(header + 1); + context->kernel_vmin = UINT64_MAX; + for (u32 index = 0; index < header->command_count; index++) { + if (command->type == MACHO_LC_SEGMENT_64) { + const struct macho_segment_64 *segment = (const void *)command; + context->kernel_vmin = min(context->kernel_vmin, segment->vm_address); + context->kernel_vmax = + max(context->kernel_vmax, segment->vm_address + segment->vm_size); + } else if (command->type == MACHO_LC_UNIXTHREAD) { + context->kernel_entry = ((const struct macho_thread_64 *)command)->pc; + } + command = (const void *)command + command->size; + } +} + +static void sptm_boot_type_kernel_frames(struct sptm_boot_context *context) +{ + const struct macho_header_64 *header = (void *)context->kernel_pa; + const struct macho_command *command = (const void *)(header + 1); + for (u32 index = 0; index < header->command_count; index++) { + if (command->type == MACHO_LC_SEGMENT_64) { + const struct macho_segment_64 *segment = (const void *)command; + u8 priority = 1; + u8 type = SPTM_FRAME_RO; + if (segment->initial_protection & MACHO_VM_PROT_EXEC) { + priority = 3; + type = SPTM_FRAME_KERNEL_CODE; + } else if (segment->initial_protection & MACHO_VM_PROT_WRITE) { + priority = 2; + type = SPTM_FRAME_DEFAULT; + } + u64 start = ALIGN_DOWN(segment->vm_address, SPTM_PAGE_SIZE); + u64 end = ALIGN_UP(segment->vm_address + segment->vm_size, SPTM_PAGE_SIZE); + for (u64 va = start; va < end; va += SPTM_PAGE_SIZE) { + u64 pa = context->kernel_pa + va - context->kernel_vmin; + if (pa < context->managed_start || pa >= context->managed_end) + continue; + size_t frame = (pa - context->managed_start) >> SPTM_PAGE_SHIFT; + if (priority > context->frame_priority[frame]) { + context->frame_priority[frame] = priority; + sptm_boot_set_frame(context, pa, type, 0, 0); + } + } + } + command = (const void *)command + command->size; + } +} + +static void sptm_boot_sort_io_ranges(struct sptm_boot_io_range *ranges, u32 count) +{ + for (u32 index = 1; index < count; index++) { + struct sptm_boot_io_range value = ranges[index]; + u32 position = index; + while (position && (ranges[position - 1].address > value.address || + (ranges[position - 1].address == value.address && + ranges[position - 1].size > value.size))) { + ranges[position] = ranges[position - 1]; + position--; + } + ranges[position] = value; + } +} + +static void sptm_boot_sort_io_filters(struct sptm_boot_io_filter *filters, u32 count) +{ + for (u32 index = 1; index < count; index++) { + struct sptm_boot_io_filter value = filters[index]; + u32 position = index; + while (position && (filters[position - 1].signature > value.signature || + (filters[position - 1].signature == value.signature && + filters[position - 1].offset > value.offset))) { + filters[position] = filters[position - 1]; + position--; + } + filters[position] = value; + } +} + +u64 hv_sptm_init(u64 guest_adt, u64 cons_ops, u64 page_shift_const, u64 xnu_text) +{ + memset(&sptm_boot, 0, sizeof(sptm_boot)); + memset(&sptm, 0, sizeof(sptm)); + spin_init(&sptm.service_lock); + void *firmware_adt = adt; + adt = (void *)guest_adt; + + int memory_map = adt_path_offset(adt, "/chosen/memory-map"); + u64 bootargs_pa, bootargs_size, kernel_size; + struct sptm_boot_context context = {0}; + sptm_boot_adt_region(memory_map, "BootArgs", &bootargs_pa, &bootargs_size); + sptm_boot_adt_region(memory_map, "Kernel-mach_header", &context.kernel_pa, &kernel_size); + struct boot_args *bootargs = (void *)bootargs_pa; + + context.managed_start = bootargs->phys_base; + context.managed_end = bootargs->phys_base + bootargs->mem_size; + context.physmap_base = bootargs->virt_base; + context.physmap_end = bootargs->virt_base + bootargs->mem_size; + assert(context.managed_end > context.managed_start); + assert(context.physmap_end > context.physmap_base); + sptm_boot_macho_info(&context); + + u64 trustcache_pa, trustcache_size; + sptm_boot_adt_region(memory_map, "TrustCache", &trustcache_pa, &trustcache_size); + u64 adt_pa = ALIGN_DOWN(guest_adt, SPTM_PAGE_SIZE); + u64 adt_offset = guest_adt - adt_pa; + u64 adt_size = ALIGN_UP(adt_offset + bootargs->devtree_size, SPTM_PAGE_SIZE); + u64 extradata_pa = min(adt_pa, ALIGN_DOWN(trustcache_pa, SPTM_PAGE_SIZE)); + u64 extradata_end = + max(adt_pa + adt_size, ALIGN_UP(trustcache_pa + trustcache_size, SPTM_PAGE_SIZE)); + u64 extradata_size = extradata_end - extradata_pa; + u64 extradata_va = ALIGN_DOWN(context.kernel_vmin - extradata_size, SPTM_PAGE_SIZE); + + // XNU adopts these empty L3s before normal pmap allocation is operational. + // The fixed 64 MiB term stands in for the boot framebuffer size. + u64 memory_segments = ALIGN_UP(bootargs->mem_size, BIT(28)) >> 28; + u64 dynamic_l3_start = ALIGN_UP(context.physmap_end, SZ_32M); + u64 dynamic_budget = (2 + memory_segments * 10) * SZ_1M; + u64 dynamic_l3_end = ALIGN_UP(dynamic_l3_start + dynamic_budget + 64 * SZ_1M, 8 * SZ_1M); + context.uat_l2_va = ALIGN_UP(dynamic_l3_end, SZ_32M); + u64 high_l3_end = SPTM_VM_MAX_KERNEL_ADDRESS + 1; + + // Two roots, then the tables for every mapping/preallocation below. + sptm_boot.page_capacity = 2 + sptm_boot_range_tables(0, SPTM_PAGE_SIZE) + + sptm_boot_range_tables(context.kernel_vmin, ALIGN_UP(context.kernel_vmax, SPTM_PAGE_SIZE)) + + sptm_boot_range_tables(context.physmap_base, context.physmap_end) + + sptm_boot_range_tables(extradata_va, extradata_va + extradata_size) + + sptm_boot_range_tables(dynamic_l3_start, ALIGN_UP(dynamic_l3_end, SZ_32M)) + + sptm_boot_range_tables(context.uat_l2_va, context.uat_l2_va + SPTM_PAGE_SIZE) + + sptm_boot_range_tables(high_l3_end - SZ_32M, high_l3_end); + size_t managed_pages = (context.managed_end - context.managed_start) / SPTM_PAGE_SIZE; + size_t frame_table_size = managed_pages * SPTM_FRAME_ENTRY_SIZE; + size_t external_ref_table_size = managed_pages * sizeof(u64); + size_t table_records_size = sptm_boot.page_capacity * sizeof(*sptm_boot.pages); + u64 aux_size = ALIGN_UP(SPTM_AUX_FIXED_SIZE + frame_table_size + external_ref_table_size + + managed_pages + table_records_size + + sptm_boot.page_capacity * SPTM_PAGE_SIZE, SPTM_PAGE_SIZE); + u64 aux_start = ALIGN_UP(bootargs->top_of_kernel_data, SPTM_PAGE_SIZE); + assert(aux_start >= context.managed_start && aux_start <= context.managed_end); + assert(aux_size <= context.managed_end - aux_start); + u64 aux_end = aux_start + aux_size; + context.allocator = (struct sptm_allocator){aux_start, aux_end}; + // Keep every handoff object below the RAM XNU is allowed to allocate. + bootargs->top_of_kernel_data = aux_end; + sptm_boot.pages = sptm_alloc(&context.allocator, table_records_size, 8); + printf("HV: SPTM boot reservation: 0x%lx bytes, %zu table records\n", + aux_size, sptm_boot.page_capacity); + + int cpus = adt_path_offset(adt, "/cpus"); + u32 cpu_count = 0; + int cpu = cpus; + ADT_FOREACH_CHILD(adt, cpu) + cpu_count = max(cpu_count, (u32)sptm_adt_integer(cpu, "cpu-id", 0) + 1); + assert(cpu_count && cpu_count <= MAX_CPUS); + + int sgx = adt_path_offset(adt, "/arm-io/sgx"); + int defaults = adt_path_offset(adt, "/defaults"); + int chosen = adt_path_offset(adt, "/chosen"); + int uat_nodes[] = {sgx, defaults, chosen}; + u8 uat_mode = + sptm_boot_property(uat_nodes, ARRAY_SIZE(uat_nodes), "agx-address-space-mgmt-mode", 0); + u8 uat_va_width = sptm_boot_property(uat_nodes, ARRAY_SIZE(uat_nodes), "uat-vaddr-size", 40); + u16 uat_segment_limit = + sptm_boot_property(uat_nodes, ARRAY_SIZE(uat_nodes), "uat-segment-limit", 64); + u16 uat_mapping_limit = + sptm_boot_property(uat_nodes, ARRAY_SIZE(uat_nodes), "uat-mapping-limit", 256); + bool uat_tlbi_at_retype = sptm_boot_bool_property(uat_nodes, ARRAY_SIZE(uat_nodes), + "issue-gmmu-tlbis-at-retype", false); + + context.bootstrap_pa = sptm_boot_alloc(&context, SPTM_BOOTSTRAP_SIZE, 16); + u64 debug_header_pa = sptm_boot_alloc(&context, 0x100, 16); + context.panic_state_pa = sptm_boot_alloc_page(&context); + u64 papt_count_pa = sptm_boot_alloc(&context, sizeof(u32), 4); + u64 papt_ranges_pa = sptm_boot_alloc(&context, 3 * sizeof(struct sptm_boot_papt), 8); + u64 trace_buffer_pa = sptm_boot_alloc_page(&context); + u64 dispatch_states_pa = sptm_boot_alloc_page(&context); + u64 saved_states_pa = sptm_boot_alloc_page(&context); + u64 event_counters_pa = sptm_boot_alloc_page(&context); + context.scratch_pa = sptm_boot_alloc(&context, cpu_count * SPTM_PAGE_SIZE, SPTM_PAGE_SIZE); + u64 txm_stack_array_pa = sptm_boot_alloc(&context, SPTM_TXM_BOOT_CHANNELS * sizeof(u64), 8); + u64 txm_stacks_pa = sptm_boot_alloc(&context, SPTM_TXM_BOOT_CHANNELS * SPTM_PAGE_SIZE, SPTM_PAGE_SIZE); + u64 txm_info_pa = sptm_boot_alloc_page(&context); + context.txm_info_va = sptm_boot_va(&context, txm_info_pa); + context.uat_global_state_pa = sptm_boot_alloc_page(&context); + context.uat_global_root_pa = sptm_boot_alloc_page(&context); + u64 kernel_stacks_pa = sptm_boot_alloc(&context, SPTM_CPU_STACK_WINDOW_SIZE, SPTM_PAGE_SIZE); + context.frame_table_pa = sptm_boot_alloc(&context, frame_table_size, SPTM_PAGE_SIZE); + context.frame_table = (void *)context.frame_table_pa; + u64 frame_params_pa = sptm_boot_alloc(&context, 256 * SPTM_FRAME_PARAM_SIZE, SPTM_PAGE_SIZE); + u64 pt_attrs_pa = sptm_boot_alloc(&context, 6 * sizeof(u64), 8); + u64 ttbr0_zero_page_pa = sptm_boot_alloc_page(&context); + + u32 io_ranges_bytes, io_filters_bytes; + const void *raw_io_ranges = adt_getprop(adt, defaults, "pmap-io-ranges", &io_ranges_bytes); + const void *raw_io_filters = adt_getprop(adt, defaults, "pmap-io-filters", &io_filters_bytes); + if (!raw_io_ranges) + io_ranges_bytes = 0; + if (!raw_io_filters) + io_filters_bytes = 0; + size_t io_ranges_size = ALIGN_UP(max(io_ranges_bytes, 1U), SPTM_PAGE_SIZE); + size_t io_filters_size = ALIGN_UP(max(io_filters_bytes, 1U), SPTM_PAGE_SIZE); + struct sptm_boot_io_range *io_ranges = + sptm_alloc_zero(&context.allocator, io_ranges_size, SPTM_PAGE_SIZE); + struct sptm_boot_io_filter *io_filters = + sptm_alloc_zero(&context.allocator, io_filters_size, SPTM_PAGE_SIZE); + memcpy(io_ranges, raw_io_ranges, io_ranges_bytes); + memcpy(io_filters, raw_io_filters, io_filters_bytes); + u32 io_range_count = io_ranges_bytes / sizeof(*io_ranges); + u32 io_filter_count = io_filters_bytes / sizeof(*io_filters); + sptm_boot_sort_io_ranges(io_ranges, io_range_count); + sptm_boot_sort_io_filters(io_filters, io_filter_count); + + context.ttbr1_pa = sptm_boot_alloc_table(&context, 1); + context.ttbr0_pa = sptm_boot_alloc_table(&context, 1); + // Locked platforms need Outer-shareable aliases for cross-cluster stability. + u64 data_leaf = + !cpu_features->apple_sysregs_unlocked ? SPTM_LEAF_DATA_OUTER : SPTM_LEAF_DATA; + sptm_boot_map_page(&context, context.ttbr0_pa, 0, ttbr0_zero_page_pa, data_leaf); + sptm_boot_map_range(&context, context.ttbr1_pa, context.kernel_vmin, context.kernel_pa, + ALIGN_UP(context.kernel_vmax - context.kernel_vmin, SPTM_PAGE_SIZE), + SPTM_LEAF_CODE); + sptm_boot_map_range(&context, context.ttbr1_pa, context.physmap_base, context.managed_start, + context.managed_end - context.managed_start, data_leaf); + // Override the global UAT root's physmap alias as Normal-NC. + sptm_boot_map_page(&context, context.ttbr1_pa, + sptm_boot_va(&context, context.uat_global_root_pa), + context.uat_global_root_pa, SPTM_LEAF_DATA_NC); + + // XNU expects the locked ADT and TrustCache as EXTRADATA just below the kernelcache. + sptm_boot_map_range(&context, context.ttbr1_pa, extradata_va, extradata_pa, extradata_size, + data_leaf); + bootargs->devtree = (void *)(extradata_va + guest_adt - extradata_pa); + + context.uat_l2_pa = sptm_adt_integer(sgx, "gfx-shared-l2-region-base", 0); + sptm_boot_preallocate_l3(&context, context.ttbr1_pa, dynamic_l3_start, dynamic_l3_end); + // Keep the firmware-owned selector-9 page outside the physmap and empty-table range. + sptm_boot_map_page(&context, context.ttbr1_pa, context.uat_l2_va, context.uat_l2_pa, + SPTM_LEAF_DATA_NC); + // The topmost L3 backs early debug data and per-CPU copy windows. + sptm_boot_preallocate_l3(&context, context.ttbr1_pa, high_l3_end - SZ_32M, high_l3_end); + + // Seed frames already live at handoff; XNU types the remaining RAM as it claims it. + for (size_t frame = 0; frame < managed_pages; frame++) + context.frame_table[frame * SPTM_FRAME_ENTRY_SIZE + 2] = SPTM_FRAME_DEFAULT; + context.frame_priority = + sptm_alloc_zero(&context.allocator, managed_pages, SPTM_CACHE_LINE_SIZE); + sptm_boot_type_kernel_frames(&context); + // INIT_STATE never sees these objects. + sptm_boot_set_frame(&context, context.uat_global_root_pa, SPTM_FRAME_XNU_IOMMU, 0, 0); + sptm_boot_set_frame(&context, context.uat_global_state_pa, SPTM_FRAME_XNU_IOMMU, 0, 0); + for (u32 index = 0; index < sptm_boot.page_count; index++) { + const struct sptm_boot_pt_page *page = &sptm_boot.pages[index]; + u8 type = page->pa == context.ttbr1_pa ? SPTM_FRAME_KERNEL_ROOT + : page->pa == context.ttbr0_pa ? SPTM_FRAME_USER_ROOT + : SPTM_FRAME_PAGE_TABLE; + sptm_boot_set_frame(&context, page->pa, type, page->level, sptm_boot_valid_count(page->pa)); + } + + // libsptm interprets kind 2 as a table body and kind 5 as a data body. + for (u32 type = 0; type < 256; type++) + ((u8 *)frame_params_pa)[type * SPTM_FRAME_PARAM_SIZE + 1] = + sptm_is_table_type(type) ? 2 : 5; + // Recover stripped pt-attr globals from stable anchors, in XNU's ABI order: + // 16K, 4K, 16K kernel, then 16K, 16K-36b, and 4K stage 2. + u64 *pt_attrs = (void *)pt_attrs_pa; + pt_attrs[0] = cons_ops + 0x338; + pt_attrs[1] = cons_ops + 0x248; + pt_attrs[2] = cons_ops + 0x2c0; + pt_attrs[3] = page_shift_const - 0xd18; + pt_attrs[4] = page_shift_const - 0xca0; + pt_attrs[5] = page_shift_const - 0xc28; + + // Put specific aliases before the catch-all physmap so overlaps resolve to them. + struct sptm_boot_papt *papt = (void *)papt_ranges_pa; + papt[0] = + (struct sptm_boot_papt){extradata_pa, extradata_va, extradata_size / SPTM_PAGE_SIZE, 0}; + papt[1] = (struct sptm_boot_papt){context.uat_l2_pa, context.uat_l2_va, 1, 0}; + papt[2] = + (struct sptm_boot_papt){context.managed_start, context.physmap_base, + (context.managed_end - context.managed_start) / SPTM_PAGE_SIZE, 0}; + write32(papt_count_pa, 3); + + // Cold boot dereferences all three image slots; reuse XNU without SPTM/TXM symbols. + write32(debug_header_pa, 0x47424544); + write32(debug_header_pa + 4, 2); + write32(debug_header_pa + 8, 3); + write64(debug_header_pa + 0x10, xnu_text); + write64(debug_header_pa + 0x18, xnu_text); + write64(debug_header_pa + 0x20, xnu_text); + + cpu = cpus; + ADT_FOREACH_CHILD(adt, cpu) + sptm.hv_phys_ids[sptm_adt_integer(cpu, "cpu-id", 0)] = sptm_adt_integer(cpu, "reg", 0); + for (u32 index = 0; index < SPTM_TXM_BOOT_CHANNELS; index++) + write64(txm_stack_array_pa + index * sizeof(u64), + sptm_boot_va(&context, txm_stacks_pa + index * SPTM_PAGE_SIZE)); + // TXM uses this zeroed policy page to disable monitor signing and enable developer mode. + write64(txm_info_pa + 0x190, context.txm_info_va); + write8(txm_info_pa + 0x318, 1); + + // This boot-created global UAT state never passes through INIT_STATE. + write8(context.uat_global_state_pa, uat_mode == 0 ? 2 : 8); + write64(context.uat_global_state_pa + 8, 0xffffffff); + write64(context.uat_global_state_pa + 0x10, context.uat_global_root_pa); + write16(context.uat_global_state_pa + 0x18, 0xffff); + write8(context.uat_global_state_pa + 0x1a, 2); + // G15+ UAT exposes the firmware-shared L2 through top-level slot 2. + write64(context.uat_global_root_pa + 2 * sizeof(u64), context.uat_l2_pa | SPTM_TABLE_DESC); + + // arm_init copies this one-shot cold-entry record before using its pointers. + u8 *bootstrap = (void *)context.bootstrap_pa; + write64((u64)bootstrap + 0x00, sptm_boot_va(&context, context.scratch_pa)); + write64((u64)bootstrap + 0x08, context.physmap_base); + write64((u64)bootstrap + 0x10, context.physmap_end); + write64((u64)bootstrap + 0x18, aux_end); + write64((u64)bootstrap + 0x30, sptm_boot_va(&context, txm_stack_array_pa)); + write32((u64)bootstrap + 0x38, cpu_count); + write64((u64)bootstrap + 0x40, sptm_boot_va(&context, kernel_stacks_pa)); + write64((u64)bootstrap + 0x48, + sptm_boot_va(&context, kernel_stacks_pa + SPTM_CPU_STACK_WINDOW_SIZE)); + write64((u64)bootstrap + 0x50, context.kernel_vmin); + write64((u64)bootstrap + 0x58, context.kernel_vmax); + write64((u64)bootstrap + 0x60, sptm_boot_va(&context, debug_header_pa)); + write32((u64)bootstrap + 0x68, sptm_adt_integer(defaults, "pmap-max-asids", 256)); + memcpy(bootstrap + 0x6c, "randseed", 8); + memcpy(bootstrap + 0x74, adt_getprop(adt, chosen, "random-seed", NULL), 256); + write64((u64)bootstrap + 0x178, 0x108); + write64((u64)bootstrap + 0x198, sptm_boot_va(&context, context.panic_state_pa)); + + // The nested libsptm state becomes XNU's persistent SPTM client state. + u64 libsptm = (u64)bootstrap + 0x1a0; + write64(libsptm + 0x00, 10); + write64(libsptm + 0x08, sptm_boot_va(&context, papt_count_pa)); + write64(libsptm + 0x10, sptm_boot_va(&context, papt_ranges_pa)); + write64(libsptm + 0x18, context.managed_start); + write64(libsptm + 0x20, context.managed_end); + write64(libsptm + 0x28, context.physmap_base); + write64(libsptm + 0x30, context.physmap_end); + write64(libsptm + 0x38, context.ttbr1_pa); + write64(libsptm + 0x40, sptm_boot_va(&context, context.frame_table_pa)); + write64(libsptm + 0x48, sptm_boot_va(&context, frame_params_pa)); + write64(libsptm + 0x50, sptm_boot_va(&context, pt_attrs_pa)); + write64(libsptm + 0x80, sptm_boot_va(&context, context.panic_state_pa + 8)); + write64(libsptm + 0x88, sptm_boot_va(&context, trace_buffer_pa)); + write64(libsptm + 0x90, sptm_boot_va(&context, dispatch_states_pa)); + write64(libsptm + 0x98, cpu_count); + write64(libsptm + 0xa0, sptm_boot_va(&context, saved_states_pa)); + write64(libsptm + 0xa8, 8); + // Early boot and persistent libsptm state each need their own I/O-range copy. + write64(libsptm + 0xc0, sptm_boot_va(&context, (u64)io_ranges)); + write64(libsptm + 0xc8, io_range_count); + write64(libsptm + 0xd0, sptm_boot_va(&context, context.panic_state_pa + 12)); + write64(libsptm + 0xd8, sptm_boot_va(&context, event_counters_pa)); + // With no AuxKC, XNU still needs this value to derive the kernelcache bounds. + write64((u64)bootstrap + 0x2e8, context.kernel_vmax); + write64((u64)bootstrap + 0x318, sptm_boot_va(&context, (u64)io_ranges)); + write32((u64)bootstrap + 0x320, io_range_count); + write64((u64)bootstrap + 0x328, sptm_boot_va(&context, (u64)io_filters)); + write32((u64)bootstrap + 0x330, io_filter_count); + write64((u64)bootstrap + 0x338, 0x10); + + // Start with no panicking CPU/domain and the cold-boot dispatch states XNU expects. + write16(context.panic_state_pa + 8, 0xffff); + write32(context.panic_state_pa + 12, 0xff); + write32(dispatch_states_pa + 0xa30, 5); + write32(dispatch_states_pa + 0xa60, 5); + + msr(TTBR0_EL12, context.ttbr0_pa); + msr(TTBR1_EL12, context.ttbr1_pa); + msr(TCR_EL12, SPTM_TCR_BOOT); + msr(MAIR_EL12, SPTM_MAIR_BOOT); + msr(AMAIR_EL12, 0); + msr(CONTEXTIDR_EL12, 0); + msr(VBAR_EL12, 0); + msr(SCTLR_EL12, SPTM_SCTLR_BOOT); + + sptm.managed_start = context.managed_start; + sptm.managed_end = context.managed_end; + sptm.physmap_base = context.physmap_base; + sptm.physmap_end = context.physmap_end; + sptm.scratch_pa = context.scratch_pa; + sptm.kernel_root = context.ttbr1_pa; + sptm.max_cpus = cpu_count; + sptm.frame_table_pa = context.frame_table_pa; + sptm.panic_state_pa = context.panic_state_pa; + sptm.txm_info_va = context.txm_info_va; + sptm.enabled = true; + + sptm.uat_global_state = context.uat_global_state_pa; + sptm.uat_gpu_contexts = sptm_adt_integer(sgx, "gpu-region-base", 0); + sptm.uat_shared_l2_pa = context.uat_l2_pa; + sptm.uat_shared_l2_va = context.uat_l2_va; + sptm.uat_segment_limit = uat_segment_limit; + sptm.uat_mapping_limit = uat_mapping_limit; + sptm.uat_va_width = uat_va_width; + sptm.uat_mode = uat_mode; + sptm.uat_tlbi_at_retype = uat_tlbi_at_retype; + sptm_uat_configure(context.uat_global_root_pa, sptm_adt_integer(sgx, "gfx-handoff-base", 0)); + sptm_init_platform(&context.allocator); + + sptm_boot.entry = context.kernel_entry; + sptm_boot.bootargs_va = sptm_boot_va(&context, bootargs_pa); + sptm_boot.bootstrap_va = sptm_boot_va(&context, context.bootstrap_pa); + sptm_boot.ready = true; + adt = firmware_adt; + return context.panic_state_pa; +} + +void sptm_boot_prepare_start(void **entry, u64 regs[4], bool secondary) +{ + if (!sptm_boot.ready) + return; + *entry = (void *)sptm_boot.entry; + regs[0] = secondary; + regs[1] = sptm_boot.bootargs_va; + regs[2] = sptm_boot.bootstrap_va; + regs[3] = 0; +} diff --git a/src/hv_sptm_core.c b/src/hv_sptm_core.c new file mode 100644 index 00000000..a9e7fdec --- /dev/null +++ b/src/hv_sptm_core.c @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include "hv_sptm_internal.h" + +struct sptm_state sptm; + +// Whether [pa, pa + size) lies in XNU-managed RAM, which has frame-table entries. +bool sptm_is_managed_range(u64 pa, size_t size) +{ + return pa >= sptm.managed_start && pa < sptm.managed_end && size <= sptm.managed_end - pa; +} + +// XNU passes its own data as physmap addresses; physical addresses pass through. +u64 sptm_pointer_pa(u64 pointer) +{ + if (pointer >= sptm.physmap_base && pointer < sptm.physmap_end) + return sptm.managed_start + pointer - sptm.physmap_base; + return pointer; +} + +u64 sptm_physmap_va(u64 pa) +{ + return sptm.physmap_base + pa - sptm.managed_start; +} + +u64 sptm_iommu_table_va(u64 pa) +{ + return pa | REGION_NORMAL_NC; +} + +struct sptm_cpu *sptm_find_cpu(u32 phys_id) +{ + for (size_t index = 0; index < ARRAY_SIZE(sptm.cpus); index++) { + struct sptm_cpu *cpu = &sptm.cpus[index]; + if (cpu->valid && cpu->phys_id == phys_id) + return cpu; + } + + return NULL; +} + +struct sptm_cpu *sptm_register_cpu(u32 phys_id) +{ + struct sptm_cpu *cpu = sptm_find_cpu(phys_id); + if (cpu) + return cpu; + + if (sptm.cpu_count >= sptm.max_cpus) + return NULL; + + cpu = &sptm.cpus[sptm.cpu_count]; + cpu->phys_id = phys_id; + cpu->logical_id = sptm.cpu_count; + cpu->valid = true; + sptm.cpu_count++; + return cpu; +} + +u64 sptm_scratch_pa(struct exc_info *ctx) +{ + struct sptm_cpu *cpu = sptm_find_cpu(sptm.hv_phys_ids[ctx->cpu_id]); + return cpu ? sptm.scratch_pa + cpu->logical_id * SPTM_PAGE_SIZE : 0; +} diff --git a/src/hv_sptm_dart.c b/src/hv_sptm_dart.c new file mode 100644 index 00000000..8e5d8322 --- /dev/null +++ b/src/hv_sptm_dart.c @@ -0,0 +1,247 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include "hv_sptm_internal.h" + +static const u32 sptm_dart_shifts[] = {36, 25, SPTM_PAGE_SHIFT}; + +static struct sptm_dart *sptm_find_dart(u32 id) +{ + for (size_t index = 0; index < sptm.dart_count; index++) { + if (sptm.darts[index].id == id) + return &sptm.darts[index]; + } + + return NULL; +} + +static bool sptm_dart_tlb_op(struct sptm_dart *dart, u32 op) +{ + // TLB registers are inaccessible while the DART is powered down. + if (!dart->powered) + return true; + + sysop("dsb sy"); + for (size_t index = 0; index < dart->instance_count; index++) { + u64 base = dart->instances[index]; + write32(base + DART_TLB_OP, op); + size_t retry = 0; + while (read32(base + DART_TLB_OP) & DART_TLB_BUSY) { + if (++retry == 10000) { + printf("HV: SPTM DART %u TLB operation 0x%x timed out at 0x%lx\n", dart->id, op, + base); + return false; + } + } + } + sysop("dsb sy"); + return true; +} + +// Every leaf or table change flushes the whole SID rather than a DVA range. +static bool sptm_dart_flush_sid(struct sptm_dart *dart, u32 sid) +{ + return sptm_dart_tlb_op(dart, DART_TLB_FLUSH_SID | sid); +} + +bool sptm_dart_flush_all(struct sptm_dart *dart) +{ + return sptm_dart_tlb_op(dart, DART_TLB_FLUSH_ALL); +} + +static u64 sptm_dart_desc_pa(u64 descriptor) +{ + return ((descriptor >> 10) & DART_PTE_OFFSET_MASK) << SPTM_PAGE_SHIFT; +} + +static u64 sptm_dart_desc(u64 pa) +{ + return (((pa >> SPTM_PAGE_SHIFT) & DART_PTE_OFFSET_MASK) << 10) | DART_PTE_VALID; +} + +/* + * Walk from the SID's root to dva's slot in the level-`level` table. Live DART + * tables are only accessed through their Normal-NC alias, which is returned. + */ +static u64 sptm_dart_slot(const struct sptm_dart_sid *state, u64 dva, u32 level) +{ + u64 table = state->root; + for (u32 current = state->root_level;; current++) { + u64 slot = + sptm_iommu_table_va(table) + ((dva >> sptm_dart_shifts[current]) & 0x7ff) * sizeof(u64); + if (current == level) + return slot; + table = sptm_dart_desc_pa(read64(slot)); + } +} + +// Firmware-locked TTBRs already hold their value and cannot be written. +static void sptm_dart_set_ttbr(struct sptm_dart *dart, u32 sid, u32 ttbr) +{ + if (!dart->powered) + return; + + for (size_t index = 0; index < dart->instance_count; index++) { + u64 base = dart->instances[index]; + if (!(read32(base + DART_REG_PROTECT) & BIT(0))) + write32(base + DART_TTBR + sid * sizeof(u32), ttbr); + } +} + +// MAP_TABLE and UNMAP_TABLE: install or clear the SID's root, or one table descriptor. +static bool sptm_dart_set_table(struct exc_info *ctx, struct sptm_dart *dart, bool map) +{ + u32 sid = ctx->regs[1]; + u64 dva = ctx->regs[2]; + u32 level = ctx->regs[3]; + u64 table_pa = map ? ctx->regs[4] : 0; + struct sptm_dart_sid *state = &dart->sid_states[sid]; + + if (level == state->root_level) { + sptm_dart_set_ttbr(dart, sid, + map ? ((table_pa >> 12) & 0xfffffffcULL) | DART_TTBR_VALID : 0); + state->root = table_pa; + } else { + write64(sptm_dart_slot(state, dva, level - 1), map ? sptm_dart_desc(table_pa) : 0); + } + + if (!sptm_dart_flush_sid(dart, sid)) + return false; + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +// Leaves expose their whole page: the subpage range is 4-byte units 0 to 0xfff. +static u64 sptm_dart_page_pte(u64 target, u64 options) +{ + return (0xfffULL << 40) | sptm_dart_desc(target) | (options & BIT(0) ? DART_PTE_RDPROT : 0) | + (options & BIT(1) ? DART_PTE_WRPROT : 0) | (options & BIT(2) ? DART_PTE_UNCACHABLE : 0); +} + +static bool sptm_dart_map_pages(struct exc_info *ctx, struct sptm_dart *dart) +{ + u32 sid = ctx->regs[1]; + u64 dva = ctx->regs[2]; + u64 targets = sptm_pointer_pa(ctx->regs[3]); + u64 size = ctx->regs[4]; + u64 options = ctx->regs[5]; + struct sptm_dart_sid *state = &dart->sid_states[sid]; + u64 start = dva & ~SPTM_PAGE_MASK; + u64 end = ALIGN_UP(dva + size, SPTM_PAGE_SIZE); + bool changed = false; + + for (u64 page_dva = start; page_dva < end; page_dva += SPTM_PAGE_SIZE) { + u64 target = read64(targets + ((page_dva - start) >> SPTM_PAGE_SHIFT) * sizeof(u64)); + u64 slot = sptm_dart_slot(state, page_dva, 2); + u64 pte = sptm_dart_page_pte(target, options); + changed |= read64(slot) != pte; + write64(slot, pte); + } + sysop("dsb sy"); + + if (changed && !sptm_dart_flush_sid(dart, sid)) + return false; + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +static bool sptm_dart_unmap_pages(struct exc_info *ctx, struct sptm_dart *dart) +{ + u32 sid = ctx->regs[1]; + u64 dva = ctx->regs[2]; + u64 size = ctx->regs[3]; + struct sptm_dart_sid *state = &dart->sid_states[sid]; + u64 start = dva & ~SPTM_PAGE_MASK; + u64 end = ALIGN_UP(dva + size, SPTM_PAGE_SIZE); + bool changed = false; + + for (u64 page_dva = start; page_dva < end; page_dva += SPTM_PAGE_SIZE) { + u64 slot = sptm_dart_slot(state, page_dva, 2); + u64 old = read64(slot); + if (!(old & DART_PTE_VALID)) + continue; + write64(slot, 0); + changed = true; + } + sysop("dsb sy"); + + if (changed && !sptm_dart_flush_sid(dart, sid)) + return false; + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +static bool sptm_dart_set_stream(struct exc_info *ctx, struct sptm_dart *dart, bool enable) +{ + u32 sid = ctx->regs[1]; + struct sptm_dart_sid *state = &dart->sid_states[sid]; + + // Non-translating SIDs keep their streams, and retention DARTs keep them enabled. + if ((state->tcr & DART_TCR_TRANSLATE) && (enable || !(dart->flags & SPTM_DART_RETENTION))) { + u32 word = sid / 32; + u32 bit = BIT(sid % 32); + if (dart->powered) { + u64 offset = (enable ? DART_ENABLE_STREAMS : DART_DISABLE_STREAMS) + word * sizeof(u32); + for (size_t index = 0; index < dart->instance_count; index++) + write32(dart->instances[index] + offset, bit); + } else if (dart->saved_valid) { + /* Power-up restores the deferred stream state. */ + for (size_t index = 0; index < dart->instance_count; index++) { + if (enable) + dart->saved_streams[index][word] |= bit; + else + dart->saved_streams[index][word] &= ~bit; + } + } + sysop("dsb sy"); + } + + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +bool sptm_handle_dart(struct exc_info *ctx, u32 endpoint) +{ + // The remaining endpoints, including error clearing, are no-ops. + if (endpoint > 8) { + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; + } + + struct sptm_dart *dart = sptm_find_dart(ctx->regs[0]); + if (!dart) + return false; + + bool handled; + spin_lock(&sptm.service_lock); + switch (endpoint) { + case 0: /* MAP_TABLE */ + handled = sptm_dart_set_table(ctx, dart, true); + break; + case 1: /* UNMAP_TABLE */ + handled = sptm_dart_set_table(ctx, dart, false); + break; + case 2: /* MAP_PAGE */ + handled = sptm_dart_map_pages(ctx, dart); + break; + case 3: /* UNMAP_PAGE */ + handled = sptm_dart_unmap_pages(ctx, dart); + break; + case 4: /* POWERDOWN */ + handled = sptm_dart_power(ctx, dart, false); + break; + case 5: /* POWERUP */ + handled = sptm_dart_power(ctx, dart, true); + break; + case 6: /* INIT */ + handled = sptm_dart_init(ctx, dart); + break; + case 7: /* DISABLE_TRANSLATION */ + handled = sptm_dart_set_stream(ctx, dart, false); + break; + default: /* ENABLE_TRANSLATION */ + handled = sptm_dart_set_stream(ctx, dart, true); + break; + } + spin_unlock(&sptm.service_lock); + return handled; +} diff --git a/src/hv_sptm_dart_power.c b/src/hv_sptm_dart_power.c new file mode 100644 index 00000000..dafdd06d --- /dev/null +++ b/src/hv_sptm_dart_power.c @@ -0,0 +1,348 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include "hv_sptm_internal.h" + +// Powered-down or absent DART registers read as DART_UNAVAILABLE or all ones. +static bool sptm_dart_live(u32 value) +{ + return value != DART_UNAVAILABLE && value != UINT32_MAX; +} + +static bool sptm_dart_clear_errors(const struct sptm_dart *dart) +{ + for (size_t index = 0; index < dart->instance_count; index++) { + u64 base = dart->instances[index]; + + if (dart->sid_exceptions) { + // DART 2.2 uses write-one-to-clear per-SID exception words. + for (size_t word = 0; word < dart->sid_words; word++) { + u64 address = base + DART_SID_EXCEPTIONS + word * sizeof(u32); + u32 pending = read32(address); + if (pending && sptm_dart_live(pending)) + write32(address, pending); + } + } else { + u32 error = read32(base + DART_ERROR); + if (!sptm_dart_live(error)) + return false; + if (error & DART_ERROR_SECONDARY) { + if (!(dart->flags & SPTM_DART_IGNORE_SECONDARY)) + return false; + u32 secondary = read32(base + 0x1c0); + if (secondary && sptm_dart_live(secondary)) + write32(base + 0x1c0, secondary); + } + if (error) + write32(base + DART_ERROR, error); + } + + write32(base + dart->interrupt_status_offset, UINT32_MAX); + } + + sysop("dsb sy"); + return true; +} + +// Set or clear an FPAG slice's clock protection; fails if the slice is unreadable. +static bool sptm_dart_clock_protect(u64 address, bool protect) +{ + u32 value = read32(address); + if (!sptm_dart_live(value)) + return false; + write32(address, protect ? value | FPAG_CLOCK_PROTECTION : value & ~FPAG_CLOCK_PROTECTION); + return true; +} + +static struct sptm_dart_clock_ref *sptm_dart_clock_ref(u64 address, bool create) +{ + struct sptm_dart_clock_ref *empty = NULL; + + for (size_t index = 0; index < ARRAY_SIZE(sptm.dart_clock_refs); index++) { + struct sptm_dart_clock_ref *ref = &sptm.dart_clock_refs[index]; + if (ref->valid && ref->address == address) + return ref; + if (!ref->valid && !empty) + empty = ref; + } + + if (!create || !empty) + return NULL; + empty->address = address; + empty->valid = true; + return empty; +} + +static bool sptm_dart_shared_clock_acquire(u64 address) +{ + struct sptm_dart_clock_ref *ref = sptm_dart_clock_ref(address, true); + if (!ref || ref->refs == UINT16_MAX) + return false; + + if (!ref->refs) { + u32 value = read32(address); + if (!sptm_dart_live(value)) + return false; + ref->original = value & FPAG_CLOCK_PROTECTION; + if (!ref->original) + write32(address, value | FPAG_CLOCK_PROTECTION); + } + ref->refs++; + return true; +} + +static void sptm_dart_shared_clock_release(u64 address) +{ + struct sptm_dart_clock_ref *ref = sptm_dart_clock_ref(address, false); + if (!ref || !ref->refs) + return; + + ref->refs--; + if (ref->refs) + return; + + if (!ref->original) + sptm_dart_clock_protect(address, false); + memset(ref, 0, sizeof(*ref)); +} + +static bool sptm_dart_powerup_sidebands(struct sptm_dart *dart) +{ + if (dart->clock_count) { + if (dart->flags & SPTM_DART_UNGANG_SHARED_PS) { + // apcie0 and apcie2 share this FPAG slice on T8132. + if (dart->clock_active) + return true; + + size_t acquired = 0; + for (; acquired < dart->clock_count; acquired++) { + if (!sptm_dart_shared_clock_acquire(dart->clock_entries[acquired])) + break; + } + if (acquired != dart->clock_count) { + while (acquired) + sptm_dart_shared_clock_release(dart->clock_entries[--acquired]); + return false; + } + dart->clock_active = true; + sysop("dsb sy"); + return true; + } + + for (size_t index = 0; index < dart->clock_count; index++) { + if (!sptm_dart_clock_protect(dart->clock_entries[index], true)) + return false; + } + } + sysop("dsb sy"); + return true; +} + +static u64 sptm_dart_read_sized(u64 address, u32 size) +{ + switch (size) { + case 1: + return read8(address); + case 2: + return read16(address); + case 4: + return read32(address); + default: + return read64(address); + } +} + +static void sptm_dart_write_sized(u64 address, u32 size, u64 value) +{ + switch (size) { + case 1: + write8(address, value); + break; + case 2: + write16(address, value); + break; + case 4: + write32(address, value); + break; + default: + write64(address, value); + break; + } +} + +static bool sptm_dart_apply_tunables(const struct sptm_dart *dart) +{ + for (size_t index = 0; index < dart->tunable_count; index++) { + const struct sptm_dart_tunable *entry = &dart->tunable_entries[index]; + u64 address = entry->base + entry->offset; + u64 wanted = entry->value & entry->mask; + u64 current = sptm_dart_read_sized(address, entry->size); + if ((current & entry->mask) != wanted) { + sptm_dart_write_sized(address, entry->size, (current & ~entry->mask) | wanted); + if ((sptm_dart_read_sized(address, entry->size) & entry->mask) != wanted) + return false; + } + } + sysop("dsb sy"); + return true; +} + +static void sptm_dart_program_dapf(const struct sptm_dart *dart) +{ + for (size_t index = 0; index < dart->dapf_count; index++) { + const struct sptm_dart_dapf *entry = &dart->dapf_entries[index]; + write32(entry->base + 0x04, entry->r4); + write64(entry->base + 0x08, entry->start); + write64(entry->base + 0x10, entry->end); + write32(entry->base + 0x00, entry->control); + write32(entry->base + 0x20, entry->r20); + } + sysop("dsb sy"); +} + +static void sptm_dart_powerdown_sidebands(struct sptm_dart *dart) +{ + if (dart->flags & SPTM_DART_UNGANG_SHARED_PS) { + if (!dart->clock_active) + return; + for (size_t index = 0; index < dart->clock_count; index++) + sptm_dart_shared_clock_release(dart->clock_entries[index]); + dart->clock_active = false; + sysop("dsb sy"); + return; + } + + for (size_t index = 0; index < dart->clock_count; index++) + sptm_dart_clock_protect(dart->clock_entries[index], false); + sysop("dsb sy"); +} + +static bool sptm_dart_program_sids(struct sptm_dart *dart) +{ + for (u32 sid = 0; sid < dart->sid_count; sid++) { + struct sptm_dart_sid *state = &dart->sid_states[sid]; + if (!(state->flags & SPTM_DART_SID_KNOWN) || (state->flags & SPTM_DART_SID_EXCLAVE)) + continue; + + u32 ttbr = 0; + if ((state->tcr & DART_TCR_TRANSLATE) && state->root) + ttbr = ((state->root >> 12) & 0xfffffffcULL) | DART_TTBR_VALID; + + // Firmware-locked instances keep the SID configuration it programmed. + for (size_t index = 0; index < dart->instance_count; index++) { + u64 base = dart->instances[index]; + if (read32(base + DART_REG_PROTECT) & BIT(0)) + continue; + write32(base + DART_TTBR + sid * sizeof(u32), ttbr); + write32(base + DART_TCR + sid * sizeof(u32), state->tcr); + } + } + return sptm_dart_flush_all(dart); +} + +// INIT discovers the version and SID capacity, which all instances of one DART share. +bool sptm_dart_init(struct exc_info *ctx, struct sptm_dart *dart) +{ + u32 params_8 = read32(dart->instances[0] + DART_PARAMS_8); + u32 params_c = read32(dart->instances[0] + DART_PARAMS_C); + if (!sptm_dart_live(params_8)) + return false; + + u8 major = (params_8 >> 8) & 0xff; + u8 minor = params_8 & 0xff; + dart->sid_exceptions = major > 2 || (major == 2 && minor >= 2); + dart->sid_words = ((params_c & 0x1ff) + 31) / 32; + dart->interrupt_status_offset = dart->sid_exceptions ? 0x164 : 0x160; + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +static bool sptm_dart_save_power_state(struct sptm_dart *dart) +{ + for (size_t index = 0; index < dart->instance_count; index++) { + u64 base = dart->instances[index]; + for (size_t word = 0; word < dart->sid_words; word++) { + u32 streams = read32(base + DART_ENABLE_STREAMS + word * sizeof(u32)); + if (!sptm_dart_live(streams)) + return false; + dart->saved_streams[index][word] = streams; + } + } + dart->saved_valid = true; + return true; +} + +static void sptm_dart_disable_translating_streams(struct sptm_dart *dart) +{ + u32 translating[(SPTM_MAX_DART_SIDS + 31) / 32] = {}; + for (u32 sid = 0; sid < dart->sid_count; sid++) { + struct sptm_dart_sid *state = &dart->sid_states[sid]; + if ((state->flags & SPTM_DART_SID_KNOWN) && !(state->flags & SPTM_DART_SID_EXCLAVE) && + (state->tcr & DART_TCR_TRANSLATE)) + translating[sid / 32] |= BIT(sid % 32); + } + for (size_t index = 0; index < dart->instance_count; index++) { + for (size_t word = 0; word < dart->sid_words; word++) { + if (translating[word]) + write32(dart->instances[index] + DART_DISABLE_STREAMS + word * sizeof(u32), + translating[word]); + } + } + sysop("dsb sy"); +} + +static void sptm_dart_restore_runtime_state(struct sptm_dart *dart) +{ + for (size_t index = 0; index < dart->instance_count; index++) { + u64 base = dart->instances[index]; + if (dart->limits_valid && dart->saved_valid) { + write32(base + DART_TEQRESERVE, dart->saved_teqreserve[index]); + write32(base + DART_TLIMIT, dart->saved_tlimit[index]); + } + for (size_t word = 0; word < dart->sid_words; word++) { + u32 streams = + dart->saved_valid ? dart->saved_streams[index][word] : dart->boot_streams[word]; + if (streams) + write32(base + DART_ENABLE_STREAMS + word * sizeof(u32), streams); + } + } + sysop("dsb sy"); +} + +bool sptm_dart_power(struct exc_info *ctx, struct sptm_dart *dart, bool power_up) +{ + if (power_up) { + if (!sptm_dart_powerup_sidebands(dart)) + return false; + + // Mark the DART powered before programming SIDs and flushing. + dart->powered = true; + + if (!sptm_dart_apply_tunables(dart) || !sptm_dart_program_sids(dart)) + return false; + if (!dart->limits_valid) { + for (size_t index = 0; index < dart->instance_count; index++) { + u64 base = dart->instances[index]; + dart->saved_tlimit[index] = read32(base + DART_TLIMIT); + dart->saved_teqreserve[index] = read32(base + DART_TEQRESERVE); + } + dart->limits_valid = true; + } + sptm_dart_program_dapf(dart); + if (!sptm_dart_clear_errors(dart)) + return false; + sptm_dart_restore_runtime_state(dart); + } else { + if (!dart->powered) { + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; + } + if (!sptm_dart_save_power_state(dart)) + return false; + sptm_dart_disable_translating_streams(dart); + sptm_dart_powerdown_sidebands(dart); + dart->powered = false; + } + + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} diff --git a/src/hv_sptm_frame.c b/src/hv_sptm_frame.c new file mode 100644 index 00000000..2040750e --- /dev/null +++ b/src/hv_sptm_frame.c @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include "hv_sptm_internal.h" + +bool sptm_is_table_type(u8 type) +{ + switch (type) { + case SPTM_FRAME_KERNEL_ROOT: + case SPTM_FRAME_PAGE_TABLE: + case SPTM_FRAME_USER_ROOT: + case SPTM_FRAME_SHARED_ROOT: + case SPTM_FRAME_XNU_ROOT: + case SPTM_FRAME_SHARED_PT: + case SPTM_FRAME_ROZONE_PT: + case SPTM_FRAME_COMMPAGE_PT: + case SPTM_FRAME_STAGE2_ROOT: + case SPTM_FRAME_STAGE2_PT: + case SPTM_FRAME_SUBPAGE: + return true; + default: + return false; + } +} + +bool sptm_is_root_type(u8 type) +{ + switch (type) { + case SPTM_FRAME_KERNEL_ROOT: + case SPTM_FRAME_USER_ROOT: + case SPTM_FRAME_SHARED_ROOT: + case SPTM_FRAME_XNU_ROOT: + case SPTM_FRAME_STAGE2_ROOT: + case SPTM_FRAME_SUBPAGE: + return true; + default: + return false; + } +} + +u8 *sptm_frame_entry(u64 pa) +{ + if ((pa & SPTM_PAGE_MASK) || !sptm_is_managed_range(pa, SPTM_PAGE_SIZE)) + return NULL; + + u64 index = (pa - sptm.managed_start) >> SPTM_PAGE_SHIFT; + return (u8 *)(sptm.frame_table_pa + index * 16); +} + +// XNU reads a page table's valid-entry count to decide when the table is empty. +void sptm_adjust_table_valid(u64 slot, int delta) +{ + u64 count = (u64)sptm_frame_entry(slot & ~SPTM_PAGE_MASK) + 8; + write16(count, read16(count) + delta); +} diff --git a/src/hv_sptm_init.c b/src/hv_sptm_init.c new file mode 100644 index 00000000..5614704b --- /dev/null +++ b/src/hv_sptm_init.c @@ -0,0 +1,349 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include "hv_sptm_internal.h" + +#include "adt.h" + +#define SPTM_INIT_MAX_INSTANCES 8 +#define FPAG_PROTECTION_BASE 0x140 +#define FPAG_PROTECTION_STRIDE 0x40 +#define DART_TCR_BYPASS BIT(1) +#define DART_TCR_APF_BYPASS BIT(2) +#define DART_TCR_REMAP_ENABLE BIT(7) + +struct sptm_init_instances { + u64 base[SPTM_INIT_MAX_INSTANCES]; + u32 count; +}; + +struct sptm_init_tunable { + u32 offset; + u32 size; + u64 mask; + u64 value; +} PACKED; + +static bool sptm_adt_has(int node, const char *name) +{ + return adt_get_property(adt, node, name) != NULL; +} + +static bool sptm_adt_u32_contains(int node, const char *name, u32 wanted) +{ + u32 size; + const u32 *values = adt_getprop(adt, node, name, &size); + if (!values) + return false; + for (size_t index = 0; index < size / sizeof(*values); index++) { + if (values[index] == wanted) + return true; + } + return false; +} + +static bool sptm_dart_mapper_sid(int node, u32 sid) +{ + int child = node; + ADT_FOREACH_CHILD(adt, child) + { + if (sptm_adt_u32_contains(child, "reg", sid)) + return true; + } + return false; +} + +static bool sptm_dart_remap(int node, u32 sid, u32 *target) +{ + u32 size; + const u32 *remaps = adt_getprop(adt, node, "remap", &size); + if (!remaps) + return false; + for (size_t index = 0; index < size / sizeof(*remaps); index++) { + if ((remaps[index] & 0xff) == sid) { + *target = (remaps[index] >> 8) & 0xff; + return true; + } + } + return false; +} + +static void sptm_dart_instances(int node, int *path, struct sptm_init_instances *trad, + struct sptm_init_instances *fpag, struct sptm_init_instances *fpad) +{ + u32 size; + const char *instances = adt_getprop(adt, node, "instance", &size); + for (size_t index = 0; instances && index < size / 16; index++) { + struct sptm_init_instances *kind = NULL; + const char *tag = instances + index * 16; + if (!memcmp(tag, "TRAD", 4)) + kind = trad; + else if (!memcmp(tag, "FPAG", 4)) + kind = fpag; + else if (!memcmp(tag, "FPAD", 4)) + kind = fpad; + if (!kind) + continue; + + assert(kind->count < ARRAY_SIZE(kind->base)); + adt_get_reg(adt, path, "reg", index, &kind->base[kind->count++], NULL); + } +} + +static void sptm_dart_sid_states(int node, const char *name, struct sptm_dart *dart) +{ + u64 global_dva_size = sptm_adt_integer(node, "vm-size", 0); + bool dynamic = sptm_adt_integer(node, "dart-options", 0) & 0x10; + + for (u32 sid = 0; sid < dart->sid_count; sid++) { + struct sptm_dart_sid *state = &dart->sid_states[sid]; + char property[32]; + state->root_level = 1; + + snprintf(property, sizeof(property), "pt-region-%u", sid); + u32 pt_size; + const u64 *pt_region = adt_getprop(adt, node, property, &pt_size); + bool translating = sptm_adt_u32_contains(node, "sid", sid) || + sptm_dart_mapper_sid(node, sid) || (dynamic && sid) || pt_region; + if (translating) { + // Address spaces larger than 64 GiB start their walk at level 0. + snprintf(property, sizeof(property), "vm-size-%u", sid); + state->root_level = sptm_adt_integer(node, property, global_dva_size) > BIT(36) ? 0 : 1; + } + + u32 remap; + if (sptm_dart_remap(node, sid, &remap)) { + state->flags = SPTM_DART_SID_KNOWN | SPTM_DART_SID_POLICY; + state->tcr = DART_TCR_REMAP_ENABLE | remap << 8; + } else { + snprintf(property, sizeof(property), "bypass-%u", sid); + bool bypass = sptm_adt_has(node, property); + snprintf(property, sizeof(property), "apf-bypass-%u", sid); + bool apf_bypass = sptm_adt_has(node, property); + if (bypass || apf_bypass) { + state->flags = SPTM_DART_SID_KNOWN | SPTM_DART_SID_ENABLED | SPTM_DART_SID_POLICY; + state->tcr = + (bypass ? DART_TCR_BYPASS : 0) | (apf_bypass ? DART_TCR_APF_BYPASS : 0); + } else if (translating) { + state->flags = SPTM_DART_SID_KNOWN; + state->tcr = DART_TCR_TRANSLATE | (state->root_level ? 0 : BIT(3)); + if (pt_region && pt_size >= 2 * sizeof(u64)) { + state->pt_start = pt_region[0]; + state->pt_end = pt_region[1]; + snprintf(property, sizeof(property), "l2-tt-%u", sid); + state->root = sptm_adt_integer(node, property, state->pt_start); + if (!cpu_features->apple_sysregs_unlocked && !strcmp(name, "dart-pmp") && + !sid) + state->flags |= SPTM_DART_SID_ENABLED; + } + } + } + + if (sptm_adt_u32_contains(node, "exclave-sid", sid)) { + state->root = 0; + state->pt_start = 0; + state->pt_end = 0; + state->tcr = 0; + state->root_level = 1; + state->flags = SPTM_DART_SID_KNOWN | SPTM_DART_SID_ENABLED | SPTM_DART_SID_EXCLAVE; + } + + // Firmware page tables bypass RETYPE, so retire their WB aliases here. + for (u64 page = state->pt_start; page < state->pt_end; page += SPTM_PAGE_SIZE) { + u64 host_page = page; + mmu_map_ram_pages_nc(&host_page, 1, true); + } + if (state->flags & SPTM_DART_SID_ENABLED) + dart->boot_streams[sid / 32] |= BIT(sid % 32); + } +} + +static u32 sptm_dart_tunables(int node, const struct sptm_init_instances *trad, + struct sptm_dart_tunable *output) +{ + u32 count = 0; + for (u32 instance = 0; instance < trad->count; instance++) { + char property[40]; + snprintf(property, sizeof(property), "dart-tunables-instance-%u", instance); + u32 size; + const struct sptm_init_tunable *tunables = adt_getprop(adt, node, property, &size); + for (size_t index = 0; tunables && index < size / sizeof(*tunables); index++) { + assert(count < SPTM_MAX_DART_TUNABLES); + output[count].base = trad->base[instance]; + output[count].offset = tunables[index].offset; + output[count].size = tunables[index].size; + output[count].mask = tunables[index].mask; + output[count].value = tunables[index].value; + count++; + } + } + return count; +} + +static u32 sptm_dart_dapf(int node, const struct sptm_init_instances *fpad, + struct sptm_dart_dapf *output) +{ + u32 count = 0; + for (u32 instance = 0; instance < fpad->count; instance++) { + char property[32]; + snprintf(property, sizeof(property), "dapf-instance-%u", instance); + u32 size; + const u8 *entries = adt_getprop(adt, node, property, &size); + if (!entries) + continue; + size_t stride = size % 52 == 0 ? 52 : (size % 56 == 0 ? 56 : 55); + for (size_t index = 0; index < size / stride; index++) { + const u8 *entry = entries + index * stride; + struct sptm_dart_dapf *dapf = &output[count++]; + assert(count <= SPTM_MAX_DART_DAPF); + dapf->base = fpad->base[instance] + index * 0x40; + memcpy(&dapf->start, entry, sizeof(dapf->start)); + memcpy(&dapf->end, entry + 8, sizeof(dapf->end)); + memcpy(&dapf->r20, entry + 16, sizeof(dapf->r20)); + memcpy(&dapf->r4, entry + 24, sizeof(dapf->r4)); + dapf->control = entry[49] << 4 | entry[50]; + } + } + return count; +} + +static u32 sptm_dart_clocks(int node, const struct sptm_init_instances *trad, + const struct sptm_init_instances *fpag, u64 *output) +{ + u32 size; + const u32 *slices = adt_getprop(adt, node, "clock-protection-slice-index", &size); + if (!slices) + return 0; + u32 count = min(size / sizeof(*slices), fpag->count * trad->count); + assert(count <= SPTM_MAX_DART_CLOCKS); + for (u32 index = 0; index < count; index++) { + u32 fpag_index = index / trad->count; + output[index] = fpag->base[fpag_index] + FPAG_PROTECTION_BASE + + (slices[index] - 1) * FPAG_PROTECTION_STRIDE; + } + return count; +} + +static u32 sptm_dart_flags(int node) +{ + return (sptm_adt_has(node, "retention") ? SPTM_DART_RETENTION : 0) | + (sptm_adt_has(node, "ignore-secondary") ? SPTM_DART_IGNORE_SECONDARY : 0) | + (sptm_adt_has(node, "dart-ungang-shared-ps") ? SPTM_DART_UNGANG_SHARED_PS : 0); +} + +static void sptm_init_darts(struct sptm_allocator *allocator) +{ + int path[8]; + int arm_io = adt_path_offset_trace(adt, "/arm-io", path); + size_t depth = 0; + while (path[depth]) + depth++; + + int node = arm_io; + ADT_FOREACH_CHILD(adt, node) + { + if (!adt_is_compatible(adt, node, "dart,t8110") || !sptm_adt_has(node, "dart-id")) + continue; + + path[depth] = node; + path[depth + 1] = 0; + struct sptm_init_instances trad = {0}, fpag = {0}, fpad = {0}; + sptm_dart_instances(node, path, &trad, &fpag, &fpad); + if (!trad.count || trad.count > SPTM_MAX_DART_INSTANCES) + continue; + + const char *name = adt_get_name(adt, node); + u32 sid_count = sptm_adt_integer(node, "sid-count", 16); + if (sptm.dart_count >= ARRAY_SIZE(sptm.darts) || sid_count > SPTM_MAX_DART_SIDS) { + printf("HV: SPTM cannot track %s\n", name); + continue; + } + + struct sptm_dart *dart = &sptm.darts[sptm.dart_count++]; + struct sptm_dart_dapf *dapf = sptm_alloc(allocator, SPTM_MAX_DART_DAPF * sizeof(*dapf), 16); + struct sptm_dart_tunable *tunables = + sptm_alloc(allocator, SPTM_MAX_DART_TUNABLES * sizeof(*tunables), 16); + u64 *clocks = sptm_alloc(allocator, SPTM_MAX_DART_CLOCKS * sizeof(*clocks), 16); + dart->id = sptm_adt_integer(node, "dart-id", 0); + dart->sid_count = sid_count; + dart->instance_count = trad.count; + memcpy(dart->instances, trad.base, sizeof(dart->instances)); + dart->flags = sptm_dart_flags(node); + dart->dapf_entries = dapf; + dart->dapf_count = sptm_dart_dapf(node, &fpad, dapf); + dart->clock_entries = clocks; + dart->clock_count = sptm_dart_clocks(node, &trad, &fpag, clocks); + dart->tunable_entries = tunables; + dart->tunable_count = sptm_dart_tunables(node, &trad, tunables); + dart->sid_states = sptm_alloc_zero(allocator, sid_count * sizeof(*dart->sid_states), 16); + sptm_dart_sid_states(node, name, dart); + } +} + +static void sptm_init_sart(void) +{ + int path[8]; + int node = adt_path_offset_trace(adt, "/arm-io/sart-ans", path); + u64 base, canary_base = 0; + adt_get_reg(adt, path, "reg", 0, &base, NULL); + adt_get_reg(adt, path, "reg", 1, &canary_base, NULL); + sptm.sart_base = base; + sptm.sart_canary = canary_base + sptm_adt_integer(node, "power-canary-offset", 0); + sptm.sart_exclusive_bounds = sptm_adt_has(node, "exclusive-bounds"); + // Entries already live at handoff belong to firmware. + for (u32 index = 0; index < SPTM_SART_ENTRIES; index++) { + if (read32(base + index * sizeof(u32)) & 0xff) + sptm.sart_protected_mask |= BIT(index); + } +} + +static void sptm_init_nvme(struct sptm_allocator *allocator) +{ + int ans_path[8]; + int ans = adt_path_offset_trace(adt, "/arm-io/ans", ans_path); + int defaults = adt_path_offset(adt, "/defaults"); + u64 main_bar, queue_bar; + adt_get_reg(adt, ans_path, "reg", 3, &main_bar, NULL); + if (sptm_adt_has(ans, "nvme-secure-reg-layout")) { + queue_bar = main_bar + 0x4000; + } else if (sptm_adt_has(defaults, "nvme-iboot-sptm-security") && + sptm_adt_has(ans, "nvme-secure-bar")) { + adt_get_reg(adt, ans_path, "reg", 9, &queue_bar, NULL); + } else { + queue_bar = main_bar; + } + + struct sptm_nvme *nvme = &sptm.nvme; + nvme->queue_entries = sptm_adt_integer(ans, "nvme-queue-entries", 64); + nvme->tl_wa = sptm_adt_has(ans, "nvme-tl-wa"); + nvme->tl_slots = sptm_adt_integer(ans, "nvme-num-sl", 0); + + u64 tcb_bytes = ALIGN_UP(nvme->queue_entries * SPTM_NVME_TCB_SIZE, SPTM_NVME_TCB_ALIGNMENT); + u64 prp_bytes = ALIGN_UP(nvme->queue_entries * SPTM_NVME_PRP_SLOT_SIZE, SPTM_PAGE_SIZE); + nvme->admin_tcbs = (u64)sptm_alloc_zero(allocator, tcb_bytes, SPTM_PAGE_SIZE); + nvme->io_tcbs = (u64)sptm_alloc_zero(allocator, tcb_bytes, SPTM_PAGE_SIZE); + nvme->prp_scratch = (u64)sptm_alloc_zero(allocator, prp_bytes, SPTM_PAGE_SIZE); + nvme->command_state = + (u64)sptm_alloc_zero(allocator, nvme->queue_entries * sizeof(struct sptm_nvme_command), 16); + + nvme->queue_bar = queue_bar; + nvme->nvmmu = main_bar + 0x28000; + nvme->prp_flush_wa = sptm_adt_has(ans, "nvme-prp-flush-wa"); + nvme->vdma_wa = sptm_adt_has(ans, "nvme-vdma-wa"); + if (nvme->tl_wa) { + adt_get_reg(adt, ans_path, "reg", 10, &nvme->tl_mask, NULL); + adt_get_reg(adt, ans_path, "reg", 11, &nvme->tl_control, NULL); + adt_get_reg(adt, ans_path, "reg", 12, &nvme->tl_status, NULL); + } + if (nvme->vdma_wa) + adt_get_reg(adt, ans_path, "reg", 13, &nvme->vdma_status, NULL); + if (sptm_adt_has(ans, "nvme-ans-sha-present")) + adt_get_reg(adt, ans_path, "reg", 14, &nvme->sha_base, NULL); +} + +// The caller has already switched adt to the guest device tree. +void sptm_init_platform(struct sptm_allocator *allocator) +{ + sptm_init_sart(); + sptm_init_nvme(allocator); + sptm_init_darts(allocator); +} diff --git a/src/hv_sptm_internal.h b/src/hv_sptm_internal.h new file mode 100644 index 00000000..b2b917da --- /dev/null +++ b/src/hv_sptm_internal.h @@ -0,0 +1,411 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef HV_SPTM_INTERNAL_H +#define HV_SPTM_INTERNAL_H + +#include "string.h" + +#include "hv.h" +#include "hv_sptm.h" +#include "memory.h" +#include "smp.h" +#include "utils.h" + +#define SPTM_PAGE_SHIFT 14 +#define SPTM_PAGE_SIZE BIT(SPTM_PAGE_SHIFT) +#define SPTM_PAGE_MASK (SPTM_PAGE_SIZE - 1) +#define SPTM_CACHE_LINE_SIZE 64 + +#define SPTM_DESC_PA_MASK_16K 0x0000ffffffffc000ULL + +#define SPTM_STATUS_SUCCESS 0 +#define SPTM_STATUS_MAP_VALID 1 +#define SPTM_STATUS_UPDATE_DELAYED_TLBI 5 +#define SPTM_STATUS_MAP_PADDR_CONFLICT 6 +#define SPTM_STATUS_TABLE_NOT_PRESENT 7 +#define SPTM_STATUS_TABLE_PRESENT 8 + +#define SPTM_MAX_SCRATCH_ENTRIES (SPTM_PAGE_SIZE / sizeof(u64)) +#define SPTM_SERIAL_LINE_SIZE 512 +#define SPTM_MAX_DARTS 64 +#define SPTM_MAX_DART_INSTANCES 4 +#define SPTM_MAX_DART_SIDS 256 +#define SPTM_MAX_DART_DAPF 256 +#define SPTM_MAX_DART_CLOCKS 32 +#define SPTM_MAX_DART_TUNABLES 256 +#define SPTM_MAX_DART_CLOCK_REFS 64 +#define SPTM_MAX_SURT_ROOTS 128 +#define SPTM_MAX_COMMPAGE_RW 2 + +#define SPTM_TXM_RECORD_OFFSET 0x3c00 +#define SPTM_TXM_RECORD_SIZE 0x58 +#define SPTM_TXM_MAX_WORDS 7 + +#define SPTM_SART_ENTRIES 16 +#define SPTM_SART_PAGE_SHIFT 12 + +#define SPTM_NVME_HARDWARE_TAGS 64 + +#define SPTM_NVME_TCB_SIZE 0x80 +#define SPTM_NVME_TCB_ALIGNMENT 0x1000 +#define SPTM_NVME_PRP_PAGE_SHIFT 12 +#define SPTM_NVME_PRP_PAGE_SIZE BIT(SPTM_NVME_PRP_PAGE_SHIFT) +#define SPTM_NVME_PRP_SLOT_SIZE 0x800 +#define SPTM_NVME_MAX_PRPS 0x101 + +#define NVMMU_QUEUE_COUNT 0x100 +#define NVMMU_ADMIN_TCB_BASE 0x108 +#define NVMMU_IO_TCB_BASE 0x110 +#define NVMMU_TCB_INVALIDATE 0x118 +#define NVMMU_TCB_STATUS 0x1120 + +#define NVME_AQA 0x24 +#define NVME_ASQ 0x28 +#define NVME_ACQ 0x30 +#define NVME_IOSQ 0x1200 +#define NVME_IOCQ 0x1208 +#define NVME_IOQA 0x1210 +#define DART_PARAMS_8 0x008 +#define DART_PARAMS_C 0x00c +#define DART_TLB_OP 0x080 +#define DART_ERROR 0x100 +#define DART_REG_PROTECT 0x200 +#define DART_TLIMIT 0x228 +#define DART_TEQRESERVE 0x22c +#define DART_SID_EXCEPTIONS 0x4000 +#define DART_ENABLE_STREAMS 0xc00 +#define DART_DISABLE_STREAMS 0xc20 +#define DART_TCR 0x1000 +#define DART_TTBR 0x1400 +#define DART_UNAVAILABLE 0xabad1dea + +#define DART_TLB_BUSY BIT(31) +#define DART_TLB_FLUSH_SID BIT(8) +#define DART_TLB_FLUSH_ALL 0 +#define DART_TCR_TRANSLATE BIT(0) +#define DART_TTBR_VALID BIT(0) +#define DART_ERROR_SECONDARY BIT(19) + +#define FPAG_CLOCK_PROTECTION BIT(4) + +#define DART_PTE_VALID BIT(0) +#define DART_PTE_UNCACHABLE BIT(1) +#define DART_PTE_WRPROT BIT(2) +#define DART_PTE_RDPROT BIT(3) +#define DART_PTE_OFFSET_MASK (BIT(28) - 1) + +#define UAT_STATE_ROOT0 0x08 +#define UAT_STATE_ROOT1 0x10 +#define UAT_STATE_CONTEXT_ID 0x18 +#define UAT_STATE_GUARD 0x1a +#define UAT_STATE_CURSOR0 0x20 +#define UAT_STATE_CURSOR1 0x28 +#define UAT_STATE_CURSOR2 0x30 +#define UAT_STATE_CURSOR3 0x38 +#define UAT_STATE_OPTIONS 0x40 +#define UAT_STATE_MAP_SEGMENTS 0x48 +#define UAT_STATE_UNMAP_SEGMENTS 0x250 + +#define UAT_STATE_SINGLE_ROOT 1 +#define UAT_STATE_GLOBAL_MODE0 2 +#define UAT_STATE_DUAL_ROOT 4 +#define UAT_STATE_GLOBAL_MODE1 8 + +#define UAT_GUARD_IDLE 2 +#define UAT_GUARD_MAP 3 +#define UAT_GUARD_PREPARE_UNMAP 4 +#define UAT_GUARD_UNMAP 5 + +#define UAT_DESC_TABLE 3 +#define UAT_DESC_VALID BIT(0) +#define UAT_DESC_FIRMWARE_OWNED BIT(3) +#define UAT_DESC_AP_SHIFT 6 +#define UAT_DESC_ATTR_SHIFT 2 +#define UAT_DESC_AF BIT(10) +#define UAT_DESC_NG BIT(11) +#define UAT_DESC_PXN BIT(53) +#define UAT_DESC_UXN BIT(54) +#define UAT_DESC_OS BIT(55) + +#define UAT_HANDOFF_MAGIC 0x4b1d000000000002ULL +#define UAT_HANDOFF_MAGIC_AP 0x000 +#define UAT_HANDOFF_MAGIC_FW 0x008 +#define UAT_HANDOFF_CURRENT_CTX 0x018 +#define UAT_HANDOFF_SLOT_BASE 0x020 +#define UAT_HANDOFF_SLOT_STRIDE 0x018 +#define UAT_HANDOFF_SLOT_STATE 0x000 +#define UAT_HANDOFF_SLOT_ADDR 0x008 +#define UAT_HANDOFF_SLOT_SIZE 0x010 +#define UAT_HANDOFF_SLOT_COUNT 0x041 +#define UAT_HANDOFF_SHARED_SLOT 0x040 +#define UAT_HANDOFF_MIN_SIZE 0x648 +#define UAT_HANDOFF_UNMAP_TAG 0xdead000000000000ULL +#define UAT_HANDOFF_ADDR_MASK 0x0000ffffffffffffULL + +#define SPTM_DART_SID_KNOWN BIT(0) +#define SPTM_DART_SID_ENABLED BIT(1) +#define SPTM_DART_SID_POLICY BIT(2) +#define SPTM_DART_SID_EXCLAVE BIT(3) + +#define SPTM_DART_RETENTION BIT(0) +#define SPTM_DART_IGNORE_SECONDARY BIT(1) +#define SPTM_DART_UNGANG_SHARED_PS BIT(2) + +#define SPTM_FRAME_KERNEL_CODE 6 +#define SPTM_FRAME_KERNEL_ROOT 8 +#define SPTM_FRAME_PAGE_TABLE 9 +#define SPTM_FRAME_USER_ROOT 18 +#define SPTM_FRAME_SHARED_ROOT 19 +#define SPTM_FRAME_XNU_ROOT 20 +#define SPTM_FRAME_SHARED_PT 21 +#define SPTM_FRAME_ROZONE_PT 22 +#define SPTM_FRAME_COMMPAGE_PT 23 +#define SPTM_FRAME_STAGE2_ROOT 33 +#define SPTM_FRAME_STAGE2_PT 34 +#define SPTM_FRAME_SUBPAGE 40 +#define SPTM_FRAME_DEFAULT 11 +#define SPTM_FRAME_RO 12 +#define SPTM_FRAME_XNU_IOMMU 24 +#define SPTM_FRAME_XNU_IO 26 +#define SPTM_FRAME_PROTECTED_IO 27 +#define SPTM_FRAME_COPROCESSOR_RO_IO 28 +#define SPTM_FRAME_XNU_COMMPAGE_RW 29 +#define SPTM_FRAME_RESTRICTED_IO 38 +#define SPTM_FRAME_RESTRICTED_IO_TELEMETRY 39 +#define SPTM_FRAME_SK_IO 65 + +#define SPTM_COMMPAGE_USER_TIMEBASE_OFFSET 0x90 +#define SPTM_COMMPAGE_CONT_HWCLOCK_OFFSET 0x91 +#define SPTM_COMMPAGE_HW_TPRO_OFFSET 0x10c +#define SPTM_COMMPAGE_CPU_CAPS64_OFFSET 0x10 +#define SPTM_COMMPAGE_CPU_CAPS32_OFFSET 0x20 +/* Either bit selects AMX in the guest vDSP capability dispatcher. */ +#define SPTM_CPU_CAP_AMX_MASK (3ULL << 28) + +struct sptm_allocator { + u64 next; + u64 end; +}; + +struct sptm_cpu { + u32 phys_id; + u8 logical_id; + bool valid; +}; + +struct sptm_geometry { + u64 descriptor_mask; + u16 entries_mask; + u8 page_shift; + u8 start_level; + u8 page_ratio; +}; + +struct sptm_dart { + u32 id; + u16 sid_count; + u16 sid_words; + u8 instance_count; + struct sptm_dart_sid *sid_states; + const struct sptm_dart_dapf *dapf_entries; + const u64 *clock_entries; + const struct sptm_dart_tunable *tunable_entries; + u16 dapf_count; + u8 clock_count; + u16 tunable_count; + u32 flags; + u64 instances[SPTM_MAX_DART_INSTANCES]; + u32 boot_streams[(SPTM_MAX_DART_SIDS + 31) / 32]; + u32 saved_streams[SPTM_MAX_DART_INSTANCES][(SPTM_MAX_DART_SIDS + 31) / 32]; + u32 saved_tlimit[SPTM_MAX_DART_INSTANCES]; + u32 saved_teqreserve[SPTM_MAX_DART_INSTANCES]; + bool sid_exceptions; + u16 interrupt_status_offset; + bool saved_valid; + bool limits_valid; + bool powered; + bool clock_active; +}; + +struct sptm_dart_sid { + u64 root; + u64 pt_start; + u64 pt_end; + u32 tcr; + u8 root_level; + u8 flags; +}; + +struct sptm_dart_dapf { + u64 base; + u64 start; + u64 end; + u32 r4; + u32 control; + u32 r20; +}; + +struct sptm_dart_tunable { + u64 base; + u32 offset; + u32 size; + u64 mask; + u64 value; +}; + +struct sptm_dart_clock_ref { + u64 address; + u16 refs; + bool original; + bool valid; +}; + +struct sptm_uat_segment { + u64 first; + u64 count; +}; + +static_assert(sizeof(struct sptm_uat_segment) == 16, "unexpected UAT segment size"); + +#define SPTM_UAT_BATCH_PAGES 256 + +struct sptm_surt_root { + u64 root; + u16 asid; + u8 attr_index; + bool valid; +}; + +struct sptm_nvme_command { + u64 prp_aux; + u64 drained_slots; + u16 count; + u8 dma_flags; +}; + +static_assert((SPTM_NVME_MAX_PRPS - 1) * sizeof(u64) <= SPTM_NVME_PRP_SLOT_SIZE, + "NVMe PRP list exceeds its per-CID slot"); + +struct sptm_nvme { + bool prp_flush_wa; + bool tl_wa; + bool vdma_wa; + u8 tl_slots; + u32 queue_entries; + u64 queue_bar; + u64 nvmmu; + u64 admin_tcbs; + u64 io_tcbs; + u64 prp_scratch; + u64 command_state; + u64 tl_mask; + u64 tl_control; + u64 tl_status; + u64 vdma_status; + u64 sha_base; +}; + +struct sptm_state { + bool enabled; + size_t host_rt_restore_count; + u64 host_rt_restore_pages[SPTM_MAX_SCRATCH_ENTRIES]; + u64 managed_start; + u64 managed_end; + u64 physmap_base; + u64 physmap_end; + u64 scratch_pa; + u64 kernel_root; + u64 panic_state_pa; + u64 frame_table_pa; + u64 uat_global_state; + u64 uat_gpu_contexts; + u64 uat_shared_l2_pa; + u64 uat_shared_l2_va; + u64 uat_handoff_pa; + u64 uat_handoff_owner[UAT_HANDOFF_SLOT_COUNT]; + u32 uat_segment_limit; + u32 uat_mapping_limit; + u32 uat_state_size; + u8 uat_va_width; + u8 uat_mode; + bool uat_tlbi_at_retype; + u64 txm_info_va; + u64 sart_base; + u64 sart_canary; + u16 sart_protected_mask; + u16 sart_guarded_mask; + bool sart_exclusive_bounds; + spinlock_t service_lock; + struct sptm_cpu cpus[MAX_CPUS]; + u32 hv_phys_ids[MAX_CPUS]; + u8 cpu_count; + u8 max_cpus; + u64 commpage_rw[SPTM_MAX_COMMPAGE_RW]; + u8 commpage_rw_count; + bool commpage_policy_published; + struct sptm_dart darts[SPTM_MAX_DARTS]; + size_t dart_count; + struct sptm_dart_clock_ref dart_clock_refs[SPTM_MAX_DART_CLOCK_REFS]; + struct sptm_surt_root surt_roots[SPTM_MAX_SURT_ROOTS]; + struct sptm_nvme nvme; +}; + +extern struct sptm_state sptm; + +/* Boot */ +void *sptm_alloc(struct sptm_allocator *allocator, size_t size, size_t alignment); +void *sptm_alloc_zero(struct sptm_allocator *allocator, size_t size, size_t alignment); +u64 sptm_adt_integer(int node, const char *name, u64 fallback); +void sptm_init_platform(struct sptm_allocator *allocator); + +/* Core */ +bool sptm_is_managed_range(u64 pa, size_t size); +u64 sptm_pointer_pa(u64 pointer); +u64 sptm_physmap_va(u64 pa); +u64 sptm_iommu_table_va(u64 pa); +struct sptm_cpu *sptm_find_cpu(u32 phys_id); +struct sptm_cpu *sptm_register_cpu(u32 phys_id); +u64 sptm_scratch_pa(struct exc_info *ctx); + +/* DART */ +bool sptm_dart_flush_all(struct sptm_dart *dart); +bool sptm_dart_power(struct exc_info *ctx, struct sptm_dart *dart, bool power_up); +bool sptm_dart_init(struct exc_info *ctx, struct sptm_dart *dart); +bool sptm_handle_dart(struct exc_info *ctx, u32 endpoint); + +/* Frame ownership */ +bool sptm_is_table_type(u8 type); +bool sptm_is_root_type(u8 type); +u8 *sptm_frame_entry(u64 pa); +void sptm_adjust_table_valid(u64 slot, int delta); + +/* UAT */ +void sptm_uat_configure(u64 global_root_pa, u64 handoff_pa); +void sptm_uat_tlbi_all(void); +bool sptm_handle_uat(struct exc_info *ctx, u32 endpoint); + +/* Stage-1 pmap */ +struct sptm_surt_root *sptm_find_surt_root(u64 root); +void sptm_publish_stage1(void); +void sptm_publish_commpage_policy(void); +bool sptm_retype_frame(struct exc_info *ctx); +bool sptm_configure_shared_region(struct exc_info *ctx); +bool sptm_map_page(struct exc_info *ctx); +bool sptm_map_table(struct exc_info *ctx); +bool sptm_unmap_table(struct exc_info *ctx); +bool sptm_update_region(struct exc_info *ctx); +bool sptm_update_disjoint(struct exc_info *ctx); +bool sptm_update_disjoint_multipage(struct exc_info *ctx); +bool sptm_unmap_region(struct exc_info *ctx); +bool sptm_unmap_disjoint(struct exc_info *ctx); +bool sptm_nest_region(struct exc_info *ctx, bool nest); +bool sptm_surt_update(struct exc_info *ctx, bool allocate); +bool sptm_condemn_leaf_table(struct exc_info *ctx, bool condemn); + +/* Service dispatch */ +bool sptm_handle_xnu_bootstrap(struct exc_info *ctx, u32 endpoint); +bool sptm_handle_txm(struct exc_info *ctx, u32 endpoint); +bool sptm_handle_sart(struct exc_info *ctx, u32 endpoint); +bool sptm_handle_nvme(struct exc_info *ctx, u32 endpoint); + +#endif diff --git a/src/hv_sptm_nvme.c b/src/hv_sptm_nvme.c new file mode 100644 index 00000000..b374c050 --- /dev/null +++ b/src/hv_sptm_nvme.c @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include "hv_sptm_internal.h" + +static struct sptm_nvme_command *sptm_nvme_command(u32 cid) +{ + return &((struct sptm_nvme_command *)sptm.nvme.command_state)[cid]; +} + +static u64 sptm_nvme_tcb(u32 qid, u32 cid) +{ + return (qid ? sptm.nvme.io_tcbs : sptm.nvme.admin_tcbs) + cid * SPTM_NVME_TCB_SIZE; +} + +static bool sptm_nvme_map_pages(struct exc_info *ctx) +{ + u32 qid = ctx->regs[0]; + u32 cid = ctx->regs[1]; + u64 segment_pa = sptm_pointer_pa(ctx->regs[3]); + size_t count = ctx->regs[4]; + struct sptm_nvme_command *command = sptm_nvme_command(cid); + u8 template[SPTM_NVME_TCB_SIZE]; + + memcpy(template, (void *)sptm_pointer_pa(ctx->regs[2]), sizeof(template)); + u8 dma_flags = template[0x01]; + template[0x02] = cid; + + // PRP1 is the first target; PRP2 is the second target or a list of the rest. + u64 first = count ? read64(segment_pa) : 0; + u64 prp_aux = 0; + if (count == 2) { + prp_aux = read64(segment_pa + sizeof(u64)); + } else if (count > 2) { + prp_aux = sptm.nvme.prp_scratch + cid * SPTM_NVME_PRP_SLOT_SIZE; + memcpy((void *)prp_aux, (void *)(segment_pa + sizeof(u64)), (count - 1) * sizeof(u64)); + if (sptm.nvme.prp_flush_wa && count > 0x10) { + size_t flush_size = + count < 0x100 ? (count * sizeof(u64) + 0x7f) & 0x1f80 : SPTM_NVME_PRP_PAGE_SIZE; + sysop("dsb sy"); + dc_civac_range((void *)prp_aux, flush_size); + } + } + write64((u64)template + 0x18, first); + write64((u64)template + 0x20, prp_aux); + + *command = (struct sptm_nvme_command){.prp_aux = prp_aux, .count = count, .dma_flags = dma_flags}; + + memcpy((void *)sptm_nvme_tcb(qid, cid), template, sizeof(template)); + sysop("dsb sy"); + return true; +} + +static bool sptm_nvme_tl_idle(struct sptm_nvme_command *command, bool retry) +{ + u32 directions = command->dma_flags & 3; + if (!sptm.nvme.tl_wa || !directions) + return true; + + if (!retry) { + u32 mask = ((directions & 1) ? 0x800000 : 0) | ((directions & 2) ? 0x80 : 0); + u32 control = ((directions & 1) ? 0x400000 : 0) | ((directions & 2) ? 0x40 : 0); + write32(sptm.nvme.tl_mask + 0x4, mask); + write32(sptm.nvme.tl_mask + 0xc, mask); + write32(sptm.nvme.tl_control + 0x4, control); + command->drained_slots = 0; + sysop("dsb sy"); + } + + u64 all_slots = BIT(sptm.nvme.tl_slots) - 1; + for (size_t poll = 0; poll < 250; poll++) { + for (u32 slot = 0; slot < sptm.nvme.tl_slots; slot++) { + if (command->drained_slots & BIT(slot)) + continue; + u32 status = read32(sptm.nvme.tl_status + slot * 0x10000); + bool writes_done = !(directions & 1) || (status & 0xff) == 0xff; + bool reads_done = !(directions & 2) || (status & 0x7f0000) == 0x7f0000; + if (writes_done && reads_done) + command->drained_slots |= BIT(slot); + } + if (command->drained_slots == all_slots) { + write32(sptm.nvme.tl_mask + 0x4, 0x800080); + write32(sptm.nvme.tl_mask + 0xc, 0x800080); + write32(sptm.nvme.tl_control + 0x4, 0x400040); + command->drained_slots = 0; + sysop("dsb sy"); + return true; + } + udelay(1); + } + + return false; +} + +static bool sptm_nvme_unmap_pages(struct exc_info *ctx) +{ + u32 qid = ctx->regs[0]; + u32 cid = ctx->regs[1]; + // A retry follows an earlier call that returned 0 while the TL drained. + bool retry = ctx->regs[2]; + struct sptm_nvme_command *command = sptm_nvme_command(cid); + + if (!retry) { + if (sptm.nvme.vdma_wa && (read32(sptm.nvme.vdma_status + 0x20000 + cid * 0x20) & 0x300)) + return false; + memset((void *)sptm_nvme_tcb(qid, cid), 0, SPTM_NVME_TCB_SIZE); + sysop("dsb sy"); + write32(sptm.nvme.nvmmu + NVMMU_TCB_INVALIDATE, cid); + sysop("dsb sy"); + u32 packed_status = read32(sptm.nvme.nvmmu + NVMMU_TCB_STATUS + (cid / 4) * sizeof(u32)); + if ((packed_status >> ((cid % 4) * 5)) & 0xf) + return false; + } + + if (!sptm_nvme_tl_idle(command, retry)) { + ctx->regs[0] = 0; + return true; + } + + if (command->count > 2) { + memset((void *)command->prp_aux, 0, SPTM_NVME_PRP_SLOT_SIZE); + // Seems weird but the documentation says so... + if (sptm.nvme.prp_flush_wa && command->count > 0xff) { + sysop("dsb sy"); + dc_civac_range((void *)(command->prp_aux + SPTM_NVME_PRP_SLOT_SIZE), + SPTM_NVME_PRP_SLOT_SIZE); + } + } + sysop("dsb sy"); + memset(command, 0, sizeof(*command)); + ctx->regs[0] = 1; + return true; +} + +static void sptm_nvme_mmio_write32(u64 address, u32 value) +{ + write32(address, value); + sysop("dsb sy"); +} + +static void sptm_nvme_mmio_write64_lo_hi(u64 address, u64 value) +{ + sptm_nvme_mmio_write32(address, value); + sptm_nvme_mmio_write32(address + sizeof(u32), value >> 32); +} + +bool sptm_handle_nvme(struct exc_info *ctx, u32 endpoint) +{ + struct sptm_nvme *nvme = &sptm.nvme; + u64 *args = ctx->regs; + bool handled = true; + spin_lock(&sptm.service_lock); + switch (endpoint) { + case 0: + sysop("dsb sy"); + sptm_nvme_mmio_write64_lo_hi(nvme->nvmmu + NVMMU_ADMIN_TCB_BASE, nvme->admin_tcbs); + sptm_nvme_mmio_write64_lo_hi(nvme->nvmmu + NVMMU_IO_TCB_BASE, nvme->io_tcbs); + sptm_nvme_mmio_write32(nvme->nvmmu + NVMMU_QUEUE_COUNT, SPTM_NVME_HARDWARE_TAGS - 1); + break; + case 1: + handled = sptm_nvme_map_pages(ctx); + break; + case 2: + // UNMAP_PAGES reports its own completion state in x0. + handled = sptm_nvme_unmap_pages(ctx); + spin_unlock(&sptm.service_lock); + return handled; + case 4: + // Admin queues: submission base and size, then completion base and size. + sptm_nvme_mmio_write32(nvme->queue_bar + NVME_AQA, args[1] | (args[3] << 16)); + sptm_nvme_mmio_write64_lo_hi(nvme->queue_bar + NVME_ASQ, args[0]); + sptm_nvme_mmio_write64_lo_hi(nvme->queue_bar + NVME_ACQ, args[2]); + break; + case 5: + sptm_nvme_mmio_write32(nvme->queue_bar + NVME_IOQA, args[0] | (args[1] << 16)); + break; + case 6: + sptm_nvme_mmio_write64_lo_hi(nvme->queue_bar + NVME_IOSQ, args[0]); + break; + case 7: + sptm_nvme_mmio_write64_lo_hi(nvme->queue_bar + NVME_IOCQ, args[0]); + break; + case 8: + write32(nvme->sha_base, args[0] >> SPTM_PAGE_SHIFT); + write32(nvme->sha_base + 4, args[2]); + sysop("dsb sy"); + break; + } + spin_unlock(&sptm.service_lock); + if (handled) + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return handled; +} diff --git a/src/hv_sptm_pmap.c b/src/hv_sptm_pmap.c new file mode 100644 index 00000000..3348a7de --- /dev/null +++ b/src/hv_sptm_pmap.c @@ -0,0 +1,608 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include "hv_sptm_internal.h" + +struct sptm_surt_root *sptm_find_surt_root(u64 root) +{ + for (size_t index = 0; index < ARRAY_SIZE(sptm.surt_roots); index++) { + struct sptm_surt_root *candidate = &sptm.surt_roots[index]; + if (candidate->valid && candidate->root == root) + return candidate; + } + + return NULL; +} + +static void sptm_set_geometry(u8 attr_index, struct sptm_geometry *geometry) +{ + if (attr_index == 1 || attr_index == 5) { + geometry->descriptor_mask = 0x0000fffffffff000ULL; + geometry->entries_mask = 0x1ff; + geometry->page_shift = 12; + geometry->start_level = 0; + geometry->page_ratio = SPTM_PAGE_SIZE >> 12; + } else { + geometry->descriptor_mask = SPTM_DESC_PA_MASK_16K; + geometry->entries_mask = 0x7ff; + geometry->page_shift = 14; + geometry->start_level = 1; + geometry->page_ratio = 1; + } +} + +static bool sptm_root_geometry(u64 root, struct sptm_geometry *geometry) +{ + struct sptm_surt_root *surt = sptm_find_surt_root(root); + if (surt) { + sptm_set_geometry(surt->attr_index, geometry); + return true; + } + + u8 *entry = sptm_frame_entry(root); + if (!entry) + return false; + + sptm_set_geometry(entry[12], geometry); + return true; +} + +static u64 sptm_walk(u64 root, u64 va, unsigned int target_level, + struct sptm_geometry *geometry_out) +{ + static const unsigned int shifts_16k[] = {0, 36, 25, 14}; + static const unsigned int shifts_4k[] = {39, 30, 21, 12}; + struct sptm_geometry geometry; + u64 table = root; + + if (!sptm_root_geometry(root, &geometry) || target_level < geometry.start_level) + return 0; + + const unsigned int *shifts = geometry.page_shift == 12 ? shifts_4k : shifts_16k; + + for (unsigned int level = geometry.start_level; level < target_level; level++) { + u64 descriptor = read64(table + ((va >> shifts[level]) & geometry.entries_mask) * 8); + // Do not descend into a condemned table. + if (!(descriptor & 1) || (descriptor & BIT(2))) + return 0; + table = descriptor & geometry.descriptor_mask; + } + + u64 slot = table + ((va >> shifts[target_level]) & geometry.entries_mask) * 8; + if (geometry_out) + *geometry_out = geometry; + return slot; +} + +static void sptm_cancel_host_rt_restore(u64 pa) +{ + for (size_t index = 0; index < sptm.host_rt_restore_count; index++) { + if (sptm.host_rt_restore_pages[index] != pa) + continue; + + sptm.host_rt_restore_count--; + sptm.host_rt_restore_pages[index] = + sptm.host_rt_restore_pages[sptm.host_rt_restore_count]; + return; + } +} + +static void sptm_queue_host_rt_restore(u64 pa) +{ + for (size_t index = 0; index < sptm.host_rt_restore_count; index++) { + if (sptm.host_rt_restore_pages[index] == pa) + return; + } + + // Queue overflow leaves the page safely protected. + if (sptm.host_rt_restore_count < ARRAY_SIZE(sptm.host_rt_restore_pages)) + sptm.host_rt_restore_pages[sptm.host_rt_restore_count++] = pa; +} + +static void sptm_flush_stage1_tlb(void) +{ + // Publish page-table writes before invalidating guest translations. + sysop("dsb sy"); + sysop("tlbi vmalle1is"); + sysop("dsb ish"); + sysop("isb"); +} + +void sptm_publish_stage1(void) +{ + sptm_flush_stage1_tlb(); + + // Restore WB aliases only after stale RT translations are gone. + if (sptm.host_rt_restore_count) { + mmu_restore_ram_pages_wb(sptm.host_rt_restore_pages, + sptm.host_rt_restore_count, false); + sptm.host_rt_restore_count = 0; + } +} + +// Publish the updated mappings unless the caller deferred TLB invalidation. +static bool sptm_finish_update(struct exc_info *ctx, bool changed, bool deferred) +{ + if (changed && !deferred) + sptm_publish_stage1(); + else + sysop("dsb sy"); + ctx->regs[0] = deferred ? SPTM_STATUS_UPDATE_DELAYED_TLBI : SPTM_STATUS_SUCCESS; + return true; +} + +void sptm_publish_commpage_policy(void) +{ + if (sptm.commpage_policy_published) + return; + + for (size_t index = 0; index < sptm.commpage_rw_count; index++) { + u64 pa = sptm.commpage_rw[index]; + u64 caps = read64(pa + SPTM_COMMPAGE_CPU_CAPS64_OFFSET); + u64 replacement = caps; + /* Other commpage frames do not contain the CPU capability fields. */ + if (!memcmp((void *)pa, "commpage 64-bit", sizeof("commpage 64-bit") - 1)) { + replacement &= ~SPTM_CPU_CAP_AMX_MASK; + write64(pa + SPTM_COMMPAGE_CPU_CAPS64_OFFSET, replacement); + clear32(pa + SPTM_COMMPAGE_CPU_CAPS32_OFFSET, SPTM_CPU_CAP_AMX_MASK); + } + write8(pa + SPTM_COMMPAGE_USER_TIMEBASE_OFFSET, 0); + write8(pa + SPTM_COMMPAGE_CONT_HWCLOCK_OFFSET, 0); + write8(pa + SPTM_COMMPAGE_HW_TPRO_OFFSET, 0); + printf("HV: SPTM commpage fallback policy at PA 0x%lx " + "(caps 0x%lx -> 0x%lx)\n", + pa, caps, replacement); + } + sysop("dsb sy"); + sptm.commpage_policy_published = true; +} + +bool sptm_retype_frame(struct exc_info *ctx) +{ + u64 pa = ctx->regs[0] & ~SPTM_PAGE_MASK; + u8 new_type = ctx->regs[2]; + + u8 *entry = sptm_frame_entry(pa); + if (!entry) + return false; + + u8 old_type = entry[2]; + bool entering_iommu = old_type != SPTM_FRAME_XNU_IOMMU && + new_type == SPTM_FRAME_XNU_IOMMU; + bool leaving_iommu = old_type == SPTM_FRAME_XNU_IOMMU && + new_type != SPTM_FRAME_XNU_IOMMU; + bool iommu_transition = entering_iommu || leaving_iommu; + u64 physmap_slot = 0; + u64 physmap_replacement = 0; + + if (iommu_transition) { + // Keep the physmap leaf invalid during the host memory-type transition. + u64 type_mask = (7ULL << 2) | (3ULL << 8); + u64 type = entering_iommu ? PTE_MAIR_IDX(MAIR_IDX_NORMAL_NC) | PTE_SH_NS + : PTE_MAIR_IDX(MAIR_IDX_NORMAL) | PTE_SH_OS; + physmap_slot = sptm_walk(sptm.kernel_root, sptm_physmap_va(pa), 3, NULL); + physmap_replacement = (read64(physmap_slot) & ~type_mask) | type; + write64(physmap_slot, 0); + sptm_flush_stage1_tlb(); + } + + // Remove WB aliases to avoid speculative accesses to them + if (entering_iommu) { + u64 host_page = pa; + mmu_map_ram_pages_nc(&host_page, 1, true); + } + + // A table's valid-entry count starts and ends at zero. + if (sptm_is_table_type(old_type) || sptm_is_table_type(new_type)) + write16((u64)entry + 8, 0); + entry[2] = new_type; + + if (new_type == SPTM_FRAME_XNU_IOMMU) { + // Zero IOMMU tables through their NC alias. + memset((void *)sptm_iommu_table_va(pa), 0, SPTM_PAGE_SIZE); + sysop("dsb sy"); + } else if (sptm_is_table_type(new_type)) { + memset((void *)pa, 0, SPTM_PAGE_SIZE); + } + if (sptm_is_root_type(new_type)) { + write16((u64)entry + 10, (ctx->regs[3] >> 32) & 0xffff); + entry[12] = ctx->regs[3] & 0xff; + } + + // Defer commpage policy changes until the shared region is configured. + if (new_type == SPTM_FRAME_XNU_COMMPAGE_RW && + sptm.commpage_rw_count < ARRAY_SIZE(sptm.commpage_rw)) { + sptm.commpage_rw[sptm.commpage_rw_count++] = pa; + } + + if (sptm_is_table_type(old_type) || sptm_is_table_type(new_type) || iommu_transition || + new_type == SPTM_FRAME_XNU_IOMMU || new_type == SPTM_FRAME_XNU_IO || + new_type == SPTM_FRAME_PROTECTED_IO || new_type == SPTM_FRAME_COPROCESSOR_RO_IO || + new_type == SPTM_FRAME_RESTRICTED_IO || new_type == SPTM_FRAME_RESTRICTED_IO_TELEMETRY || + new_type == SPTM_FRAME_SK_IO) + sptm_publish_stage1(); + else + sysop("dsb ishst"); + if (iommu_transition && sptm.uat_tlbi_at_retype) + sptm_uat_tlbi_all(); + + // Restore WB aliases only after guest and IOMMU invalidation. + if (leaving_iommu) { + u64 host_page = pa; + mmu_restore_ram_pages_wb(&host_page, 1, true); + } + if (iommu_transition) { + write64(physmap_slot, physmap_replacement); + sptm_publish_stage1(); + } + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +bool sptm_configure_shared_region(struct exc_info *ctx) +{ + u64 pa = ctx->regs[0] & ~SPTM_PAGE_MASK; + u8 *entry = sptm_frame_entry(pa); + if (!entry) + return false; + + entry[2] = SPTM_FRAME_SHARED_ROOT; + write16((u64)entry + 10, 0); + entry[12] = 0; + sysop("dsb ishst"); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +bool sptm_map_page(struct exc_info *ctx) +{ + u64 root = ctx->regs[0]; + u64 va = ctx->regs[1]; + u64 pte = ctx->regs[2]; + struct sptm_geometry geometry; + u64 slot = sptm_walk(root, va, 3, &geometry); + if (!slot) { + ctx->regs[0] = SPTM_STATUS_TABLE_NOT_PRESENT; + return true; + } + + u64 old = read64(slot); + u64 scratch = sptm_scratch_pa(ctx); + write64(scratch, old); + write64(scratch + sizeof(u64), sptm_physmap_va(slot)); + + if ((old & 1) && ((old & geometry.descriptor_mask) != (pte & geometry.descriptor_mask))) { + sysop("dsb sy"); + ctx->regs[0] = SPTM_STATUS_MAP_PADDR_CONFLICT; + return true; + } + + if (!(old & 1) && (pte & 1)) + sptm_adjust_table_valid(slot, 1); + + ctx->regs[0] = old & 1 ? SPTM_STATUS_MAP_VALID : SPTM_STATUS_SUCCESS; + write64(slot, pte); + + if ((old & 1) && old != pte) + sptm_publish_stage1(); + else + sysop("dsb sy"); + + return true; +} + +bool sptm_map_table(struct exc_info *ctx) +{ + u64 root = ctx->regs[0]; + u64 va = ctx->regs[1]; + unsigned int target_level = ctx->regs[2]; + u64 tte = ctx->regs[3]; + + struct sptm_geometry geometry; + u64 slot = sptm_walk(root, va, target_level, &geometry); + if (!slot) { + ctx->regs[0] = SPTM_STATUS_TABLE_NOT_PRESENT; + return true; + } + + slot &= ~(geometry.page_ratio * sizeof(u64) - 1); + for (size_t index = 0; index < geometry.page_ratio; index++) { + if (read64(slot + index * sizeof(u64)) & 1) { + ctx->regs[0] = SPTM_STATUS_TABLE_PRESENT; + return true; + } + } + + u64 child = tte & geometry.descriptor_mask; + u64 attrs = tte & ~geometry.descriptor_mask; + for (size_t index = 0; index < geometry.page_ratio; index++) { + u64 entry = slot + index * sizeof(u64); + u64 replacement = + ((child + index * BIT(geometry.page_shift)) & geometry.descriptor_mask) | attrs; + write64(entry, replacement); + } + sysop("dsb sy"); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +bool sptm_unmap_table(struct exc_info *ctx) +{ + struct sptm_geometry geometry; + u64 slot = sptm_walk(ctx->regs[0], ctx->regs[1], ctx->regs[2], &geometry); + slot &= ~(geometry.page_ratio * sizeof(u64) - 1); + for (size_t index = 0; index < geometry.page_ratio; index++) + write64(slot + index * sizeof(u64), 0); + sptm_publish_stage1(); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +// Merge a template into a leaf, recording the displaced PTE in scratch. +static void sptm_update_leaf(u64 slot, u64 template, u64 mask, u64 scratch) +{ + u64 old = read64(slot); + write64(scratch, old); + write64(slot, (old & ~mask) | (template & mask)); +} + +// Clear a leaf if it is mapped, recording the displaced PTE in scratch. +static void sptm_unmap_leaf(u64 slot, u64 scratch, bool *changed) +{ + u64 old = slot ? read64(slot) : 0; + write64(scratch, old); + if (!(old & 1)) + return; + sptm_adjust_table_valid(slot, -1); + write64(slot, 0); + *changed = true; +} + +static u64 sptm_update_mask(u64 flags) +{ + u64 mask = 0; + + if (flags & BIT(0)) + mask |= BIT(58); + if (flags & BIT(1)) + mask |= (3ULL << 6) | BIT(53) | BIT(54) | BIT(59); + if (flags & BIT(2)) + mask |= BIT(11); + if (flags & BIT(3)) + mask |= BIT(10); + if (flags & BIT(4)) + mask |= 3ULL << 8; + if (flags & BIT(5)) + mask |= 7ULL << 2; + + return mask; +} + +bool sptm_update_region(struct exc_info *ctx) +{ + u64 root = ctx->regs[0]; + u64 start = ctx->regs[1]; + size_t count = ctx->regs[2]; + u64 flags = ctx->regs[4]; + + struct sptm_geometry geometry; + if (!sptm_root_geometry(root, &geometry)) + return false; + + u64 page_size = BIT(geometry.page_shift); + u64 templates = sptm_pointer_pa(ctx->regs[3]); + u64 scratch = sptm_scratch_pa(ctx); + u64 mask = sptm_update_mask(flags); + for (size_t index = 0; index < count; index++) + sptm_update_leaf(sptm_walk(root, start + index * page_size, 3, NULL), + read64(templates + index * sizeof(u64)), mask, + scratch + index * sizeof(u64)); + + // Do not invalidate a partial deferred alias batch. + return sptm_finish_update(ctx, count, flags & BIT(8)); +} + +bool sptm_update_disjoint(struct exc_info *ctx) +{ + size_t count = ctx->regs[2]; + u64 flags = ctx->regs[3]; + + u64 ops = sptm_pointer_pa(ctx->regs[1]); + u64 scratch = sptm_scratch_pa(ctx); + u64 mask = sptm_update_mask(flags); + for (size_t index = 0; index < count; index++) { + u64 op = ops + index * 24; + sptm_update_leaf(sptm_walk(read64(op), read64(op + 8), 3, NULL), read64(op + 16), mask, + scratch + index * sizeof(u64)); + } + + return sptm_finish_update(ctx, count, flags & BIT(8)); +} + +bool sptm_update_disjoint_multipage(struct exc_info *ctx) +{ + size_t slots = ctx->regs[1]; + u64 stream = sptm_pointer_pa(ctx->regs[0]); + u64 scratch = sptm_scratch_pa(ctx); + size_t used; + + if (!cpu_features->apple_sysregs_unlocked) { + // Mirror RT/WCOMB memory types in m1n1, restoring WB only after TLBI. The + // last transition of each page in the stream wins. + u64 *protect = (u64 *)scratch; + size_t protect_count = 0; + for (used = 0; used < slots; used += 1 + read32(stream + used * 24 + 16)) { + u64 header = stream + used * 24; + u64 paddr = read64(header); + if ((read32(header + 20) & BIT(9)) || !sptm_is_managed_range(paddr, 1)) + continue; + + for (size_t index = 0; index < protect_count; index++) { + if (protect[index] == paddr) { + protect[index] = protect[--protect_count]; + break; + } + } + if ((read64(header + 8) & (7ULL << 2)) == PTE_MAIR_IDX(1)) { + sptm_cancel_host_rt_restore(paddr); + protect[protect_count++] = paddr; + } else { + sptm_queue_host_rt_restore(paddr); + } + } + mmu_map_ram_pages_nc(protect, protect_count, false); + } + + bool changed = false; + bool deferred = false; + size_t scratch_index = 0; + used = 0; + + while (used < slots) { + u64 header = stream + used * 24; + u64 paddr = read64(header); + u64 papt_template = read64(header + 8); + size_t inner_count = read32(header + 16); + u64 options = read32(header + 20); + u64 mask = sptm_update_mask(options); + used++; + + deferred |= options & BIT(8); + + // Keep the physmap memory type in sync unless SKIP_PAPT is set. + if (!(options & BIT(9)) && sptm_is_managed_range(paddr, 1)) { + u64 physmap_slot = sptm_walk(sptm.kernel_root, sptm_physmap_va(paddr), 3, NULL); + u64 physmap_mask = (3ULL << 8) | (7ULL << 2); + write64(physmap_slot, + (read64(physmap_slot) & ~physmap_mask) | (papt_template & physmap_mask)); + changed = true; + } + + for (size_t index = 0; index < inner_count; index++, scratch_index++) { + u64 op = stream + (used + index) * 24; + sptm_update_leaf(sptm_walk(read64(op), read64(op + 8), 3, NULL), read64(op + 16), mask, + scratch + scratch_index * sizeof(u64)); + changed = true; + } + used += inner_count; + } + + return sptm_finish_update(ctx, changed, deferred); +} + +bool sptm_unmap_region(struct exc_info *ctx) +{ + u64 root = ctx->regs[0]; + u64 start = ctx->regs[1]; + size_t count = ctx->regs[2]; + u64 options = ctx->regs[3]; + + struct sptm_geometry geometry; + if (!sptm_root_geometry(root, &geometry)) + return false; + + u64 page_size = BIT(geometry.page_shift); + u64 scratch = sptm_scratch_pa(ctx); + bool changed = false; + for (size_t index = 0; index < count; index++) + sptm_unmap_leaf(sptm_walk(root, start + index * page_size, 3, NULL), + scratch + index * sizeof(u64), &changed); + + return sptm_finish_update(ctx, changed, changed && (options & BIT(8))); +} + +bool sptm_unmap_disjoint(struct exc_info *ctx) +{ + size_t count = ctx->regs[2]; + u64 ops = sptm_pointer_pa(ctx->regs[1]); + u64 scratch = sptm_scratch_pa(ctx); + bool changed = false; + for (size_t index = 0; index < count; index++) { + u64 op = ops + index * 24; + sptm_unmap_leaf(sptm_walk(read64(op), read64(op + 8), 3, NULL), + scratch + index * sizeof(u64), &changed); + } + + return sptm_finish_update(ctx, changed, false); +} + +bool sptm_nest_region(struct exc_info *ctx, bool nest) +{ + u64 user_root = ctx->regs[0]; + u64 shared_root = ctx->regs[1]; + u64 start = ctx->regs[2]; + u64 count = ctx->regs[3]; + struct sptm_geometry geometry; + if (!sptm_root_geometry(user_root, &geometry)) + return false; + + // Nesting copies whole L2 entries from the shared root into the user root. + u64 bytes = count << geometry.page_shift; + u8 l2_shift = geometry.page_shift == 12 ? 21 : 25; + u64 first_block = start >> l2_shift; + u64 iterations = bytes ? ((start + bytes - 1) >> l2_shift) - first_block + 1 : 0; + bool changed = false; + for (u64 index = 0; index < iterations; index++) { + u64 va = (first_block + index) << l2_shift; + u64 user_slot = sptm_walk(user_root, va, 2, NULL); + u64 value = nest ? read64(sptm_walk(shared_root, va, 2, NULL)) : 0; + if (read64(user_slot) != value) { + write64(user_slot, value); + changed = true; + } + } + + return sptm_finish_update(ctx, changed, false); +} + +bool sptm_surt_update(struct exc_info *ctx, bool allocate) +{ + // Subpage roots are 128-byte slots inside a SUBPAGE frame. + u64 root = (ctx->regs[0] & ~SPTM_PAGE_MASK) + ctx->regs[1] * 128; + struct sptm_surt_root *surt = sptm_find_surt_root(root); + + if (allocate) { + if (!surt) { + for (size_t index = 0; index < ARRAY_SIZE(sptm.surt_roots); index++) { + if (!sptm.surt_roots[index].valid) { + surt = &sptm.surt_roots[index]; + break; + } + } + } + if (!surt) + return false; + + surt->root = root; + surt->attr_index = ctx->regs[2]; + surt->asid = ctx->regs[4]; + surt->valid = true; + } else if (surt) { + memset(surt, 0, sizeof(*surt)); + } + + memset((void *)root, 0, 128); + sysop("dsb sy"); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +bool sptm_condemn_leaf_table(struct exc_info *ctx, bool condemn) +{ + u64 slot = sptm_walk(ctx->regs[0], ctx->regs[1], 2, NULL); + u64 value = slot ? read64(slot) : 0; + if (!(value & 1)) { + ctx->regs[0] = SPTM_STATUS_TABLE_NOT_PRESENT; + return true; + } + + // Only one caller may condemn a table; clearing is idempotent. + if (!!(value & BIT(2)) == condemn) { + ctx->regs[0] = condemn ? SPTM_STATUS_TABLE_NOT_PRESENT : SPTM_STATUS_SUCCESS; + return true; + } + + write64(slot, value ^ BIT(2)); + sptm_publish_stage1(); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} diff --git a/src/hv_sptm_sart.c b/src/hv_sptm_sart.c new file mode 100644 index 00000000..79d1a642 --- /dev/null +++ b/src/hv_sptm_sart.c @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include "hv_sptm_internal.h" + +static void sptm_sart_read_entry(size_t index, u32 *flags, u64 *pa, u64 *size) +{ + *flags = read32(sptm.sart_base + index * sizeof(u32)) & 0xff; + *pa = (u64)read32(sptm.sart_base + 0x40 + index * sizeof(u32)) << SPTM_SART_PAGE_SHIFT; + u64 pages = read32(sptm.sart_base + 0x80 + index * sizeof(u32)) & 0x3fffffff; + if (*flags && !sptm.sart_exclusive_bounds) + pages++; + *size = pages << SPTM_SART_PAGE_SHIFT; +} + +static bool sptm_sart_map(struct exc_info *ctx) +{ + u64 pa = ctx->regs[0]; + u64 size = ctx->regs[1]; + u32 permission = ctx->regs[2]; + bool guard = ctx->regs[3]; + u64 pages = size >> SPTM_SART_PAGE_SHIFT; + u64 encoded_pages = sptm.sart_exclusive_bounds ? pages : pages - 1; + + spin_lock(&sptm.service_lock); + size_t free_index = SPTM_SART_ENTRIES; + for (size_t index = 0; index < SPTM_SART_ENTRIES; index++) { + if ((sptm.sart_protected_mask & BIT(index)) || + (read32(sptm.sart_base + index * sizeof(u32)) & 0xff)) + continue; + free_index = index; + break; + } + + if (free_index == SPTM_SART_ENTRIES) { + printf("HV: SPTM SART has no free hardware entry " + "(protected=0x%x guarded=0x%x)\n", + sptm.sart_protected_mask, sptm.sart_guarded_mask); + spin_unlock(&sptm.service_lock); + return false; + } + + if (guard) { + if (!sptm.sart_guarded_mask && sptm.sart_canary) + write32(sptm.sart_canary, 0xabfedeed); + sptm.sart_guarded_mask |= BIT(free_index); + } + + write32(sptm.sart_base + 0x40 + free_index * sizeof(u32), pa >> SPTM_SART_PAGE_SHIFT); + write32(sptm.sart_base + 0x80 + free_index * sizeof(u32), encoded_pages); + write32(sptm.sart_base + free_index * sizeof(u32), permission ? 0xff : 0xea); + sysop("dsb sy"); + spin_unlock(&sptm.service_lock); + + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +static bool sptm_sart_unmap(struct exc_info *ctx) +{ + u64 pa = ctx->regs[0]; + u64 size = ctx->regs[1]; + + spin_lock(&sptm.service_lock); + size_t found = 0; + u32 found_flags = 0; + for (; found < SPTM_SART_ENTRIES; found++) { + u64 entry_pa, entry_size; + if (sptm.sart_protected_mask & BIT(found)) + continue; + sptm_sart_read_entry(found, &found_flags, &entry_pa, &entry_size); + if (found_flags && entry_pa == pa && entry_size == size) + break; + } + + sptm.sart_guarded_mask &= ~BIT(found); + + write32(sptm.sart_base + found * sizeof(u32), 0); + write32(sptm.sart_base + 0x80 + found * sizeof(u32), 0); + write32(sptm.sart_base + 0x40 + found * sizeof(u32), 0); + sysop("dsb sy"); + spin_unlock(&sptm.service_lock); + + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +bool sptm_handle_sart(struct exc_info *ctx, u32 endpoint) +{ + switch (endpoint) { + case 0: + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; + case 1: + return sptm_sart_map(ctx); + case 2: + return sptm_sart_unmap(ctx); + default: + __builtin_unreachable(); + } +} diff --git a/src/hv_sptm_txm.c b/src/hv_sptm_txm.c new file mode 100644 index 00000000..283eaa6e --- /dev/null +++ b/src/hv_sptm_txm.c @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include "hv_sptm_internal.h" + +bool sptm_handle_txm(struct exc_info *ctx, u32 endpoint) +{ + const u64 success = 0; + const u64 generic = 1; + const u64 not_found = 8; + const u64 not_permitted = 38; + const u64 not_supported = 41; + u64 code = success; + u64 words[SPTM_TXM_MAX_WORDS] = {0}; + size_t word_count = 0; + + switch (endpoint) { + case 0: + code = generic; + break; + case 1: + word_count = 3; + break; + case 2: + words[0] = sptm.txm_info_va; + words[1] = sptm.txm_info_va + 0x318; + words[5] = sptm.txm_info_va; + word_count = 6; + break; + case 3: + word_count = 4; + break; + case 4: + case 15: + case 27: + case 32: + case 43: + word_count = 1; + break; + case 7: + case 11: + case 17: + case 22: + case 24: + case 34: + case 38: + case 41: + code = not_supported; + break; + case 12: + case 35: + words[0] = 1; + word_count = 1; + break; + case 14: + case 16: + case 30: + case 44: + code = not_found; + break; + case 19: + case 23: + case 25: + word_count = 2; + break; + case 5: + case 6: + case 8: + case 9: + case 10: + case 13: + case 18: + case 20: + case 21: + case 26: + case 28: + case 29: + case 31: + case 33: + case 36: + case 37: + case 39: + case 40: + case 42: + case 45: + break; + case 46: + case 47: + case 48: + case 49: + case 50: + case 51: + code = not_permitted; + break; + default: + code = not_supported; + break; + } + + u64 record = sptm_pointer_pa(ctx->regs[0] + SPTM_TXM_RECORD_OFFSET); + memset((void *)record, 0, SPTM_TXM_RECORD_SIZE); + write64(record + 0x08, code); + write64(record + 0x18, word_count); + for (size_t index = 0; index < word_count; index++) + write64(record + 0x20 + index * sizeof(u64), words[index]); + dc_civac_range((void *)record, SPTM_TXM_RECORD_SIZE); + sysop("dsb sy"); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} diff --git a/src/hv_sptm_uat.c b/src/hv_sptm_uat.c new file mode 100644 index 00000000..19933ba9 --- /dev/null +++ b/src/hv_sptm_uat.c @@ -0,0 +1,464 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include "hv_sptm_internal.h" + +static bool sptm_uat_sentinel(u64 root) +{ + return root == UINT32_MAX || root == UINT64_MAX; +} + +static void sptm_uat_handoff_acquire(u64 address, size_t size) +{ + u64 start = ALIGN_DOWN(address, SPTM_CACHE_LINE_SIZE); + size = ALIGN_UP(address - start + size, SPTM_CACHE_LINE_SIZE); + sysop("dsb osh"); + dc_ivac_range((void *)start, size); + sysop("dsb osh"); +} + +static u64 sptm_uat_handoff_slot_addr(u32 slot) +{ + return sptm.uat_handoff_pa + UAT_HANDOFF_SLOT_BASE + slot * UAT_HANDOFF_SLOT_STRIDE; +} + +// Global states share one handoff slot; per-context states use their context's slot. +static u32 sptm_uat_handoff_slot_for_state(u64 state_pa) +{ + u8 state_type = read8(state_pa); + if (state_type == UAT_STATE_GLOBAL_MODE0 || state_type == UAT_STATE_GLOBAL_MODE1) + return UAT_HANDOFF_SHARED_SLOT; + return read16(state_pa + UAT_STATE_CONTEXT_ID); +} + +static u64 sptm_uat_handoff_post(u64 state_pa, u32 slot, u64 va, u64 count) +{ + if (!count) + return 0; + + u64 handoff = sptm.uat_handoff_pa; + sptm_uat_handoff_acquire(handoff, UAT_HANDOFF_SLOT_BASE); + bool firmware_accepting = read64(handoff + UAT_HANDOFF_MAGIC_FW) == UAT_HANDOFF_MAGIC; + bool could_be_cached = + slot == UAT_HANDOFF_SHARED_SLOT || read32(handoff + UAT_HANDOFF_CURRENT_CTX) == slot; + bool request_flush = firmware_accepting && could_be_cached; + u64 slot_addr = sptm_uat_handoff_slot_addr(slot); + u64 address = request_flush ? va : UAT_HANDOFF_UNMAP_TAG | (va & UAT_HANDOFF_ADDR_MASK); + + write64(slot_addr + UAT_HANDOFF_SLOT_ADDR, address); + write64(slot_addr + UAT_HANDOFF_SLOT_SIZE, count * SPTM_PAGE_SIZE); + sysop("dsb osh"); + write64(slot_addr + UAT_HANDOFF_SLOT_STATE, request_flush ? 1 : 2); + sysop("dsb osh"); + + sptm.uat_handoff_owner[slot] = state_pa; + return request_flush ? 2 : 0; +} + +static void sptm_uat_handoff_complete(u64 state_pa) +{ + for (u32 slot = 0; slot < UAT_HANDOFF_SLOT_COUNT; slot++) { + if (sptm.uat_handoff_owner[slot] != state_pa) + continue; + + u64 state_addr = sptm_uat_handoff_slot_addr(slot) + UAT_HANDOFF_SLOT_STATE; + sptm_uat_handoff_acquire(state_addr, sizeof(u32)); + write32(state_addr, 0); + sysop("dsb osh"); + sptm.uat_handoff_owner[slot] = 0; + } +} + +static void sptm_uat_publish_guard(u64 state_pa, u8 guard) +{ + sysop("dsb osh"); + write8(state_pa + UAT_STATE_GUARD, guard); + sysop("dsb osh"); +} + +// Start a resumable operation: set its cursors and guard. CURSOR2 always starts at 0. +static void sptm_uat_begin(u64 state_pa, u8 guard, u64 cursor0, u64 cursor1, u64 cursor3) +{ + write64(state_pa + UAT_STATE_CURSOR0, cursor0); + write64(state_pa + UAT_STATE_CURSOR1, cursor1); + write64(state_pa + UAT_STATE_CURSOR2, 0); + write64(state_pa + UAT_STATE_CURSOR3, cursor3); + sptm_uat_publish_guard(state_pa, guard); +} + +static void sptm_uat_write_descriptor(u64 slot, u64 descriptor) +{ + // this is where we would use the coprocessor flush op + write64(slot, descriptor); + sysop("dsb oshst"); +} + +static u64 sptm_uat_normalize_va(u64 va) +{ + return va & (BIT(sptm.uat_va_width) - 1); +} + +void sptm_uat_tlbi_all(void) +{ + // Match VMALLE1OS's outer-shareable domain. + sysop("dsb oshst"); + __asm__ volatile("sys #0, c8, c1, #0, xzr" ::: "memory"); + sysop("dsb osh"); + sysop("isb"); +} + +static u64 sptm_uat_root(u64 state_pa, u64 va) +{ + u64 root_offset = UAT_STATE_ROOT0; + u8 state_type = read8(state_pa); + /* Boot-seeded shared-TTBR1 objects intentionally have no root0. */ + if (state_type == UAT_STATE_GLOBAL_MODE0 || state_type == UAT_STATE_GLOBAL_MODE1 || + (state_type == UAT_STATE_DUAL_ROOT && ((va >> (sptm.uat_va_width - 1)) & 1))) + root_offset = UAT_STATE_ROOT1; + return read64(state_pa + root_offset) & SPTM_DESC_PA_MASK_16K; +} + +// Return the Normal-NC alias of va's slot in the level-`level` table. +static u64 sptm_uat_slot(u64 state_pa, u64 va, u32 level) +{ + va = sptm_uat_normalize_va(va); + u64 table = sptm_uat_root(state_pa, va); + + // The L1 index uses the VA bits above the 16 KiB three-level walk. + u64 l1_mask = BIT(max((int)sptm.uat_va_width - 37, 0)) - 1; + u64 indices[] = { + (va >> 36) & l1_mask, + (va >> 25) & 0x7ff, + (va >> SPTM_PAGE_SHIFT) & 0x7ff, + }; + + for (u32 current_level = 1; current_level < level; current_level++) + table = read64(sptm_iommu_table_va(table) + indices[current_level - 1] * sizeof(u64)) & + SPTM_DESC_PA_MASK_16K; + return sptm_iommu_table_va(table) + indices[level - 1] * sizeof(u64); +} + +static bool sptm_uat_init_state(struct exc_info *ctx) +{ + u64 state_pa = ctx->regs[0]; + u64 root0 = ctx->regs[1]; + u64 root1 = ctx->regs[2]; + bool dual = !sptm_uat_sentinel(root1); + + memset((void *)state_pa, 0, SPTM_PAGE_SIZE); + write8(state_pa, dual ? UAT_STATE_DUAL_ROOT : UAT_STATE_SINGLE_ROOT); + write64(state_pa + UAT_STATE_ROOT0, root0); + write64(state_pa + UAT_STATE_ROOT1, root1); + write16(state_pa + UAT_STATE_CONTEXT_ID, UINT16_MAX); + sptm_uat_publish_guard(state_pa, UAT_GUARD_IDLE); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +static bool sptm_uat_destroy_state(struct exc_info *ctx) +{ + u64 state_pa = ctx->regs[0]; + memset((void *)state_pa, 0, SPTM_PAGE_SIZE); + write64(state_pa + UAT_STATE_ROOT0, UINT32_MAX); + write64(state_pa + UAT_STATE_ROOT1, UINT32_MAX); + write16(state_pa + UAT_STATE_CONTEXT_ID, UINT16_MAX); + sysop("dsb osh"); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +static bool sptm_uat_map_table(struct exc_info *ctx) +{ + u64 slot = sptm_uat_slot(ctx->regs[0], ctx->regs[1], ctx->regs[2]); + sptm_uat_write_descriptor(slot, (ctx->regs[3] & SPTM_DESC_PA_MASK_16K) | UAT_DESC_TABLE); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +static bool sptm_uat_unmap_table(struct exc_info *ctx) +{ + sptm_uat_write_descriptor(sptm_uat_slot(ctx->regs[0], ctx->regs[1], ctx->regs[2]), 0); + sptm_uat_tlbi_all(); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +static u64 sptm_uat_leaf_descriptor(u64 state_pa, u64 target, u32 options) +{ + // Preserve per-context memory types and permissions from the map options. + static const u8 permissions[4][4] = { + {0, 2, 2 | BIT(2), 2 | BIT(3)}, + {1, BIT(2), 0, 0}, + {1 | BIT(2), 0, BIT(3), 0}, + {1 | BIT(3), 1 | BIT(2) | BIT(3), 0, BIT(2) | BIT(3)}, + }; + + u8 permission = permissions[options & 3][(options >> 8) & 3]; + u8 state_type = read8(state_pa); + u64 attr = options & BIT(2) ? 1 : options & BIT(3) ? 2 : 0; + u64 flags = UAT_DESC_OS | UAT_DESC_AF | UAT_DESC_VALID | BIT(1) | + (attr << UAT_DESC_ATTR_SHIFT) | ((u64)(permission & 3) << UAT_DESC_AP_SHIFT); + if (state_type == UAT_STATE_SINGLE_ROOT || state_type == UAT_STATE_DUAL_ROOT) + flags |= UAT_DESC_NG; + if (permission & BIT(2)) + flags |= UAT_DESC_PXN; + if (permission & BIT(3)) + flags |= UAT_DESC_UXN; + + return (target & SPTM_DESC_PA_MASK_16K) | flags; +} + +static bool sptm_uat_map_continue(struct exc_info *ctx) +{ + u64 state_pa = ctx->regs[0]; + u64 va = read64(state_pa + UAT_STATE_CURSOR0); + u64 segment_count = read64(state_pa + UAT_STATE_CURSOR1); + u64 segment_index = read64(state_pa + UAT_STATE_CURSOR2); + u64 page_index = read64(state_pa + UAT_STATE_CURSOR3); + u32 options = read32(state_pa + UAT_STATE_OPTIONS); + u32 budget = min(sptm.uat_mapping_limit, SPTM_UAT_BATCH_PAGES); + + while (budget && segment_index < segment_count) { + struct sptm_uat_segment *segment = + (void *)(state_pa + UAT_STATE_MAP_SEGMENTS + segment_index * sizeof(*segment)); + if (page_index >= segment->count) { + segment_index++; + page_index = 0; + continue; + } + + u64 slot = sptm_uat_slot(state_pa, va, 3); + u64 target = segment->first + page_index * SPTM_PAGE_SIZE; + u64 descriptor = sptm_uat_leaf_descriptor(state_pa, target, options); + u64 old_descriptor = read64(slot); + if (old_descriptor != descriptor) { + sptm_uat_write_descriptor(slot, descriptor); + if (old_descriptor & UAT_DESC_VALID) + sptm_uat_tlbi_all(); + } + va += SPTM_PAGE_SIZE; + page_index++; + budget--; + } + + write64(state_pa + UAT_STATE_CURSOR0, va); + write64(state_pa + UAT_STATE_CURSOR2, segment_index); + write64(state_pa + UAT_STATE_CURSOR3, page_index); + bool done = segment_index >= segment_count; + sptm_uat_publish_guard(state_pa, done ? UAT_GUARD_IDLE : UAT_GUARD_MAP); + ctx->regs[0] = done ? 0 : 1; + return true; +} + +static bool sptm_uat_map_begin(struct exc_info *ctx) +{ + u64 state_pa = ctx->regs[0]; + u64 segment_count = ctx->regs[3]; + + memcpy((void *)(state_pa + UAT_STATE_MAP_SEGMENTS), (void *)ctx->regs[2], + segment_count * sizeof(struct sptm_uat_segment)); + write32(state_pa + UAT_STATE_OPTIONS, ctx->regs[4]); + sptm_uat_begin(state_pa, UAT_GUARD_MAP, sptm_uat_normalize_va(ctx->regs[1]), segment_count, 0); + return sptm_uat_map_continue(ctx); +} + +static bool sptm_uat_prepare_unmap_continue(struct exc_info *ctx) +{ + u64 state_pa = ctx->regs[0]; + u64 va = read64(state_pa + UAT_STATE_CURSOR0); + u64 count = read64(state_pa + UAT_STATE_CURSOR1); + u64 processed = read64(state_pa + UAT_STATE_CURSOR2); + u32 budget = sptm.uat_mapping_limit; + + for (; budget && processed < count; processed++, budget--) { + u64 slot = sptm_uat_slot(state_pa, va + processed * SPTM_PAGE_SIZE, 3); + u64 descriptor = read64(slot); + u32 attr = (descriptor >> UAT_DESC_ATTR_SHIFT) & 7; + u32 ap = (descriptor >> UAT_DESC_AP_SHIFT) & 3; + bool firmware_candidate = + !attr && ((!ap && (descriptor & UAT_DESC_UXN)) || + (ap == 1 && (descriptor & (UAT_DESC_PXN | UAT_DESC_UXN)))); + if ((descriptor & UAT_DESC_VALID) && firmware_candidate) + sptm_uat_write_descriptor(slot, descriptor | UAT_DESC_FIRMWARE_OWNED); + } + + write64(state_pa + UAT_STATE_CURSOR2, processed); + bool done = processed >= count; + u64 result = 1; + if (done) { + // CURSOR3 holds the handoff slot plus one, or zero for an empty range. + u64 slot_cookie = read64(state_pa + UAT_STATE_CURSOR3); + result = slot_cookie ? sptm_uat_handoff_post(state_pa, slot_cookie - 1, va, count) : 0; + } + sptm_uat_publish_guard(state_pa, done ? UAT_GUARD_IDLE : UAT_GUARD_PREPARE_UNMAP); + ctx->regs[0] = result; + return true; +} + +static bool sptm_uat_prepare_unmap_begin(struct exc_info *ctx) +{ + u64 state_pa = ctx->regs[0]; + u64 count = ctx->regs[2]; + + sptm_uat_begin(state_pa, UAT_GUARD_PREPARE_UNMAP, sptm_uat_normalize_va(ctx->regs[1]), count, + count ? sptm_uat_handoff_slot_for_state(state_pa) + 1 : 0); + return sptm_uat_prepare_unmap_continue(ctx); +} + +static bool sptm_uat_unmap_continue(struct exc_info *ctx) +{ + u64 state_pa = ctx->regs[0]; + u64 segment_count = read64(state_pa + UAT_STATE_CURSOR0); + u64 segment_index = read64(state_pa + UAT_STATE_CURSOR1); + u64 page_index = read64(state_pa + UAT_STATE_CURSOR2); + u32 budget = min(sptm.uat_mapping_limit, SPTM_UAT_BATCH_PAGES); + bool cleared = false; + + // Clear VALID/TYPE while keeping the payload for handoff. + while (budget && segment_index < segment_count) { + struct sptm_uat_segment *segment = + (void *)(state_pa + UAT_STATE_UNMAP_SEGMENTS + segment_index * sizeof(*segment)); + if (page_index >= segment->count) { + segment_index++; + page_index = 0; + continue; + } + + u64 slot = sptm_uat_slot(state_pa, segment->first + page_index * SPTM_PAGE_SIZE, 3); + sptm_uat_write_descriptor(slot, read64(slot) & ~3ULL); + cleared = true; + page_index++; + budget--; + } + + if (cleared) + sptm_uat_tlbi_all(); + + write64(state_pa + UAT_STATE_CURSOR1, segment_index); + write64(state_pa + UAT_STATE_CURSOR2, page_index); + bool done = segment_index >= segment_count; + if (done) + sptm_uat_handoff_complete(state_pa); + sptm_uat_publish_guard(state_pa, done ? UAT_GUARD_IDLE : UAT_GUARD_UNMAP); + ctx->regs[0] = !done; + return true; +} + +static bool sptm_uat_unmap_begin(struct exc_info *ctx) +{ + u64 state_pa = ctx->regs[0]; + u64 segment_count = ctx->regs[2]; + + memcpy((void *)(state_pa + UAT_STATE_UNMAP_SEGMENTS), (void *)ctx->regs[1], + segment_count * sizeof(struct sptm_uat_segment)); + sptm_uat_begin(state_pa, UAT_GUARD_UNMAP, segment_count, 0, 0); + return sptm_uat_unmap_continue(ctx); +} + +static u64 sptm_uat_ttbr(u64 root, u16 context_id) +{ + if (sptm_uat_sentinel(root)) + return 0; + return (root & SPTM_DESC_PA_MASK_16K) | ((u64)context_id << 48) | 1; +} + +static bool sptm_uat_set_context(struct exc_info *ctx) +{ + u64 state_pa = ctx->regs[0]; + u64 context_id = ctx->regs[1]; + u64 context = sptm.uat_gpu_contexts + context_id * 16; + u64 root0 = read64(state_pa + UAT_STATE_ROOT0); + u64 root1 = read64(state_pa + UAT_STATE_ROOT1); + // Single-root contexts share the global state's TTBR1. + if (read8(state_pa) == UAT_STATE_SINGLE_ROOT) + root1 = read64(sptm.uat_global_state + UAT_STATE_ROOT1); + + write64(context, sptm_uat_ttbr(root0, context_id)); + write64(context + 8, sptm_uat_ttbr(root1, context_id)); + sysop("dsb osh"); + write16(state_pa + UAT_STATE_CONTEXT_ID, context_id); + sptm_uat_publish_guard(state_pa, UAT_GUARD_IDLE); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +static bool sptm_uat_remove_context(struct exc_info *ctx) +{ + u64 state_pa = ctx->regs[0]; + u16 context_id = read16(state_pa + UAT_STATE_CONTEXT_ID); + if (context_id != UINT16_MAX) { + u64 context = sptm.uat_gpu_contexts + context_id * 16; + write64(context, read64(context) & ~1ULL); + write64(context + 8, read64(context + 8) & ~1ULL); + sysop("dsb osh"); + u64 asid = (u64)context_id << 48; + __asm__ volatile("sys #0, c8, c1, #2, %0" : : "r"(asid) : "memory"); + sysop("dsb osh"); + sysop("isb"); + } + write16(state_pa + UAT_STATE_CONTEXT_ID, UINT16_MAX); + sptm_uat_publish_guard(state_pa, UAT_GUARD_IDLE); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +static bool sptm_uat_get_info(struct exc_info *ctx) +{ + const u64 values[] = { + sptm.uat_mode, + sptm.uat_mode == 0 ? sptm.uat_global_state : UINT64_MAX, + sptm.uat_mode == 1 ? sptm.uat_global_state : UINT64_MAX, + sptm.uat_va_width - 1, + // The L1 bits, in their original VA positions. + (BIT(max((int)sptm.uat_va_width - 37, 0)) - 1) << 36, + sptm.uat_segment_limit, + 0, + BIT(sptm.uat_va_width), + sptm.uat_state_size, + sptm.uat_shared_l2_va, + }; + ctx->regs[0] = ctx->regs[0] < ARRAY_SIZE(values) ? values[ctx->regs[0]] : UINT64_MAX; + return true; +} + +bool sptm_handle_uat(struct exc_info *ctx, u32 endpoint) +{ + static bool (*const handlers[])(struct exc_info *) = { + sptm_uat_init_state, + sptm_uat_destroy_state, + sptm_uat_map_table, + sptm_uat_unmap_table, + sptm_uat_map_begin, + sptm_uat_map_continue, + sptm_uat_prepare_unmap_begin, + sptm_uat_prepare_unmap_continue, + sptm_uat_unmap_begin, + sptm_uat_unmap_continue, + sptm_uat_set_context, + sptm_uat_remove_context, + sptm_uat_get_info, + }; + spin_lock(&sptm.service_lock); + bool handled = handlers[endpoint](ctx); + spin_unlock(&sptm.service_lock); + return handled; +} + +// Boot fills in the UAT parameters from the device tree before calling this. +void sptm_uat_configure(u64 global_root_pa, u64 handoff_pa) +{ + sptm.uat_state_size = + UAT_STATE_UNMAP_SEGMENTS + sptm.uat_segment_limit * sizeof(struct sptm_uat_segment); + + // Boot-seeded UAT tables bypass RETYPE, so retire their WB aliases here. + u64 host_pages[] = {global_root_pa, sptm.uat_shared_l2_pa}; + mmu_map_ram_pages_nc(host_pages, ARRAY_SIZE(host_pages), true); + + // GFXHandoff still exposes CUR_CTX under its historical UNK name. Publish the AP + // magic only after the rest of the page is reset. + sptm.uat_handoff_pa = handoff_pa; + memset((void *)handoff_pa, 0, UAT_HANDOFF_MIN_SIZE); + write32(handoff_pa + UAT_HANDOFF_CURRENT_CTX, UINT32_MAX); + sysop("dsb osh"); + write64(handoff_pa + UAT_HANDOFF_MAGIC_AP, UAT_HANDOFF_MAGIC); + sysop("dsb osh"); +} diff --git a/src/hv_sptm_xnu.c b/src/hv_sptm_xnu.c new file mode 100644 index 00000000..93e145b0 --- /dev/null +++ b/src/hv_sptm_xnu.c @@ -0,0 +1,161 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include "hv_sptm_internal.h" + +static char sptm_serial_line[SPTM_SERIAL_LINE_SIZE]; +static size_t sptm_serial_line_len; + +// Pointer signing is a no-op, so each 24-byte operation's pointer is returned unchanged. +static bool sptm_batch_sign_user_pointers(struct exc_info *ctx) +{ + u64 operations = sptm_pointer_pa(ctx->regs[0]); + u64 scratch = sptm_scratch_pa(ctx); + for (size_t index = 0; index < ctx->regs[1]; index++) + write64(scratch + index * sizeof(u64), read64(operations + index * 24)); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +static void sptm_serial_flush(void) +{ + if (!sptm_serial_line_len) + return; + + sptm_serial_line[sptm_serial_line_len] = '\0'; + printf("HV: SPTM serial: %s\n", sptm_serial_line); + sptm_serial_line_len = 0; +} + +static void sptm_serial_putc(u8 character) +{ + if (!character || character == '\n' || character == '\r') { + sptm_serial_flush(); + return; + } + if (sptm_serial_line_len == sizeof(sptm_serial_line) - 1) + sptm_serial_flush(); + sptm_serial_line[sptm_serial_line_len++] = character; +} + +static bool sptm_switch_root(struct exc_info *ctx) +{ + u64 root = ctx->regs[0] & 0x0000ffffffffffffULL; + struct sptm_surt_root *surt = sptm_find_surt_root(root); + u8 *entry = surt ? NULL : sptm_frame_entry(root); + u8 type = surt ? SPTM_FRAME_SUBPAGE : entry[2]; + u64 asid = surt ? surt->asid : read16((u64)entry + 10); + + if (type == SPTM_FRAME_KERNEL_ROOT) + msr(TTBR1_EL12, root); + else + msr(TTBR0_EL12, root | (asid << 48)); + sptm_publish_stage1(); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; +} + +bool sptm_handle_xnu_bootstrap(struct exc_info *ctx, u32 endpoint) +{ + switch (endpoint) { + case 0: /* LOCKDOWN */ + case 12: /* CONFIGURE_ROOT */ + case 15: /* FIXUPS_COMPLETE */ + case 18: /* REGISTER_EXC_RETURN */ + case 20: /* SLIDE_REGION */ + case 23: /* REG_WRITE */ + case 28: /* GUEST_EXIT */ + case 29: /* MAP_SK_DOMAIN */ + case 30: /* HIB_BEGIN */ + case 31: /* HIB_VERIFY_HASH_NON_WIRED */ + case 32: /* HIB_FINALIZE_NON_WIRED */ + case 37: /* SPTM_SYSCTL */ + case 38: /* DISABLE_KERNEL_MODE_CPA2 */ + case 39: /* SET_SHARED_REGION */ + case 48: /* PROGRAM_IRGKEY */ + case 49: /* REG_SNAPSHOT */ + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; + case 1: /* RETYPE */ + return sptm_retype_frame(ctx); + case 2: /* MAP_PAGE */ + return sptm_map_page(ctx); + case 3: /* MAP_TABLE */ + return sptm_map_table(ctx); + case 4: /* UNMAP_TABLE */ + return sptm_unmap_table(ctx); + case 5: /* UPDATE_REGION */ + return sptm_update_region(ctx); + case 6: /* UPDATE_DISJOINT */ + return sptm_update_disjoint(ctx); + case 7: /* UNMAP_REGION */ + return sptm_unmap_region(ctx); + case 8: /* UNMAP_DISJOINT */ + return sptm_unmap_disjoint(ctx); + case 9: /* CONFIGURE_SHAREDREGION */ + return sptm_configure_shared_region(ctx); + case 10: /* NEST_REGION */ + if (!sptm_nest_region(ctx, true)) + return false; + sptm_publish_commpage_policy(); + return true; + case 11: /* UNNEST_REGION */ + return sptm_nest_region(ctx, false); + case 13: /* SWITCH_ROOT */ + return sptm_switch_root(ctx); + case 14: /* REGISTER_CPU */ + if (!sptm_register_cpu(ctx->regs[0])) + return false; + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; + case 16: /* SIGN_USER_POINTER */ + case 17: /* AUTH_USER_POINTER */ + /* Pointer-valued calls: leave x0 as the unmodified pointer. */ + return true; + case 19: { /* CPU_ID */ + struct sptm_cpu *cpu = sptm_find_cpu(ctx->regs[0]); + if (!cpu) + return false; + ctx->regs[0] = cpu->logical_id; + return true; + } + case 21: /* UPDATE_DISJOINT_MULTIPAGE */ + return sptm_update_disjoint_multipage(ctx); + case 22: /* REG_READ */ + ctx->regs[0] = 0; + return true; + case 24: /* GUEST_VA_TO_IPA */ + ctx->regs[0] = ~0ULL; + return true; + case 25: /* GUEST_STAGE1_TLBOP */ + sptm_publish_stage1(); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; + case 26: /* GUEST_STAGE2_TLBOP */ + sysop("dsb ishst"); + sysop("tlbi vmalls12e1is"); + sysop("dsb ish"); + sysop("isb"); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; + case 40: /* BATCH_SIGN_USER_POINTER */ + return sptm_batch_sign_user_pointers(ctx); + case 41: /* SURT_ALLOC */ + return sptm_surt_update(ctx, true); + case 42: /* SURT_FREE */ + return sptm_surt_update(ctx, false); + case 43: /* CONDEMN_LEAF_TABLE */ + return sptm_condemn_leaf_table(ctx, true); + case 44: /* UNCONDEMN_LEAF_TABLE */ + return sptm_condemn_leaf_table(ctx, false); + case 45: /* SPTM_SERIAL_PUTC */ + sptm_serial_putc(ctx->regs[0]); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; + case 46: /* SPTM_SERIAL_DISABLE */ + sptm_serial_flush(); + ctx->regs[0] = SPTM_STATUS_SUCCESS; + return true; + default: + return false; + } +} diff --git a/src/hv_t8140.c b/src/hv_t8140.c deleted file mode 100644 index a6b4bb14..00000000 --- a/src/hv_t8140.c +++ /dev/null @@ -1,108 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * This T8140 CPU/ACC/CPM hook is a horrible hack that we don't know how to - * get rid of. We spent almost two weeks experimenting and ultimately - * converged to this file. - * - * Known failed edits: - * - * - Deleting this hook or forwarding all writes causes PEH/DPE bringup - * failures. - * - Dropping all writes produced SPMI errors and hard-rebooted the machine, - * and did not fix SPMI. - * - */ - -#include "hv.h" -#include "cpu_regs.h" -#include "memory.h" -#include "smp.h" -#include "soc.h" -#include "string.h" -#include "utils.h" - -#define T8140_ACC_PAGE_SIZE 0x4000UL -#define T8140_ACC_GUARD_FIRST_PAGE 2 - -static const u64 t8140_acc_pages[] = { - 0x210074000UL, 0x211074000UL, 0x210e44000UL, 0x211e44000UL, - 0x210e48000UL, 0x211e48000UL, 0x210058000UL, 0x210158000UL, - 0x210258000UL, 0x210358000UL, 0x211058000UL, 0x211158000UL, -}; - -static void t8140_flush_stage2_hooks(void) -{ - sysop("dsb ishst"); - sysop("tlbi vmalls12e1is"); - sysop("dsb ish"); - sysop("isb"); -} - -static int t8140_acc_page_index(u64 ipa) -{ - u64 page = ipa & ~(T8140_ACC_PAGE_SIZE - 1); - - for (u32 i = 0; i < ARRAY_SIZE(t8140_acc_pages); i++) - if (page == t8140_acc_pages[i]) - return i; - - return -1; -} - -#define T8140_ACC_PC_DPE_LOOP_LO 0xfffffe00097b7b00UL -#define T8140_ACC_PC_DPE_LOOP_HI 0xfffffe00097b8d00UL - -static bool t8140_acc_hook(struct exc_info *ctx, u64 ipa, u64 *val, bool write, int width) -{ - if (width < 0 || width > 6) - return false; - - int idx = t8140_acc_page_index(ipa); - if (idx < 0) - return false; - - u64 bytes = 1UL << width; - u64 off = ipa & (T8140_ACC_PAGE_SIZE - 1); - if (off + bytes > T8140_ACC_PAGE_SIZE) - return false; - - if (write) { - u64 pc = ctx->elr; - if (pc >= T8140_ACC_PC_DPE_LOOP_LO && pc < T8140_ACC_PC_DPE_LOOP_HI) { - return hv_pa_rw(ctx, ipa, val, true, width); - } - if (idx >= 6 && idx <= 9 && (off == 0x1000 || off == 0x1008)) { - return hv_pa_rw(ctx, ipa, val, true, width); - } - /* - * For E-core CPUs (idx 6..9 -> pages 0x21x058000), forward so the - * silicon power state matches XNU's view; otherwise cpu0 re-enters - * peh.c with a stale SError that never clears. P-core CPUs (idx - * 10..11) are HV-managed; drop those. - */ - if (idx >= 6 && idx <= 9) { - return hv_pa_rw(ctx, ipa, val, true, width); - } - return true; - } - - return hv_pa_rw(ctx, ipa, val, false, width); -} - -int hv_t8140_map_accumulators(void) -{ - if (chip_id != T8140) - return 0; - - int failures = 0; - for (u32 i = T8140_ACC_GUARD_FIRST_PAGE; i < ARRAY_SIZE(t8140_acc_pages); i++) - failures += hv_map_hook(t8140_acc_pages[i], t8140_acc_hook, - T8140_ACC_PAGE_SIZE) != 0; - - u32 requested = ARRAY_SIZE(t8140_acc_pages) - T8140_ACC_GUARD_FIRST_PAGE; - u32 installed = requested - failures; - printf("HV: T8140 CPU/ACC/CPM write guard for %lu/%lu pages\n", - (u64)installed, (u64)requested); - t8140_flush_stage2_hooks(); - return failures ? -failures : 0; -} diff --git a/src/hv_vm.c b/src/hv_vm.c index 35d4186e..89ed390f 100644 --- a/src/hv_vm.c +++ b/src/hv_vm.c @@ -21,13 +21,14 @@ extern uint64_t ram_base; #define CACHE_LINE_LOG2 6 #define PTE_ACCESS BIT(10) -#define PTE_SH_NS (0b11L << 8) +#define PTE_SH_IS (0b11L << 8) #define PTE_S2AP_RW (0b11L << 6) #define PTE_S2AP_RO (0b01L << 6) +#define PTE_MEMATTR GENMASK(5, 2) #define PTE_MEMATTR_UNCHANGED (0b1111L << 2) -#define PTE_ATTRIBUTES (PTE_ACCESS | PTE_SH_NS | PTE_S2AP_RW | PTE_MEMATTR_UNCHANGED) -#define PTE_ATTRIBUTES_RO (PTE_ACCESS | PTE_SH_NS | PTE_S2AP_RO | PTE_MEMATTR_UNCHANGED) +#define PTE_ATTRIBUTES (PTE_ACCESS | PTE_SH_IS | PTE_S2AP_RW | PTE_MEMATTR_UNCHANGED) +#define PTE_ATTRIBUTES_RO (PTE_ACCESS | PTE_SH_IS | PTE_S2AP_RO | PTE_MEMATTR_UNCHANGED) #define PTE_LOWER_ATTRIBUTES GENMASK(13, 2) @@ -309,6 +310,9 @@ int hv_map(u64 from, u64 to, u64 size, u64 incr) { u64 chunk; bool hw = IS_HW(to); + bool force_l3 = + hw && FIELD_GET(PTE_MEMATTR, to) != + FIELD_GET(PTE_MEMATTR, PTE_MEMATTR_UNCHANGED); if (from & MASK(VADDR_L4_OFFSET_BITS) || size & MASK(VADDR_L4_OFFSET_BITS)) return -1; @@ -344,7 +348,8 @@ int hv_map(u64 from, u64 to, u64 size, u64 incr) // L2 mappings chunk = ALIGN_DOWN(size, BIT(VADDR_L2_OFFSET_BITS)); - if (chunk && (!hw || (to & VADDR_L2_ALIGN_MASK) == 0)) { + if (chunk && !force_l3 && + (!hw || (to & VADDR_L2_ALIGN_MASK) == 0)) { hv_pt_map_l2(from, to, chunk, incr); from += chunk; to += incr * chunk; diff --git a/src/kboot.c b/src/kboot.c index c5ed8486..eec8fa20 100644 --- a/src/kboot.c +++ b/src/kboot.c @@ -376,7 +376,7 @@ static int dt_set_memory(void) // iBoot makes this CTRR range writable only from the boot CPU. Keep Linux from assigning its // pages to secondary CPUs. - if (chip_id == T8132 || chip_id == T8140) { + if (!cpu_features->apple_sysregs_unlocked) { u64 ctrr_start = ALIGN_DOWN(mrs(SYS_IMP_APL_CTRR_LOWER_EL1), get_page_size()); u64 ctrr_end = ALIGN_UP(mrs(SYS_IMP_APL_CTRR_UPPER_EL1) + SZ_4K, get_page_size()); diff --git a/src/main.c b/src/main.c index e9315b47..04e71411 100644 --- a/src/main.c +++ b/src/main.c @@ -21,6 +21,7 @@ #include "pcie.h" #include "pmgr.h" #include "pmu.h" +#include "sapt.h" #include "sep.h" #include "smp.h" #include "string.h" @@ -157,6 +158,7 @@ void m1n1_main(void) gxf_init(); mcc_init(); mmu_init(); + sapt_disable(); aic_init(); #endif wdt_disable(); diff --git a/src/mcc.c b/src/mcc.c index 52230413..e4c8b529 100644 --- a/src/mcc.c +++ b/src/mcc.c @@ -250,13 +250,10 @@ static void mcc_unmap_adt_tz_carveouts(void) int mcc_unmap_carveouts(void) { - if (!mcc_initialized) - return -1; - mcc_carveout_count = 0; memset(mcc_carveouts, 0, sizeof mcc_carveouts); - if (mcc_regs[0].tz) { + if (mcc_initialized && mcc_regs[0].tz) { // All MCCs and planes should have identical configs // Note: For unhandled machines, the TZ regions can be found (on m1, m2, m3) by looking at // region-id-2 and region-id-4 on a booted macos, in the /chosen/carveout-memory-map DT node. diff --git a/src/memory.c b/src/memory.c index 2328bd0f..fe780faa 100644 --- a/src/memory.c +++ b/src/memory.c @@ -38,6 +38,7 @@ CACHE_RANGE_OP(ic_ivau_range, "ic ivau") CACHE_RANGE_OP(dc_ivac_range, "dc ivac") CACHE_RANGE_OP(dc_zva_range, "dc zva") +/* WARNING: This is a noop on Apple Silicon. */ CACHE_RANGE_OP(dc_cvac_range, "dc cvac") CACHE_RANGE_OP(dc_cvau_range, "dc cvau") CACHE_RANGE_OP(dc_civac_range, "dc civac") @@ -175,11 +176,80 @@ enum SPRR_val_t { static u64 *mmu_pt_L0; +#define MMU_MAX_PENDING_PTE_UPDATES 16384 +struct mmu_pending_pte_update { + u64 *slot; + u64 value; +}; +static struct mmu_pending_pte_update mmu_pending_pte_updates[MMU_MAX_PENDING_PTE_UPDATES]; +static size_t mmu_pending_pte_update_count; + +static void mmu_publish_mappings(void); + +static bool mmu_pt_pending_value(u64 *slot, u64 *value) +{ + for (size_t index = mmu_pending_pte_update_count; index; index--) { + struct mmu_pending_pte_update *update = &mmu_pending_pte_updates[index - 1]; + if (update->slot == slot) { + *value = update->value; + return true; + } + } + *value = *slot; + return false; +} + +static void mmu_pt_queue_update(u64 *slot, u64 value) +{ + if (!mmu_active()) { + *slot = value; + return; + } + + if (mmu_pending_pte_update_count >= ARRAY_SIZE(mmu_pending_pte_updates)) + panic("Too many pending MMU PTE updates before publication\n"); + mmu_pending_pte_updates[mmu_pending_pte_update_count++] = + (struct mmu_pending_pte_update){slot, value}; +} + +static void mmu_pt_install_table(u64 from, u64 *parent, u64 parent_idx, u64 *child, + u64 block_offset_bits, bool needs_break) +{ + u64 descriptor = (u64)child | FIELD_PREP(PTE_TYPE, PTE_TABLE) | PTE_VALID; + + if (needs_break) { + u64 *slot = &parent[parent_idx]; + u64 *make_slot = slot; + + // Use a WB alias if the parent lives inside the block being split. + if (((u64)slot & ~MASK(block_offset_bits)) == + (from & ~MASK(block_offset_bits))) + make_slot = (u64 *)((u64)slot | REGION_RW_EL0); + + // Keep asynchronous handlers out of the identity-map break. + u64 daif = mrs(DAIF); + msr(DAIF, daif | (7UL << 6)); + sysop("isb"); + + mmu_pt_queue_update(slot, 0); + mmu_publish_mappings(); + mmu_pt_queue_update(make_slot, descriptor); + mmu_publish_mappings(); + + msr(DAIF, daif); + sysop("isb"); + return; + } + + mmu_pt_queue_update(&parent[parent_idx], descriptor); +} + static u64 *mmu_pt_get_l1(u64 from) { u64 l0idx = from >> VADDR_L0_OFFSET_BITS; assert(l0idx < ENTRIES_PER_L0_TABLE); - u64 l0d = mmu_pt_L0[l0idx]; + u64 l0d; + mmu_pt_pending_value(&mmu_pt_L0[l0idx], &l0d); if (L0_IS_TABLE(l0d)) return (u64 *)(l0d & PTE_TARGET_MASK); @@ -188,13 +258,15 @@ static u64 *mmu_pt_get_l1(u64 from) assert(!IS_PTE(l0d)); memset64(l1, 0, ENTRIES_PER_L1_TABLE * sizeof(u64)); - l0d = ((u64)l1) | FIELD_PREP(PTE_TYPE, PTE_TABLE) | PTE_VALID; - mmu_pt_L0[l0idx] = l0d; + mmu_pt_install_table(from, mmu_pt_L0, l0idx, l1, VADDR_L0_OFFSET_BITS, false); + if (mmu_active()) + mmu_publish_mappings(); return l1; } static void mmu_pt_map_l1(u64 from, u64 to, u64 size) { + assert(!mmu_active()); assert((from & MASK(VADDR_L1_OFFSET_BITS)) == 0); assert((to & PTE_TARGET_MASK & MASK(VADDR_L1_OFFSET_BITS)) == 0); assert((size & MASK(VADDR_L1_OFFSET_BITS)) == 0); @@ -219,7 +291,8 @@ static u64 *mmu_pt_get_l2(u64 from) u64 *l1 = mmu_pt_get_l1(from); u64 l1idx = (from >> VADDR_L1_OFFSET_BITS) & MASK(VADDR_L1_INDEX_BITS); assert(l1idx < ENTRIES_PER_L1_TABLE); - u64 l1d = l1[l1idx]; + u64 l1d; + mmu_pt_pending_value(&l1[l1idx], &l1d); if (L1_IS_TABLE(l1d)) return (u64 *)(l1d & PTE_TARGET_MASK); @@ -235,8 +308,10 @@ static u64 *mmu_pt_get_l2(u64 from) memset64(l2, 0, ENTRIES_PER_L2_TABLE * sizeof(u64)); } - l1d = ((u64)l2) | FIELD_PREP(PTE_TYPE, PTE_TABLE) | PTE_VALID; - l1[l1idx] = l1d; + bool needs_break = mmu_active() && IS_PTE(l1d); + mmu_pt_install_table(from, l1, l1idx, l2, VADDR_L1_OFFSET_BITS, needs_break); + if (mmu_active() && !needs_break) + mmu_publish_mappings(); return l2; } @@ -252,41 +327,17 @@ static void mmu_pt_map_l2(u64 from, u64 to, u64 size) u64 idx = (from >> VADDR_L2_OFFSET_BITS) & MASK(VADDR_L2_INDEX_BITS); u64 *l2 = mmu_pt_get_l2(from); - if (L2_IS_TABLE(l2[idx])) - free((void *)(l2[idx] & PTE_TARGET_MASK)); + u64 old; + mmu_pt_pending_value(&l2[idx], &old); + if (L2_IS_TABLE(old) && !mmu_active()) + free((void *)(old & PTE_TARGET_MASK)); - l2[idx] = to; + mmu_pt_queue_update(&l2[idx], to); from += BIT(VADDR_L2_OFFSET_BITS); to += BIT(VADDR_L2_OFFSET_BITS); } } -static u64 *mmu_pt_get_l3(u64 from) -{ - u64 *l2 = mmu_pt_get_l2(from); - u64 l2idx = (from >> VADDR_L2_OFFSET_BITS) & MASK(VADDR_L2_INDEX_BITS); - assert(l2idx < ENTRIES_PER_L2_TABLE); - u64 l2d = l2[l2idx]; - - if (L2_IS_TABLE(l2d)) - return (u64 *)(l2d & PTE_TARGET_MASK); - - u64 *l3 = (u64 *)memalign(PAGE_SIZE, ENTRIES_PER_L3_TABLE * sizeof(u64)); - if (IS_PTE(l2d)) { - u64 l3d = l2d; - l3d &= ~PTE_TYPE; - l3d |= FIELD_PREP(PTE_TYPE, PTE_PAGE); - for (u64 idx = 0; idx < ENTRIES_PER_L3_TABLE; idx++, l3d += BIT(VADDR_L3_OFFSET_BITS)) - l3[idx] = l3d; - } else { - memset64(l3, 0, ENTRIES_PER_L3_TABLE * sizeof(u64)); - } - - l2d = ((u64)l3) | FIELD_PREP(PTE_TYPE, PTE_TABLE) | PTE_VALID; - l2[l2idx] = l2d; - return l3; -} - static void mmu_pt_map_l3(u64 from, u64 to, u64 size) { assert((from & MASK(VADDR_L3_OFFSET_BITS)) == 0); @@ -295,13 +346,60 @@ static void mmu_pt_map_l3(u64 from, u64 to, u64 size) to |= FIELD_PREP(PTE_TYPE, PTE_PAGE); - for (; size; size -= BIT(VADDR_L3_OFFSET_BITS)) { - u64 idx = (from >> VADDR_L3_OFFSET_BITS) & MASK(VADDR_L3_INDEX_BITS); - u64 *l3 = mmu_pt_get_l3(from); + while (size) { + u64 remaining = BIT(VADDR_L2_OFFSET_BITS) - + (from & MASK(VADDR_L2_OFFSET_BITS)); + u64 chunk = min(size, remaining); - l3[idx] = to; - from += BIT(VADDR_L3_OFFSET_BITS); - to += BIT(VADDR_L3_OFFSET_BITS); + u64 *l2 = mmu_pt_get_l2(from); + u64 l2idx = (from >> VADDR_L2_OFFSET_BITS) & MASK(VADDR_L2_INDEX_BITS); + assert(l2idx < ENTRIES_PER_L2_TABLE); + u64 old; + bool pending = mmu_pt_pending_value(&l2[l2idx], &old); + bool new_table = !L2_IS_TABLE(old); + bool write_directly = !mmu_active() || pending || new_table; + u64 *l3; + + // Find the existing table, or initialize a replacement. + if (!new_table) { + l3 = (u64 *)(old & PTE_TARGET_MASK); + } else { + l3 = memalign(PAGE_SIZE, ENTRIES_PER_L3_TABLE * sizeof(u64)); + + if (IS_PTE(old)) { + u64 entry = (old & ~PTE_TYPE) | FIELD_PREP(PTE_TYPE, PTE_PAGE); + for (u64 i = 0; i < ENTRIES_PER_L3_TABLE; i++) { + l3[i] = entry; + entry += BIT(VADDR_L3_OFFSET_BITS); + } + } else { + memset64(l3, 0, ENTRIES_PER_L3_TABLE * sizeof(u64)); + } + } + + // Edit hidden tables directly; queue edits to live tables. + for (u64 offset = 0; offset < chunk; offset += BIT(VADDR_L3_OFFSET_BITS)) { + u64 idx = ((from + offset) >> VADDR_L3_OFFSET_BITS) & + MASK(VADDR_L3_INDEX_BITS); + + // Invalid entries don't need a physical address. + u64 value = (to & PTE_VALID) ? to + offset : to; + if (write_directly) + l3[idx] = value; + else + mmu_pt_queue_update(&l3[idx], value); + } + + // Install the new table only after finishing its entries. + if (new_table) { + mmu_pt_install_table(from, l2, l2idx, l3, VADDR_L2_OFFSET_BITS, + mmu_active() && IS_PTE(old)); + } + + from += chunk; + if (to & PTE_VALID) + to += chunk; + size -= chunk; } } @@ -327,7 +425,7 @@ int mmu_map(u64 from, u64 to, u64 size) } // 16K does not support L1 blocks without FEAT_LPA2 - if (!is_16k()) { + if (!mmu_active() && !is_16k()) { // Map L2 until L1-aligned or reached end of mapping u64 boundary_l1 = ALIGN_UP(from, MASK(VADDR_L1_OFFSET_BITS)); chunk = min(ALIGN_DOWN(size, MASK(VADDR_L1_OFFSET_BITS)), boundary_l1 - from); @@ -349,12 +447,14 @@ int mmu_map(u64 from, u64 to, u64 size) } // L2 mappings - chunk = ALIGN_DOWN(size, MASK(VADDR_L2_OFFSET_BITS)); - if (chunk && (to & VADDR_L2_ALIGN_MASK) == 0) { - mmu_pt_map_l2(from, to, chunk); - from += chunk; - to += chunk; - size -= chunk; + if (!mmu_active()) { + chunk = ALIGN_DOWN(size, MASK(VADDR_L2_OFFSET_BITS)); + if (chunk && (to & VADDR_L2_ALIGN_MASK) == 0) { + mmu_pt_map_l2(from, to, chunk); + from += chunk; + to += chunk; + size -= chunk; + } } // L3 mappings to end @@ -372,25 +472,197 @@ static void mmu_init_pagetables(void) return; } -void mmu_add_mapping(u64 from, u64 to, size_t size, u8 attribute_index, u64 perms) +void mmu_add_mapping(u64 from, u64 to, size_t size, u8 attribute_index, + u64 perms) { if (mmu_map(from, - to | PTE_MAIR_IDX(attribute_index) | PTE_ACCESS | PTE_VALID | PTE_SH_OS | perms, + to | PTE_MAIR_IDX(attribute_index) | PTE_ACCESS | PTE_VALID | + PTE_SH_OS | perms, size) < 0) panic("Failed to add MMU mapping 0x%lx -> 0x%lx (0x%lx)\n", from, to, size); + mmu_publish_mappings(); +} + +static void mmu_publish_mappings(void) +{ + if (mmu_pending_pte_update_count) { + // Fill new child tables completely before publishing their parent. + for (size_t index = 0; index < mmu_pending_pte_update_count; index++) { + struct mmu_pending_pte_update *update = &mmu_pending_pte_updates[index]; + *update->slot = update->value; + } + mmu_pending_pte_update_count = 0; + } + sysop("dsb ishst"); sysop("tlbi vmalle1is"); sysop("dsb ish"); sysop("isb"); } +void mmu_retype_mapping(u64 from, u64 to, size_t size, u8 attribute_index, + u64 perms) +{ + if (mmu_map(from, 0, size) < 0) + panic("Failed to break MMU mapping at 0x%lx (0x%lx)\n", from, size); + mmu_publish_mappings(); + + if (mmu_map(from, + to | PTE_MAIR_IDX(attribute_index) | PTE_ACCESS | PTE_VALID | + PTE_SH_OS | perms, + size) < 0) + panic("Failed to remake MMU mapping 0x%lx -> 0x%lx (0x%lx)\n", + from, to, size); + mmu_publish_mappings(); +} + void mmu_rm_mapping(u64 from, size_t size) { if (mmu_map(from, 0, size) < 0) panic("Failed to rm MMU mapping at 0x%lx (0x%lx)\n", from, size); } +static size_t mmu_sort_unique_pages(u64 *pages, size_t count) +{ + for (size_t index = 1; index < count; index++) { + u64 value = pages[index]; + size_t insert = index; + while (insert && pages[insert - 1] > value) { + pages[insert] = pages[insert - 1]; + insert--; + } + pages[insert] = value; + } + + size_t unique = 0; + for (size_t index = 0; index < count; index++) { + if (!unique || pages[index] != pages[unique - 1]) + pages[unique++] = pages[index]; + } + return unique; +} + +/* + * (Un)map [pa, pa + size) at m1n1's four ordinary RAM aliases with one memory + * type. Live IOMMU tables also get a dedicated Normal-NC alias that is never WB. + */ +static void mmu_map_ram_aliases(u64 pa, u64 size, u8 attribute_index, bool valid, bool dedicated) +{ + static const u64 aliases[][2] = { + {0, PERM_RWX}, + {REGION_RWX_EL0, PERM_RWX_EL0}, + {REGION_RW_EL0, PERM_RW_EL0}, + {REGION_RX_EL1, PERM_RX_EL0}, + }; + u64 attributes = PTE_MAIR_IDX(attribute_index) | PTE_ACCESS | PTE_VALID | + (attribute_index == MAIR_IDX_NORMAL_NC ? PTE_SH_NS : PTE_SH_OS); + + for (size_t index = 0; index < ARRAY_SIZE(aliases); index++) + mmu_pt_map_l3(pa | aliases[index][0], valid ? pa | attributes | aliases[index][1] : 0, + size); + if (dedicated) + mmu_pt_map_l3(pa | REGION_NORMAL_NC, valid ? pa | attributes | PERM_RW : 0, size); +} + +// Apply mmu_map_ram_aliases() to each contiguous run of a sorted page list. +static void mmu_map_ram_page_aliases(u64 *pages, size_t count, u8 attribute_index, bool valid, + bool dedicated) +{ + for (size_t first = 0, end; first < count; first = end) { + for (end = first + 1; end < count && pages[end] == pages[end - 1] + PAGE_SIZE; end++) + ; + mmu_map_ram_aliases(pages[first], (end - first) * PAGE_SIZE, attribute_index, valid, + dedicated); + } +} + +static u64 mmu_get_mapping(u64 va) +{ + u64 l0idx = va >> VADDR_L0_OFFSET_BITS; + if (l0idx >= ENTRIES_PER_L0_TABLE) + return 0; + + u64 l0d; + mmu_pt_pending_value(&mmu_pt_L0[l0idx], &l0d); + if (!L0_IS_TABLE(l0d)) + return l0d; + + u64 *l1 = (u64 *)(l0d & PTE_TARGET_MASK); + u64 l1idx = (va >> VADDR_L1_OFFSET_BITS) & MASK(VADDR_L1_INDEX_BITS); + u64 l1d; + mmu_pt_pending_value(&l1[l1idx], &l1d); + if (!L1_IS_TABLE(l1d)) + return l1d; + + u64 *l2 = (u64 *)(l1d & PTE_TARGET_MASK); + u64 l2idx = (va >> VADDR_L2_OFFSET_BITS) & MASK(VADDR_L2_INDEX_BITS); + u64 l2d; + mmu_pt_pending_value(&l2[l2idx], &l2d); + if (!L2_IS_TABLE(l2d)) + return l2d; + + u64 *l3 = (u64 *)(l2d & PTE_TARGET_MASK); + u64 l3d; + mmu_pt_pending_value(&l3[(va >> VADDR_L3_OFFSET_BITS) & MASK(VADDR_L3_INDEX_BITS)], &l3d); + return l3d; +} + +static bool mmu_ram_page_has_type(u64 pa, u8 attribute_index) +{ + const u64 aliases[] = {pa, pa | REGION_RWX_EL0, pa | REGION_RW_EL0, pa | REGION_RX_EL1}; + u64 shareability = attribute_index == MAIR_IDX_NORMAL_NC ? PTE_SH_NS : PTE_SH_OS; + + for (size_t index = 0; index < ARRAY_SIZE(aliases); index++) { + u64 descriptor = mmu_get_mapping(aliases[index]); + if (!(descriptor & PTE_VALID) || + (descriptor & PTE_MAIR_IDX(7)) != PTE_MAIR_IDX(attribute_index) || + (descriptor & (3ULL << 8)) != shareability) + return false; + } + return true; +} + +/* + * Switch RAM pages between Normal-WB and Normal-NC at all of m1n1's aliases, + * skipping pages already in the requested state. Arm requires the old aliases + * to be broken and invalidated before the new memory type is mapped. + */ +static void mmu_retype_ram_pages(u64 *pages, size_t count, u8 attribute_index, bool dedicated) +{ + size_t pending = 0; + for (size_t index = 0; index < count; index++) { + if (!mmu_ram_page_has_type(pages[index], attribute_index)) + pages[pending++] = pages[index]; + } + count = mmu_sort_unique_pages(pages, pending); + if (!count) + return; + + // Retire cacheable state before changing any of m1n1's aliases. + if (attribute_index == MAIR_IDX_NORMAL_NC) { + for (size_t index = 0; index < count; index++) + dc_civac_range((void *)pages[index], PAGE_SIZE); + sysop("dsb sy"); + } + + mmu_map_ram_page_aliases(pages, count, 0, false, dedicated); + mmu_publish_mappings(); + mmu_map_ram_page_aliases(pages, count, attribute_index, true, + dedicated && attribute_index == MAIR_IDX_NORMAL_NC); + mmu_publish_mappings(); +} + +void mmu_map_ram_pages_nc(u64 *pages, size_t count, bool dedicated_alias) +{ + mmu_retype_ram_pages(pages, count, MAIR_IDX_NORMAL_NC, dedicated_alias); +} + +void mmu_restore_ram_pages_wb(u64 *pages, size_t count, bool dedicated_alias) +{ + mmu_retype_ram_pages(pages, count, MAIR_IDX_NORMAL, dedicated_alias); +} + static void mmu_map_mmio(void) { int node = adt_path_offset(adt, "/arm-io"); @@ -453,100 +725,18 @@ static void mmu_remap_ranges(void) } void mmu_map_framebuffer(u64 addr, size_t size) -{ - printf("MMU: Adding Normal-NC mapping at 0x%lx (0x%zx) for framebuffer\n", addr, size); - dc_civac_range((void *)addr, size); - mmu_add_mapping(addr, addr, size, MAIR_IDX_NORMAL_NC, PERM_RW_EL0); -} - -int display_get_vram(u64 *paddr, u64 *size); - -static int mmu_get_iboot_handoff_range(u64 *start, u64 *size) -{ - u64 range[2]; - int chosen = adt_path_offset(adt, "/chosen"); - - if (chosen >= 0 && - ADT_GETPROP_ARRAY(adt, chosen, "iboot-handoff", range) == sizeof(range)) - goto found; - - int carveouts = adt_path_offset(adt, "/chosen/carveout-memory-map"); - if (carveouts >= 0 && - ADT_GETPROP_ARRAY(adt, carveouts, "region-id-1", range) == sizeof(range)) - goto found; - - return -1; - -found: - if (!range[0] || !range[1]) - return -1; - - *start = ALIGN_DOWN(range[0], PAGE_SIZE); - *size = ALIGN_UP(range[0] + range[1], PAGE_SIZE) - *start; - return 0; -} - -static void mmu_add_ram_mapping_handoff_nc(u64 from, u64 to, size_t size, - u8 attribute_index, u64 perms, - u64 handoff_start, - u64 handoff_size) -{ - u64 end = to + size; - u64 handoff_end = handoff_start + handoff_size; - - if (!handoff_start || !handoff_size || handoff_end <= to || - handoff_start >= end) { - mmu_add_mapping(from, to, size, attribute_index, perms); - return; - } - - u64 cur_to = to; - if (handoff_start > cur_to) { - u64 before_size = handoff_start - cur_to; - mmu_add_mapping(from, cur_to, before_size, attribute_index, perms); - from += before_size; - cur_to += before_size; - } - - u64 nc_end = min(handoff_end, end); - u64 nc_size = nc_end - cur_to; - mmu_add_mapping(from, cur_to, nc_size, MAIR_IDX_NORMAL_NC, perms); - from += nc_size; - cur_to += nc_size; - - if (cur_to < end) - mmu_add_mapping(from, cur_to, end - cur_to, attribute_index, perms); -} - -static void mmu_map_nc_aliases(u64 addr, size_t size, const char *name) { u64 start = ALIGN_DOWN(addr, PAGE_SIZE); - u64 end = ALIGN_UP(addr + size, PAGE_SIZE); - size_t map_size = end - start; - - if (!addr || !size) - return; - - printf("MMU: Adding Normal-NC mappings at 0x%lx..0x%lx for %s\n", - start, end, name); - mmu_add_mapping(start, start, map_size, MAIR_IDX_NORMAL_NC, PERM_RWX); - mmu_add_mapping(start | REGION_RWX_EL0, start, map_size, - MAIR_IDX_NORMAL_NC, PERM_RWX_EL0); - mmu_add_mapping(start | REGION_RW_EL0, start, map_size, - MAIR_IDX_NORMAL_NC, PERM_RW_EL0); - mmu_add_mapping(start | REGION_RX_EL1, start, map_size, - MAIR_IDX_NORMAL_NC, PERM_RX_EL0); -} - -static void mmu_map_iboot_handoff_nc(void) -{ - u64 start, size; - - if (mmu_get_iboot_handoff_range(&start, &size) < 0) - return; - - mmu_map_nc_aliases(start, size, "/chosen/iboot-handoff"); + u64 map_size = ALIGN_UP(addr + size, PAGE_SIZE) - start; + printf("MMU: Adding Normal-NC mappings at 0x%lx..0x%lx for framebuffer\n", start, + start + map_size); + dc_civac_range((void *)addr, size); + sysop("dsb sy"); + mmu_map_ram_aliases(start, map_size, 0, false, false); + mmu_publish_mappings(); + mmu_map_ram_aliases(start, map_size, MAIR_IDX_NORMAL_NC, true, false); + mmu_publish_mappings(); } static void mmu_add_default_mappings(void) @@ -554,8 +744,6 @@ static void mmu_add_default_mappings(void) ram_base = ALIGN_DOWN(cur_boot_args.phys_base, BIT(32)); uint64_t ram_size = cur_boot_args.mem_size + cur_boot_args.phys_base - ram_base; ram_size = ALIGN_DOWN(ram_size, PAGE_SIZE); - u64 handoff_start = 0, handoff_size = 0; - mmu_get_iboot_handoff_range(&handoff_start, &handoff_size); printf("MMU: RAM base: 0x%lx\n", ram_base); printf("MMU: Top of normal RAM: 0x%lx\n", ram_base + ram_size); @@ -567,25 +755,11 @@ static void mmu_add_default_mappings(void) * With SPRR enabled, this becomes RW. * This range includes all real RAM, including carveouts */ - mmu_add_ram_mapping_handoff_nc(ram_base, ram_base, mem_size_actual, - MAIR_IDX_NORMAL, PERM_RWX, - handoff_start, handoff_size); + mmu_add_mapping(ram_base, ram_base, mem_size_actual, MAIR_IDX_NORMAL, PERM_RWX); /* Unmap carveout regions */ mcc_unmap_carveouts(); - /* - * Re-map the boot framebuffer (/vram) Normal-NC. The Normal-WB identity map - * lets m1n1/iBoot cacheable accesses allocate AMCC SLC directory tags - * for /vram; after handoff XNU's non-coherent framebuffer writes hit those - * stale tags and the AMCC raises UNEXP_RT_HIT_DIR -> PEH panic. - */ - { - u64 fb_pa = 0, fb_size = 0; - if (display_get_vram(&fb_pa, &fb_size) == 0 && fb_pa && fb_size) - mmu_map_framebuffer(fb_pa, fb_size); - } - /* * Remap m1n1 executable code as RX. */ @@ -602,32 +776,22 @@ static void mmu_add_default_mappings(void) * read/writable/exec by EL0 (but not executable by EL1) * With SPRR enabled, this becomes RX_EL0. */ - mmu_add_ram_mapping_handoff_nc(ram_base | REGION_RWX_EL0, ram_base, - ram_size, MAIR_IDX_NORMAL, PERM_RWX_EL0, - handoff_start, handoff_size); + mmu_add_mapping(ram_base | REGION_RWX_EL0, ram_base, ram_size, + MAIR_IDX_NORMAL, PERM_RWX_EL0); /* * Create mapping for RAM from 0x98_0000_0000, * read/writable by EL0 (but not executable by EL1) * With SPRR enabled, this becomes RW_EL0. */ - mmu_add_ram_mapping_handoff_nc(ram_base | REGION_RW_EL0, ram_base, - ram_size, MAIR_IDX_NORMAL, PERM_RW_EL0, - handoff_start, handoff_size); + mmu_add_mapping(ram_base | REGION_RW_EL0, ram_base, ram_size, + MAIR_IDX_NORMAL, PERM_RW_EL0); /* * Create mapping for RAM from 0xa8_0000_0000, * read/executable by EL1 * This allows executing from dynamic regions in EL1 */ - mmu_add_ram_mapping_handoff_nc(ram_base | REGION_RX_EL1, ram_base, - ram_size, MAIR_IDX_NORMAL, PERM_RX_EL0, - handoff_start, handoff_size); - - /* - * The iBoot handoff page is a protected low carveout on T8140 - * (/chosen/iboot-handoff, region-id-1). Keep m1n1's aliases uncached so - * we do not add any new WB state for this page. - */ - mmu_map_iboot_handoff_nc(); + mmu_add_mapping(ram_base | REGION_RX_EL1, ram_base, ram_size, + MAIR_IDX_NORMAL, PERM_RX_EL0); /* * Create four separate full mappings of MMIO space, with different access types diff --git a/src/memory.h b/src/memory.h index b4f231d0..0df5db0b 100644 --- a/src/memory.h +++ b/src/memory.h @@ -9,6 +9,7 @@ #define REGION_RWX_EL0 0x80000000000 #define REGION_RW_EL0 0xa0000000000 #define REGION_RX_EL1 0xc0000000000 +#define REGION_NORMAL_NC 0xe0000000000 /* * https://armv8-ref.codingbelief.com/en/chapter_d4/d43_2_armv8_translation_table_level_3_descriptor_formats.html @@ -72,8 +73,11 @@ void mmu_init(void); void mmu_init_secondary(int cpu); void mmu_shutdown(void); void mmu_add_mapping(u64 from, u64 to, size_t size, u8 attribute_index, u64 perms); +void mmu_retype_mapping(u64 from, u64 to, size_t size, u8 attribute_index, u64 perms); void mmu_rm_mapping(u64 from, size_t size); void mmu_map_framebuffer(u64 addr, size_t size); +void mmu_map_ram_pages_nc(u64 *pages, size_t count, bool dedicated_alias); +void mmu_restore_ram_pages_wb(u64 *pages, size_t count, bool dedicated_alias); u64 mmu_disable(void); void mmu_restore(u64 state); diff --git a/src/proxy.c b/src/proxy.c index ed29f364..9a8c4f39 100644 --- a/src/proxy.c +++ b/src/proxy.c @@ -10,6 +10,7 @@ #include "gxf.h" #include "heapblock.h" #include "hv.h" +#include "hv_sptm.h" #include "iodev.h" #include "kboot.h" #include "malloc.h" @@ -543,7 +544,10 @@ int proxy_process(ProxyRequest *request, ProxyReply *reply) case P_HV_ADD_TIME: hv_add_time(request->args[0]); break; - + case P_HV_SPTM_INIT: + reply->retval = hv_sptm_init(request->args[0], request->args[1], request->args[2], + request->args[3]); + break; case P_FB_INIT: fb_init(request->args[0]); break; diff --git a/src/proxy.h b/src/proxy.h index 7ed5d1b5..1f0ac4d2 100644 --- a/src/proxy.h +++ b/src/proxy.h @@ -146,6 +146,7 @@ typedef enum { P_VIRTIO_PUT_BUFFER, P_HV_EXIT_CPU, P_HV_ADD_TIME, + P_HV_SPTM_INIT, P_FB_INIT = 0xd00, P_FB_SHUTDOWN, diff --git a/src/sapt.c b/src/sapt.c new file mode 100644 index 00000000..6c4c0a73 --- /dev/null +++ b/src/sapt.c @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include "sapt.h" +#include "adt.h" +#include "string.h" +#include "types.h" +#include "utils.h" + +#define SAPT_PAGES_PER_BYTE 4 + +void sapt_disable(void) +{ + if (!cpu_features->sapt) + return; + + int node = adt_path_offset(adt, "/arm-io/sapt"); + u64 table_address; + u64 entries; + + if (node < 0 || + ADT_GETPROP(adt, node, "table-address", &table_address) != sizeof(table_address) || + ADT_GETPROP(adt, node, "n-entries", &entries) != sizeof(entries)) { + printf("SAPT: Configuration not found\n"); + return; + } + + u64 expected_entries = mem_size_actual / SZ_16K; + if (entries != expected_entries) { + printf("SAPT: Invalid entry count 0x%lx (expected 0x%lx)\n", entries, expected_entries); + return; + } + + size_t table_size = (entries + SAPT_PAGES_PER_BYTE - 1) / SAPT_PAGES_PER_BYTE; + printf("SAPT: Disabling protection at 0x%lx (0x%zx bytes)\n", table_address, table_size); + memset((void *)table_address, 0, table_size); + sysop("dsb sy"); +} diff --git a/src/sapt.h b/src/sapt.h new file mode 100644 index 00000000..a8c03754 --- /dev/null +++ b/src/sapt.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __SAPT_H__ +#define __SAPT_H__ + +void sapt_disable(void); + +#endif diff --git a/src/utils.h b/src/utils.h index 8a0c453a..dd60f4a6 100644 --- a/src/utils.h +++ b/src/utils.h @@ -319,6 +319,7 @@ static inline void write64_lo_hi(u64 addr, u64 val) #define dc_csw(p) cacheop("dc csw", p) #define dc_cisw(p) cacheop("dc cisw", p) #define dc_zva(p) cacheop("dc zva", p) +/* WARNING: This is a noop on Apple Silicon. */ #define dc_cvac(p) cacheop("dc cvac", p) #define dc_cvau(p) cacheop("dc cvau", p) #define dc_civac(p) cacheop("dc civac", p) @@ -511,6 +512,7 @@ struct midr_part_features { bool actlr_el2; bool counter_redirect; bool sme_enabled; + bool sapt; }; extern bool is_mac;