diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index e0d07c4ec7f09..7bcc33bddb6a6 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1805,16 +1805,70 @@ static void arm_smmu_init_initial_stes(struct arm_smmu_ste *strtab, } } +static int arm_smmu_kdump_adopt_l2_strtab(struct arm_smmu_device *smmu, u32 sid, + phys_addr_t base, u32 span, + struct arm_smmu_strtab_l2 **l2table) +{ + struct arm_smmu_strtab_l2 *table; + size_t size; + + /* + * Only a coherent SMMU is supported at this moment. For a non-coherent + * SMMU that wants to support ARM_SMMU_OPT_KDUMP_ADOPT, try MEMREMAP_WC. + */ + if (WARN_ON(!(smmu->features & ARM_SMMU_FEAT_COHERENCY))) + return -EOPNOTSUPP; + + /* + * Retest the span in case the L1 descriptor has been overwritten since + * the adopt. Reject this master's insert; panic or SMMU-disable would + * either lose the vmcore or cascade aborts. Do not try to fix it, as it + * would break all other SIDs in the same bus (PCI case). The corruption + * blast radius is already bounded to that bus range. + */ + if (span != STRTAB_SPLIT + 1) { + dev_err(smmu->dev, + "kdump: L1[%u] span %u changed since adopt (was %u)\n", + arm_smmu_strtab_l1_idx(sid), span, STRTAB_SPLIT + 1); + return -EINVAL; + } + + size = (1UL << (span - 1)) * sizeof(struct arm_smmu_ste); + + table = devm_memremap(smmu->dev, base, size, MEMREMAP_WB); + if (IS_ERR(table)) { + dev_err(smmu->dev, + "kdump: failed to adopt l2 stream table for SID %u\n", + sid); + return PTR_ERR(table); + } + + *l2table = table; + return 0; +} + static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) { dma_addr_t l2ptr_dma; struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; struct arm_smmu_strtab_l2 **l2table; + u32 l1_idx = arm_smmu_strtab_l1_idx(sid); - l2table = &cfg->l2.l2ptrs[arm_smmu_strtab_l1_idx(sid)]; + l2table = &cfg->l2.l2ptrs[l1_idx]; if (*l2table) return 0; + /* Deferred adoption of the crashed kernel's L2 table */ + if (smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT) { + u64 l2ptr = le64_to_cpu(cfg->l2.l1tab[l1_idx].l2ptr); + phys_addr_t base = l2ptr & STRTAB_L1_DESC_L2PTR_MASK; + u32 span = FIELD_GET(STRTAB_L1_DESC_SPAN, l2ptr); + + if (span && base) + return arm_smmu_kdump_adopt_l2_strtab(smmu, sid, base, + span, l2table); + } + *l2table = dmam_alloc_coherent(smmu->dev, sizeof(**l2table), &l2ptr_dma, GFP_KERNEL); if (!*l2table) { @@ -1826,8 +1880,7 @@ static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) arm_smmu_init_initial_stes((*l2table)->stes, ARRAY_SIZE((*l2table)->stes)); - arm_smmu_write_strtab_l1_desc(&cfg->l2.l1tab[arm_smmu_strtab_l1_idx(sid)], - l2ptr_dma); + arm_smmu_write_strtab_l1_desc(&cfg->l2.l1tab[l1_idx], l2ptr_dma); return 0; } @@ -2176,7 +2229,11 @@ static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev) static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev) { - arm_smmu_gerror_handler(irq, dev); + irqreturn_t ret = arm_smmu_gerror_handler(irq, dev); + + /* In kdump, EVTQ/PRIQ are disabled and there is no thread to wake */ + if (is_kdump_kernel()) + return ret; return IRQ_WAKE_THREAD; } @@ -3605,6 +3662,29 @@ static void arm_smmu_remove_master(struct arm_smmu_master *master) kfree(master->streams); } +static bool arm_smmu_is_attach_deferred(struct device *dev) +{ + struct arm_smmu_master *master = dev_iommu_priv_get(dev); + struct arm_smmu_device *smmu = master->smmu; + int i; + + if (!(smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT)) + return false; + + for (i = 0; i < master->num_streams; i++) { + struct arm_smmu_ste *ste = + arm_smmu_get_step_for_sid(smmu, master->streams[i].id); + u64 ent0 = le64_to_cpu(ste->data[0]); + + /* Defer only when there might be in-flight DMAs */ + if ((ent0 & STRTAB_STE_0_V) && + FIELD_GET(STRTAB_STE_0_CFG, ent0) != STRTAB_STE_0_CFG_ABORT) + return true; + } + + return false; +} + static int arm_smmu_master_prepare_ats(struct arm_smmu_master *master) { bool s1p = master->smmu->features & ARM_SMMU_FEAT_TRANS_S1; @@ -3806,6 +3886,7 @@ static const struct iommu_ops arm_smmu_ops = { .hw_info = arm_smmu_hw_info, .domain_alloc_sva = arm_smmu_sva_domain_alloc, .domain_alloc_paging_flags = arm_smmu_domain_alloc_paging_flags, + .is_attach_deferred = arm_smmu_is_attach_deferred, .probe_device = arm_smmu_probe_device, .release_device = arm_smmu_release_device, .device_group = arm_smmu_device_group, @@ -3985,10 +4066,204 @@ static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) return 0; } +static int arm_smmu_kdump_adopt_strtab_2lvl(struct arm_smmu_device *smmu, + u32 cfg_reg, phys_addr_t base) +{ + u32 log2size = FIELD_GET(STRTAB_BASE_CFG_LOG2SIZE, cfg_reg); + u32 split = FIELD_GET(STRTAB_BASE_CFG_SPLIT, cfg_reg); + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + u32 num_l1_ents; + size_t size; + int i; + + /* + * Only a coherent SMMU is supported at this moment. For a non-coherent + * SMMU that wants to support ARM_SMMU_OPT_KDUMP_ADOPT, try MEMREMAP_WC. + */ + if (WARN_ON(!(smmu->features & ARM_SMMU_FEAT_COHERENCY))) + return -EOPNOTSUPP; + + if (log2size < split || log2size > smmu->sid_bits) { + dev_err(smmu->dev, "kdump: log2size %u out of range [%u, %u]\n", + log2size, split, smmu->sid_bits); + return -EINVAL; + } + if (split != STRTAB_SPLIT) { + dev_err(smmu->dev, + "kdump: unsupported STRTAB_SPLIT %u (expected %u)\n", + split, STRTAB_SPLIT); + return -EINVAL; + } + + num_l1_ents = 1U << (log2size - split); + if (num_l1_ents > STRTAB_MAX_L1_ENTRIES) { + dev_err(smmu->dev, "kdump: l1 entries %u exceeds max %u\n", + num_l1_ents, STRTAB_MAX_L1_ENTRIES); + return -EINVAL; + } + + cfg->l2.num_l1_ents = num_l1_ents; + + size = num_l1_ents * sizeof(struct arm_smmu_strtab_l1); + cfg->l2.l1tab = memremap(base, size, MEMREMAP_WB); + if (!cfg->l2.l1tab) + return -ENOMEM; + + cfg->l2.l2ptrs = + kcalloc(num_l1_ents, sizeof(*cfg->l2.l2ptrs), GFP_KERNEL); + if (!cfg->l2.l2ptrs) + return -ENOMEM; + + for (i = 0; i < num_l1_ents; i++) { + u64 l2ptr = le64_to_cpu(cfg->l2.l1tab[i].l2ptr); + phys_addr_t l2_base = l2ptr & STRTAB_L1_DESC_L2PTR_MASK; + u32 span = FIELD_GET(STRTAB_L1_DESC_SPAN, l2ptr); + + if (!span || !l2_base) + continue; + + if (span != STRTAB_SPLIT + 1) { + dev_err(smmu->dev, + "kdump: L1[%u] unsupported span %u (vs %u)\n", + i, span, STRTAB_SPLIT + 1); + return -EINVAL; + } + + /* + * If the crashed kernel's l1 descriptors are deeply corrupted, + * blindly memremapping every l2 table here could lead to OOM. + * + * Defer the l2 memremap to arm_smmu_init_l2_strtab(), so peak + * memory is bounded by the kdump kernel's actual demand. + */ + } + + return 0; +} + +static int arm_smmu_kdump_adopt_strtab_linear(struct arm_smmu_device *smmu, + u32 cfg_reg, phys_addr_t base) +{ + u32 log2size = FIELD_GET(STRTAB_BASE_CFG_LOG2SIZE, cfg_reg); + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + unsigned int max_log2size; + size_t size; + + /* + * Only a coherent SMMU is supported at this moment. For a non-coherent + * SMMU that wants to support ARM_SMMU_OPT_KDUMP_ADOPT, try MEMREMAP_WC. + */ + if (WARN_ON(!(smmu->features & ARM_SMMU_FEAT_COHERENCY))) + return -EOPNOTSUPP; + + /* Cap the size at what the kdump kernel itself would have allocated */ + if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) + max_log2size = + ilog2(STRTAB_MAX_L1_ENTRIES * STRTAB_NUM_L2_STES); + else + max_log2size = smmu->sid_bits; + + /* cfg->linear.num_ents is unsigned int, so cap log2size at 31 */ + max_log2size = min(max_log2size, 31U); + if (log2size > max_log2size) { + dev_err(smmu->dev, "kdump: unsupported log2size %u (> %u)\n", + log2size, max_log2size); + return -EINVAL; + } + + /* + * We might end up with a num_ents != sid_bits, which is fine. In the + * ARM_SMMU_OPT_KDUMP_ADOPT case, arm_smmu_write_strtab() is bypassed. + */ + cfg->linear.num_ents = 1U << log2size; + + size = cfg->linear.num_ents * sizeof(struct arm_smmu_ste); + cfg->linear.table = memremap(base, size, MEMREMAP_WB); + if (!cfg->linear.table) + return -ENOMEM; + return 0; +} + +static void arm_smmu_kdump_adopt_cleanup(void *data) +{ + struct arm_smmu_device *smmu = data; + u32 cfg_reg = readl_relaxed(smmu->base + ARM_SMMU_STRTAB_BASE_CFG); + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + u32 fmt = FIELD_GET(STRTAB_BASE_CFG_FMT, cfg_reg); + + if (fmt == STRTAB_BASE_CFG_FMT_2LVL) { + kfree(cfg->l2.l2ptrs); + if (cfg->l2.l1tab) + memunmap(cfg->l2.l1tab); + } else if (fmt == STRTAB_BASE_CFG_FMT_LINEAR) { + if (cfg->linear.table) + memunmap(cfg->linear.table); + } +} + +static int arm_smmu_kdump_adopt_strtab(struct arm_smmu_device *smmu) +{ + u32 cfg_reg = readl_relaxed(smmu->base + ARM_SMMU_STRTAB_BASE_CFG); + u64 base_reg = readq_relaxed(smmu->base + ARM_SMMU_STRTAB_BASE); + u32 fmt = FIELD_GET(STRTAB_BASE_CFG_FMT, cfg_reg); + phys_addr_t base = base_reg & STRTAB_BASE_ADDR_MASK; + int ret; + + dev_info(smmu->dev, "kdump: adopting crashed kernel's stream table\n"); + + if (fmt == STRTAB_BASE_CFG_FMT_2LVL) { + /* + * Both kernels run on the same hardware, so it's impossible for + * kdump kernel to see the support for linear stream table only. + */ + if (WARN_ON(!(smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB))) + ret = -EINVAL; + else + ret = arm_smmu_kdump_adopt_strtab_2lvl(smmu, cfg_reg, + base); + } else if (fmt == STRTAB_BASE_CFG_FMT_LINEAR) { + /* + * In case that the old kernel for some reason used the linear + * format, enforce the same format to match the adopted table. + */ + ret = arm_smmu_kdump_adopt_strtab_linear(smmu, cfg_reg, base); + if (!ret) + smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB; + } else { + dev_err(smmu->dev, "kdump: invalid STRTAB format %u\n", fmt); + ret = -EINVAL; + } + + if (ret) { + arm_smmu_kdump_adopt_cleanup(smmu); + goto err; + } + + ret = devm_add_action_or_reset(smmu->dev, arm_smmu_kdump_adopt_cleanup, + smmu); + /* devm_add_action_or_reset ran the cleanup upon failure */ + if (ret) { + dev_warn(smmu->dev, "kdump: failed to set up cleanup action\n"); + goto err; + } + + return 0; + +err: + dev_warn(smmu->dev, "kdump: falling back to full reset\n"); + memset(&smmu->strtab_cfg, 0, sizeof(smmu->strtab_cfg)); + smmu->options &= ~ARM_SMMU_OPT_KDUMP_ADOPT; + return ret; +} + static int arm_smmu_init_strtab(struct arm_smmu_device *smmu) { int ret; + if ((smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT) && + !arm_smmu_kdump_adopt_strtab(smmu)) + goto out; + if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) ret = arm_smmu_init_strtab_2lvl(smmu); else @@ -3996,6 +4271,7 @@ static int arm_smmu_init_strtab(struct arm_smmu_device *smmu) if (ret) return ret; +out: ida_init(&smmu->vmid_map); return 0; @@ -4120,6 +4396,21 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) arm_smmu_setup_msis(smmu); /* Request interrupt lines */ + irq = smmu->gerr_irq; + if (irq) { + ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler, + 0, "arm-smmu-v3-gerror", smmu); + if (ret < 0) + dev_warn(smmu->dev, "failed to enable gerror irq\n"); + } else { + dev_warn(smmu->dev, + "no gerr irq - errors will not be reported!\n"); + } + + /* No EVTQ/PRIQ interrupts in kdump -- queues are disabled */ + if (is_kdump_kernel()) + return; + irq = smmu->evtq.q.irq; if (irq) { ret = devm_request_threaded_irq(smmu->dev, irq, NULL, @@ -4132,16 +4423,6 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n"); } - irq = smmu->gerr_irq; - if (irq) { - ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler, - 0, "arm-smmu-v3-gerror", smmu); - if (ret < 0) - dev_warn(smmu->dev, "failed to enable gerror irq\n"); - } else { - dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n"); - } - if (smmu->features & ARM_SMMU_FEAT_PRI) { irq = smmu->priq.q.irq; if (irq) { @@ -4162,7 +4443,7 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) { int ret, irq; - u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN; + u32 irqen_flags = IRQ_CTRL_GERROR_IRQEN; /* Disable IRQs first */ ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL, @@ -4177,19 +4458,30 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) /* * Cavium ThunderX2 implementation doesn't support unique irq * lines. Use a single irq line for all the SMMUv3 interrupts. + * + * In kdump, EVTQ/PRIQ are disabled, so no threaded handling. */ - ret = devm_request_threaded_irq(smmu->dev, irq, - arm_smmu_combined_irq_handler, - arm_smmu_combined_irq_thread, - IRQF_ONESHOT, - "arm-smmu-v3-combined-irq", smmu); + if (is_kdump_kernel()) + ret = devm_request_irq(smmu->dev, irq, + arm_smmu_combined_irq_handler, 0, + "arm-smmu-v3-combined-irq", + smmu); + else + ret = devm_request_threaded_irq( + smmu->dev, irq, arm_smmu_combined_irq_handler, + arm_smmu_combined_irq_thread, IRQF_ONESHOT, + "arm-smmu-v3-combined-irq", smmu); if (ret < 0) dev_warn(smmu->dev, "failed to enable combined irq\n"); } else arm_smmu_setup_unique_irqs(smmu); - if (smmu->features & ARM_SMMU_FEAT_PRI) - irqen_flags |= IRQ_CTRL_PRIQ_IRQEN; + /* No EVTQ/PRIQ IRQ generation in kdump -- queues are disabled */ + if (!is_kdump_kernel()) { + irqen_flags |= IRQ_CTRL_EVTQ_IRQEN; + if (smmu->features & ARM_SMMU_FEAT_PRI) + irqen_flags |= IRQ_CTRL_PRIQ_IRQEN; + } /* Enable interrupt generation on the SMMU */ ret = arm_smmu_write_reg_sync(smmu, irqen_flags, @@ -4238,11 +4530,28 @@ static void arm_smmu_write_strtab(struct arm_smmu_device *smmu) static int arm_smmu_device_reset(struct arm_smmu_device *smmu) { int ret; - u32 reg, enables; + u32 reg, enables = 0; struct arm_smmu_cmdq_ent cmd; - /* Clear CR0 and sync (disables SMMU and queue processing) */ reg = readl_relaxed(smmu->base + ARM_SMMU_CR0); + + /* + * In a kdump case (set when CR0_SMMUEN=1 and !GERROR_SFM_ERR), retain + * CR0_SMMUEN to avoid aborting in-flight DMA, and CR0_ATSCHK to carry + * on the ATS-check policy. + * + * According to spec, updating STRTAB_BASE/CR1/CR2 when CR0_SMMUEN=1 is + * CONSTRAINED UNPREDICTABLE. So, skip those register updates and rely + * on the adopted stream table from the crashed kernel. + */ + if (smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT) { + dev_info(smmu->dev, + "kdump: retaining SMMUEN for in-flight DMA\n"); + enables = reg & (CR0_SMMUEN | CR0_ATSCHK); + goto reset_queues; + } + + /* Clear CR0 and sync (disables SMMU and queue processing) */ if (reg & CR0_SMMUEN) { dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n"); arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0); @@ -4272,12 +4581,36 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu) /* Stream table */ arm_smmu_write_strtab(smmu); +reset_queues: + if (smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT) { + /* Disable queues since arm_smmu_device_disable() was skipped */ + ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, + ARM_SMMU_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to disable queues\n"); + return ret; + } + } + + /* + * GERROR bits are latched. Read after queue disabling so that unhandled + * errors would be visible. Ack everything prior to re-enabling the CMDQ + * as a stale CMDQ_ERR would halt the CMDQ and new command will timeout. + */ + if (is_kdump_kernel()) { + u32 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR); + u32 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN); + + if ((gerror ^ gerrorn) & GERROR_ERR_MASK) + writel(gerror, smmu->base + ARM_SMMU_GERRORN); + } + /* Command queue */ writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE); writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD); writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS); - enables = CR0_CMDQEN; + enables |= CR0_CMDQEN; ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, ARM_SMMU_CR0ACK); if (ret) { @@ -4298,21 +4631,35 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu) cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL; arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); - /* Event queue */ - writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE); - writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD); - writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS); - - enables |= CR0_EVTQEN; - ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, - ARM_SMMU_CR0ACK); - if (ret) { - dev_err(smmu->dev, "failed to enable event queue\n"); - return ret; + /* + * Event queue + * + * Do not enable in a kdump case, as the crashed kernel's CDs and page + * tables might be corrupted, triggering event spamming. + */ + if (!is_kdump_kernel()) { + writeq_relaxed(smmu->evtq.q.q_base, + smmu->base + ARM_SMMU_EVTQ_BASE); + writel_relaxed(smmu->evtq.q.llq.prod, + smmu->page1 + ARM_SMMU_EVTQ_PROD); + writel_relaxed(smmu->evtq.q.llq.cons, + smmu->page1 + ARM_SMMU_EVTQ_CONS); + + enables |= CR0_EVTQEN; + ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, + ARM_SMMU_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to enable event queue\n"); + return ret; + } } - /* PRI queue */ - if (smmu->features & ARM_SMMU_FEAT_PRI) { + /* + * PRI queue + * + * Do not enable in a kdump case, as we cannot serve page requests. + */ + if (!is_kdump_kernel() && (smmu->features & ARM_SMMU_FEAT_PRI)) { writeq_relaxed(smmu->priq.q.q_base, smmu->base + ARM_SMMU_PRIQ_BASE); writel_relaxed(smmu->priq.q.llq.prod, @@ -4345,9 +4692,6 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu) return ret; } - if (is_kdump_kernel()) - enables &= ~(CR0_EVTQEN | CR0_PRIQEN); - /* Enable the SMMU interface */ enables |= CR0_SMMUEN; ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, @@ -4428,6 +4772,33 @@ static void arm_smmu_get_httu(struct arm_smmu_device *smmu, u32 reg) hw_features, fw_features); } +static void arm_smmu_device_hw_probe_kdump(struct arm_smmu_device *smmu) +{ + u32 gerror, gerrorn, active; + + /* No adoption if SMMU is disabled (i.e., there is no in-flight DMA) */ + if (!(readl_relaxed(smmu->base + ARM_SMMU_CR0) & CR0_SMMUEN)) + return; + + /* For now, only support a coherent SMMU that works with MEMREMAP_WB */ + if (!(smmu->features & ARM_SMMU_FEAT_COHERENCY)) { + dev_warn(smmu->dev, + "kdump: non-coherent SMMU unsupported; reset to block all DMAs\n"); + return; + } + + gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR); + gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN); + active = gerror ^ gerrorn; + if (active & GERROR_SFM_ERR) { + dev_warn(smmu->dev, + "kdump: SMMU in Service Failure Mode, must reset\n"); + return; + } + + smmu->options |= ARM_SMMU_OPT_KDUMP_ADOPT; +} + static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) { u32 reg; @@ -4650,6 +5021,10 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n", smmu->ias, smmu->oas, smmu->features); + + if (is_kdump_kernel()) + arm_smmu_device_hw_probe_kdump(smmu); + return 0; } @@ -4781,6 +5156,14 @@ static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu) struct list_head rmr_list; struct iommu_resv_region *e; + /* + * Kdump adoption keeps the crashed kernel's table live. Rewriting the + * adopted STE here could expose an in-flight fetch to a transient V=0 + * entry, or change Cfg=translate to Cfg=bypass. Must skip here. + */ + if (smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT) + return; + INIT_LIST_HEAD(&rmr_list); iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); @@ -4797,10 +5180,7 @@ static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu) continue; } - /* - * STE table is not programmed to HW, see - * arm_smmu_initial_bypass_stes() - */ + /* The fresh stream table is not yet live. */ arm_smmu_make_bypass_ste(smmu, arm_smmu_get_step_for_sid(smmu, rmr->sids[i])); } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index d813ec6c3919e..6d433efcf522b 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -775,6 +775,7 @@ struct arm_smmu_device { #define ARM_SMMU_OPT_MSIPOLL (1 << 2) #define ARM_SMMU_OPT_CMDQ_FORCE_SYNC (1 << 3) #define ARM_SMMU_OPT_TEGRA241_CMDQV (1 << 4) +#define ARM_SMMU_OPT_KDUMP_ADOPT (1 << 5) u32 options; struct arm_smmu_cmdq cmdq;