summaryrefslogtreecommitdiff
path: root/drivers/iommu
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/iommu')
-rw-r--r--drivers/iommu/amd/debugfs.c12
-rw-r--r--drivers/iommu/amd/init.c36
-rw-r--r--drivers/iommu/amd/ppr.c2
-rw-r--r--drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c9
-rw-r--r--drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c173
-rw-r--r--drivers/iommu/arm/arm-smmu/qcom_iommu.c6
-rw-r--r--drivers/iommu/dma-iommu.c26
-rw-r--r--drivers/iommu/intel/iommu.c32
-rw-r--r--drivers/iommu/intel/iommu.h2
-rw-r--r--drivers/iommu/intel/pasid.c9
-rw-r--r--drivers/iommu/iommufd/device.c13
-rw-r--r--drivers/iommu/iommufd/iommufd_private.h9
-rw-r--r--drivers/iommu/iommufd/main.c20
-rw-r--r--drivers/iommu/iommufd/selftest.c5
-rw-r--r--drivers/iommu/msm_iommu.c2
-rw-r--r--drivers/iommu/mtk_iommu_v1.c2
16 files changed, 241 insertions, 117 deletions
diff --git a/drivers/iommu/amd/debugfs.c b/drivers/iommu/amd/debugfs.c
index 4c53b6361314..5c573ec8e27a 100644
--- a/drivers/iommu/amd/debugfs.c
+++ b/drivers/iommu/amd/debugfs.c
@@ -176,19 +176,13 @@ static ssize_t devid_write(struct file *filp, const char __user *ubuf,
kfree(srcid_ptr);
return -ENODEV;
}
- break;
- }
-
- if (pci_seg->id != seg) {
+ sbdf = PCI_SEG_DEVID_TO_SBDF(seg, devid);
kfree(srcid_ptr);
- return -EINVAL;
+ return cnt;
}
- sbdf = PCI_SEG_DEVID_TO_SBDF(seg, devid);
-
kfree(srcid_ptr);
-
- return cnt;
+ return -EINVAL;
}
static int devid_show(struct seq_file *m, void *unused)
diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index e7d7b4cb9337..2563ebe9f246 100644
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -3091,14 +3091,28 @@ static void __init free_iommu_resources(void)
/* SB IOAPIC for Hygon family 18h model 4h is on the device 0xb */
#define IOAPIC_SB_DEVID_FAM18H_M4H ((0x00 << 8) | PCI_DEVFN(0xb, 0))
+/*
+ * The Southbridge IOAPIC is assigned a GSI Base of 0 (handling interrupts
+ * 0 through 23).
+ */
+static int __init get_sb_ioapic_id(void)
+{
+ int idx = mp_find_ioapic(0);
+
+ if (idx < 0)
+ return -ENODEV;
+
+ return mpc_ioapic_id(idx);
+}
+
static bool __init check_ioapic_information(void)
{
const char *fw_bug = FW_BUG;
bool ret, has_sb_ioapic;
- int idx;
+ int idx, sb_apicid;
has_sb_ioapic = false;
- ret = false;
+ ret = true;
/*
* If we have map overrides on the kernel command line the
@@ -3108,6 +3122,16 @@ static bool __init check_ioapic_information(void)
if (cmdline_maps)
fw_bug = "";
+ sb_apicid = get_sb_ioapic_id();
+ if (sb_apicid < 0) {
+ /*
+ * Lack of SB IOAPIC registration is not a firmware bug,
+ * e.g. kernel booted with noapic or noacpi.
+ */
+ fw_bug = "";
+ goto out;
+ }
+
for (idx = 0; idx < nr_ioapics; idx++) {
int devid, id = mpc_ioapic_id(idx);
@@ -3116,17 +3140,16 @@ static bool __init check_ioapic_information(void)
pr_err("%s: IOAPIC[%d] not in IVRS table\n",
fw_bug, id);
ret = false;
- } else if (devid == IOAPIC_SB_DEVID ||
+ } else if (id == sb_apicid && (devid == IOAPIC_SB_DEVID ||
(boot_cpu_data.x86_vendor == X86_VENDOR_HYGON &&
boot_cpu_data.x86 == 0x18 &&
boot_cpu_data.x86_model >= 0x4 &&
boot_cpu_data.x86_model <= 0xf &&
- devid == IOAPIC_SB_DEVID_FAM18H_M4H)) {
+ devid == IOAPIC_SB_DEVID_FAM18H_M4H))) {
has_sb_ioapic = true;
- ret = true;
}
}
-
+out:
if (!has_sb_ioapic) {
/*
* We expect the SB IOAPIC to be listed in the IVRS
@@ -3137,6 +3160,7 @@ static bool __init check_ioapic_information(void)
* device id for the IOAPIC in the system.
*/
pr_err("%s: No southbridge IOAPIC found\n", fw_bug);
+ ret = false;
}
if (!ret)
diff --git a/drivers/iommu/amd/ppr.c b/drivers/iommu/amd/ppr.c
index 80369ca1e316..1dde19a79e80 100644
--- a/drivers/iommu/amd/ppr.c
+++ b/drivers/iommu/amd/ppr.c
@@ -140,7 +140,7 @@ static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw)
if (event.fault.prm.pasid == 0 ||
event.fault.prm.pasid >= dev_data->max_pasids) {
pr_info_ratelimited("Invalid PASID : 0x%x, device : 0x%x\n",
- event.fault.prm.pasid, pdev->dev.id);
+ event.fault.prm.pasid, dev_data->devid);
goto out;
}
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 5f933d806a14..35b7b2fd4a12 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -1240,9 +1240,9 @@ VISIBLE_IF_KUNIT
void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target,
__le64 *safe_bits)
{
- const __le64 eats_s1chk =
+ const u64 eats_s1chk =
FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_S1CHK);
- const __le64 eats_trans =
+ const u64 eats_trans =
FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_TRANS);
/*
@@ -4553,8 +4553,9 @@ static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
u32 reg;
writel_relaxed(val, smmu->base + reg_off);
- return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
- 1, ARM_SMMU_POLL_TIMEOUT_US);
+ return readl_relaxed_poll_timeout_atomic(smmu->base + ack_off, reg,
+ reg == val, 1,
+ ARM_SMMU_POLL_TIMEOUT_US);
}
/* GBPA is "special" */
diff --git a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
index e5f52c0f6e9a..ef08367567d3 100644
--- a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
+++ b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
@@ -320,16 +320,30 @@ static void tegra241_vintf0_handle_error(struct tegra241_vintf *vintf)
while (map) {
unsigned long lidx = __ffs64(map);
- struct tegra241_vcmdq *vcmdq = vintf->lvcmdqs[lidx];
- u32 gerror = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR));
+ struct tegra241_vcmdq *vcmdq;
+ u32 gerror;
+ map &= ~BIT_ULL(lidx);
+
+ /* Pairs with smp_store_release() publishing it */
+ vcmdq = smp_load_acquire(&vintf->lvcmdqs[lidx]);
+ if (!vcmdq)
+ continue;
+
+ gerror = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR));
__arm_smmu_cmdq_skip_err(&vintf->cmdqv->smmu, &vcmdq->cmdq);
writel(gerror, REG_VCMDQ_PAGE0(vcmdq, GERRORN));
- map &= ~BIT_ULL(lidx);
}
}
}
+/*
+ * The CMDQV error interrupt is edge-triggered, so a pending VINTF error fires
+ * this ISR once and does not re-assert. An unacked guest therefore cannot
+ * storm the host. The HW latches and forwards each new error event on its
+ * own, so an already-set ERR_MAP bit does not suppress the interrupt for a
+ * new error.
+ */
static irqreturn_t tegra241_cmdqv_isr(int irq, void *devid)
{
struct tegra241_cmdqv *cmdqv = (struct tegra241_cmdqv *)devid;
@@ -352,16 +366,27 @@ static irqreturn_t tegra241_cmdqv_isr(int irq, void *devid)
/* Handle VINTF0 and its LVCMDQs */
if (vintf_map & BIT_ULL(0)) {
- tegra241_vintf0_handle_error(cmdqv->vintfs[0]);
+ struct tegra241_vintf *vintf0;
+
vintf_map &= ~BIT_ULL(0);
+
+ /* NULL until tegra241_cmdqv_init_structures() publishes it */
+ vintf0 = smp_load_acquire(&cmdqv->vintfs[0]);
+ if (vintf0)
+ tegra241_vintf0_handle_error(vintf0);
}
/* Handle other user VINTFs and their LVCMDQs */
while (vintf_map) {
unsigned long idx = __ffs64(vintf_map);
+ struct tegra241_vintf *vintf;
- tegra241_vintf_user_handle_error(cmdqv->vintfs[idx]);
vintf_map &= ~BIT_ULL(idx);
+
+ /* The slot may be published or torn down (NULL'd) concurrently */
+ vintf = smp_load_acquire(&cmdqv->vintfs[idx]);
+ if (vintf)
+ tegra241_vintf_user_handle_error(vintf);
}
return IRQ_HANDLED;
@@ -668,7 +693,6 @@ static int tegra241_vintf_init_lvcmdq(struct tegra241_vintf *vintf, u16 lidx,
vcmdq->page0 = cmdqv->base + TEGRA241_VINTFi_LVCMDQ_PAGE0(idx, lidx);
vcmdq->page1 = cmdqv->base + TEGRA241_VINTFi_LVCMDQ_PAGE1(idx, lidx);
- vintf->lvcmdqs[lidx] = vcmdq;
return 0;
}
@@ -684,7 +708,7 @@ static void tegra241_vintf_free_lvcmdq(struct tegra241_vintf *vintf, u16 lidx)
dev_dbg(vintf->cmdqv->dev,
"%sdeallocated\n", lvcmdq_error_header(vcmdq, header, 64));
/* Guest-owned VCMDQ is free-ed with hw_queue by iommufd core */
- if (vcmdq->vintf->hyp_own)
+ if (!vcmdq->vintf->idx)
kfree(vcmdq);
}
@@ -707,14 +731,15 @@ tegra241_vintf_alloc_lvcmdq(struct tegra241_vintf *vintf, u16 lidx)
/* Build an arm_smmu_cmdq for each LVCMDQ */
ret = tegra241_vcmdq_alloc_smmu_cmdq(vcmdq);
if (ret)
- goto deinit_lvcmdq;
+ goto free_vcmdq;
+
+ /* Pairs with the smp_load_acquire() in the error ISR */
+ smp_store_release(&vintf->lvcmdqs[lidx], vcmdq);
dev_dbg(cmdqv->dev,
"%sallocated\n", lvcmdq_error_header(vcmdq, header, 64));
return vcmdq;
-deinit_lvcmdq:
- tegra241_vintf_deinit_lvcmdq(vintf, lidx);
free_vcmdq:
kfree(vcmdq);
return ERR_PTR(ret);
@@ -725,8 +750,18 @@ free_vcmdq:
static void tegra241_cmdqv_deinit_vintf(struct tegra241_cmdqv *cmdqv, u16 idx)
{
kfree(cmdqv->vintfs[idx]->lvcmdqs);
+ /*
+ * Clear the slot and drain any in-flight ISR before returning idx to
+ * the IDA, so a concurrent create that reuses idx cannot have its
+ * freshly published VINTF erased here. A plain WRITE_ONCE() suffices
+ * since clearing the slot publishes no data. This also covers the
+ * init-failure unwind, which reaches deinit_vintf() without the
+ * destroy callback.
+ */
+ WRITE_ONCE(cmdqv->vintfs[idx], NULL);
+ if (cmdqv->irq > 0)
+ synchronize_irq(cmdqv->irq);
ida_free(&cmdqv->vintf_ids, idx);
- cmdqv->vintfs[idx] = NULL;
}
static int tegra241_cmdqv_init_vintf(struct tegra241_cmdqv *cmdqv, u16 max_idx,
@@ -752,7 +787,8 @@ static int tegra241_cmdqv_init_vintf(struct tegra241_cmdqv *cmdqv, u16 max_idx,
return -ENOMEM;
}
- cmdqv->vintfs[idx] = vintf;
+ /* Pairs with the smp_load_acquire() in tegra241_cmdqv_isr() */
+ smp_store_release(&cmdqv->vintfs[idx], vintf);
return ret;
}
@@ -770,7 +806,7 @@ static void tegra241_cmdqv_remove_vintf(struct tegra241_cmdqv *cmdqv, u16 idx)
dev_dbg(cmdqv->dev, "VINTF%u: deallocated\n", vintf->idx);
tegra241_cmdqv_deinit_vintf(cmdqv, idx);
- if (!vintf->hyp_own) {
+ if (vintf->idx) {
mutex_destroy(&vintf->lvcmdq_mutex);
ida_destroy(&vintf->sids);
/* Guest-owned VINTF is free-ed with viommu by iommufd core */
@@ -796,6 +832,14 @@ static void tegra241_cmdqv_remove(struct arm_smmu_device *smmu)
container_of(smmu, struct tegra241_cmdqv, smmu);
u16 idx;
+ /*
+ * Free the IRQ before tearing down the VINTFs. free_irq() waits for any
+ * in-flight tegra241_cmdqv_isr() to finish and blocks new ones, so the
+ * ISR cannot dereference a VINTF that is freed by the loop below.
+ */
+ if (cmdqv->irq > 0)
+ free_irq(cmdqv->irq, cmdqv);
+
/* Remove VINTF resources */
for (idx = 0; idx < cmdqv->num_vintfs; idx++) {
if (cmdqv->vintfs[idx]) {
@@ -808,8 +852,6 @@ static void tegra241_cmdqv_remove(struct arm_smmu_device *smmu)
/* Remove cmdqv resources */
ida_destroy(&cmdqv->vintf_ids);
- if (cmdqv->irq > 0)
- free_irq(cmdqv->irq, cmdqv);
iounmap(cmdqv->base);
kfree(cmdqv->vintfs);
put_device(cmdqv->dev); /* smmu->impl_dev */
@@ -881,6 +923,12 @@ static int tegra241_cmdqv_init_structures(struct arm_smmu_device *smmu)
ret = tegra241_cmdqv_init_vintf(cmdqv, 0, vintf);
if (ret) {
dev_err(cmdqv->dev, "failed to init vintf0: %d\n", ret);
+ /*
+ * tegra241_cmdqv_init_vintf() failed to publish the vintf0 to
+ * cmdqv->vintfs[], so the probe unwind path that goes through
+ * cmdqv->vintfs[] would miss it. Free it here.
+ */
+ kfree(vintf);
return ret;
}
@@ -902,16 +950,22 @@ static int tegra241_cmdqv_init_structures(struct arm_smmu_device *smmu)
static struct dentry *cmdqv_debugfs_dir;
#endif
-static struct arm_smmu_device *
-__tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res,
- int irq)
+/*
+ * Probe the CMDQV and reallocate @smmu into the larger cmdqv->smmu.
+ *
+ * devm_krealloc() may relocate and free the original @smmu, so update *smmu to
+ * the new pointer once it succeeds. The error paths after it do the same, so a
+ * caller falling back keeps a live @smmu instead of the freed original.
+ */
+static int __tegra241_cmdqv_probe(struct arm_smmu_device **smmu,
+ struct resource *res, int irq)
{
static const struct arm_smmu_impl_ops init_ops = {
.init_structures = tegra241_cmdqv_init_structures,
.device_remove = tegra241_cmdqv_remove,
};
- struct tegra241_cmdqv *cmdqv = NULL;
- struct arm_smmu_device *new_smmu;
+ struct device *dev = (*smmu)->dev;
+ struct tegra241_cmdqv *cmdqv;
void __iomem *base;
u32 regval;
int ret;
@@ -920,38 +974,30 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res,
base = ioremap(res->start, resource_size(res));
if (!base) {
- dev_err(smmu->dev, "failed to ioremap\n");
- return NULL;
+ dev_err(dev, "failed to ioremap\n");
+ return -ENOMEM;
}
regval = readl(base + TEGRA241_CMDQV_CONFIG);
if (disable_cmdqv) {
- dev_info(smmu->dev, "Detected disable_cmdqv=true\n");
+ dev_info(dev, "Detected disable_cmdqv=true\n");
writel(regval & ~CMDQV_EN, base + TEGRA241_CMDQV_CONFIG);
+ ret = -ENODEV;
goto iounmap;
}
- cmdqv = devm_krealloc(smmu->dev, smmu, sizeof(*cmdqv), GFP_KERNEL);
- if (!cmdqv)
+ cmdqv = devm_krealloc(dev, *smmu, sizeof(*cmdqv), GFP_KERNEL);
+ if (!cmdqv) {
+ ret = -ENOMEM;
goto iounmap;
- new_smmu = &cmdqv->smmu;
+ }
+ *smmu = &cmdqv->smmu;
cmdqv->irq = irq;
cmdqv->base = base;
- cmdqv->dev = smmu->impl_dev;
+ cmdqv->dev = (*smmu)->impl_dev;
cmdqv->base_phys = res->start;
- if (cmdqv->irq > 0) {
- ret = request_threaded_irq(irq, NULL, tegra241_cmdqv_isr,
- IRQF_ONESHOT, "tegra241-cmdqv",
- cmdqv);
- if (ret) {
- dev_err(cmdqv->dev, "failed to request irq (%d): %d\n",
- cmdqv->irq, ret);
- goto iounmap;
- }
- }
-
regval = readl_relaxed(REG_CMDQV(cmdqv, PARAM));
cmdqv->num_vintfs = 1 << FIELD_GET(CMDQV_NUM_VINTF_LOG2, regval);
cmdqv->num_vcmdqs = 1 << FIELD_GET(CMDQV_NUM_VCMDQ_LOG2, regval);
@@ -961,11 +1007,28 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res,
cmdqv->vintfs =
kzalloc_objs(*cmdqv->vintfs, cmdqv->num_vintfs);
- if (!cmdqv->vintfs)
- goto free_irq;
+ if (!cmdqv->vintfs) {
+ ret = -ENOMEM;
+ goto iounmap;
+ }
ida_init(&cmdqv->vintf_ids);
+ /*
+ * Request the IRQ only after cmdqv->vintfs is allocated and zeroed, so
+ * the ISR would not walk an uninitialized array.
+ */
+ if (cmdqv->irq > 0) {
+ ret = request_threaded_irq(irq, NULL, tegra241_cmdqv_isr,
+ IRQF_ONESHOT, "tegra241-cmdqv",
+ cmdqv);
+ if (ret) {
+ dev_err(cmdqv->dev, "failed to request irq (%d): %d\n",
+ cmdqv->irq, ret);
+ goto free_vintfs;
+ }
+ }
+
#ifdef CONFIG_IOMMU_DEBUGFS
if (!cmdqv_debugfs_dir) {
cmdqv_debugfs_dir =
@@ -976,24 +1039,23 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res,
#endif
/* Provide init-level ops only, until tegra241_cmdqv_init_structures */
- new_smmu->impl_ops = &init_ops;
+ cmdqv->smmu.impl_ops = &init_ops;
- return new_smmu;
+ return 0;
-free_irq:
- if (cmdqv->irq > 0)
- free_irq(cmdqv->irq, cmdqv);
+free_vintfs:
+ ida_destroy(&cmdqv->vintf_ids);
+ kfree(cmdqv->vintfs);
iounmap:
iounmap(base);
- return NULL;
+ return ret;
}
struct arm_smmu_device *tegra241_cmdqv_probe(struct arm_smmu_device *smmu)
{
struct platform_device *pdev = to_platform_device(smmu->impl_dev);
- struct arm_smmu_device *new_smmu;
struct resource *res;
- int irq;
+ int irq, ret;
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (!res) {
@@ -1006,15 +1068,15 @@ struct arm_smmu_device *tegra241_cmdqv_probe(struct arm_smmu_device *smmu)
dev_warn(&pdev->dev,
"no interrupt. errors will not be reported\n");
- new_smmu = __tegra241_cmdqv_probe(smmu, res, irq);
- if (new_smmu)
- return new_smmu;
+ ret = __tegra241_cmdqv_probe(&smmu, res, irq);
+ if (!ret)
+ return smmu;
out_fallback:
dev_info(smmu->impl_dev, "Falling back to standard SMMU CMDQ\n");
smmu->options &= ~ARM_SMMU_OPT_TEGRA241_CMDQV;
put_device(smmu->impl_dev);
- return ERR_PTR(-ENODEV);
+ return smmu;
}
/* User space VINTF and VCMDQ Functions */
@@ -1142,13 +1204,15 @@ static int tegra241_vintf_alloc_lvcmdq_user(struct iommufd_hw_queue *hw_queue,
if (ret)
goto unmap_lvcmdq;
+ /* No lockless reader of a user VINTF's lvcmdqs[]; mutex-serialized */
+ vintf->lvcmdqs[lidx] = vcmdq;
+
hw_queue->destroy = &tegra241_vintf_destroy_lvcmdq_user;
mutex_unlock(&vintf->lvcmdq_mutex);
return 0;
unmap_lvcmdq:
tegra241_vcmdq_unmap_lvcmdq(vcmdq);
- tegra241_vintf_deinit_lvcmdq(vintf, lidx);
undepend_vcmdq:
if (vcmdq->prev)
iommufd_hw_queue_undepend(vcmdq, vcmdq->prev, core);
@@ -1194,7 +1258,8 @@ static int tegra241_vintf_init_vsid(struct iommufd_vdevice *vdev)
if (virt_sid > FIELD_MAX(VINTF_SID_MATCH_VIRT_SID))
return -EINVAL;
- WARN_ON_ONCE(master->num_streams != 1);
+ if (master->num_streams != 1)
+ return -EOPNOTSUPP;
/* Find an empty pair of SID_REPLACE and SID_MATCH */
sidx = ida_alloc_max(&vintf->sids, vintf->cmdqv->num_sids_per_vintf - 1,
diff --git a/drivers/iommu/arm/arm-smmu/qcom_iommu.c b/drivers/iommu/arm/arm-smmu/qcom_iommu.c
index a1e8cf29f594..09f2ee6be988 100644
--- a/drivers/iommu/arm/arm-smmu/qcom_iommu.c
+++ b/drivers/iommu/arm/arm-smmu/qcom_iommu.c
@@ -200,7 +200,7 @@ static irqreturn_t qcom_iommu_fault(int irq, void *dev)
fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0);
iova = iommu_readq(ctx, ARM_SMMU_CB_FAR);
- if (!report_iommu_fault(ctx->domain, ctx->dev, iova, 0)) {
+ if (report_iommu_fault(ctx->domain, ctx->dev, iova, 0)) {
dev_err_ratelimited(ctx->dev,
"Unhandled context fault: fsr=0x%x, "
"iova=0x%016llx, fsynr=0x%x, cb=%d\n",
@@ -855,7 +855,7 @@ static int qcom_iommu_device_probe(struct platform_device *pdev)
ret = iommu_device_register(&qcom_iommu->iommu, &qcom_iommu_ops, dev);
if (ret) {
dev_err(dev, "Failed to register iommu\n");
- goto err_pm_disable;
+ goto err_sysfs_remove;
}
if (qcom_iommu->local_base) {
@@ -866,6 +866,8 @@ static int qcom_iommu_device_probe(struct platform_device *pdev)
return 0;
+err_sysfs_remove:
+ iommu_device_sysfs_remove(&qcom_iommu->iommu);
err_pm_disable:
pm_runtime_disable(dev);
return ret;
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 9abaec0703ef..ae814d8c99c0 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -1671,13 +1671,16 @@ void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
}
if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
- !gfpflags_allow_blocking(gfp) && !coherent)
+ !gfpflags_allow_blocking(gfp) && !coherent) {
page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr,
- gfp, NULL);
- else
+ gfp, NULL);
+ if (!page)
+ return NULL;
+ } else {
cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);
- if (!cpu_addr)
- return NULL;
+ if (!cpu_addr)
+ return NULL;
+ }
*handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot,
dev->coherent_dma_mask);
@@ -2204,6 +2207,19 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
dma_addr_t iova;
int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
size_t size = cookie_msi_granule(domain);
+ static DEFINE_MUTEX(msi_prepare_lock);
+
+ /*
+ * Normally a device's default domain is only ever attached to that
+ * device's own group, and the group mutex held by
+ * iommu_group_mutex_assert()'s callers is enough on its own. A VFIO
+ * type1 container is the one case that breaks that assumption: it
+ * can merge devices from different groups onto one domain, so two
+ * devices' group mutexes don't serialize each other here. A static
+ * lock is sufficient due to the expectation that this is a corner
+ * case that will never be contended in practice.
+ */
+ guard(mutex)(&msi_prepare_lock);
msi_addr &= ~(phys_addr_t)(size - 1);
list_for_each_entry(msi_page, msi_page_list, list)
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index 4e7ba60f3a0a..9a87ac1250d7 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -1108,7 +1108,8 @@ static void copied_context_tear_down(struct intel_iommu *iommu,
assert_spin_locked(&iommu->lock);
did_old = context_domain_id(context);
- context_clear_entry(context);
+ context_clear_present(context);
+ __iommu_flush_cache(iommu, context, sizeof(*context));
if (did_old < cap_ndoms(iommu->cap)) {
iommu->flush.flush_context(iommu, did_old,
@@ -1119,6 +1120,9 @@ static void copied_context_tear_down(struct intel_iommu *iommu,
DMA_TLB_DSI_FLUSH);
}
+ context_clear_entry(context);
+ __iommu_flush_cache(iommu, context, sizeof(*context));
+
clear_context_copied(iommu, bus, devfn);
}
@@ -1246,7 +1250,7 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8
context_clear_present(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
spin_unlock(&iommu->lock);
- intel_context_flush_no_pasid(info, context, did);
+ intel_context_flush_no_pasid(info, context, did, PCI_DEVID(bus, devfn));
context_clear_entry(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
}
@@ -1446,7 +1450,7 @@ static int copy_context_table(struct intel_iommu *iommu,
struct context_entry **tbl,
int bus, bool ext)
{
- int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
+ int tbl_idx, tbl_slot = 0, idx, devfn, ret = 0, did;
struct context_entry *new_ce = NULL, ce;
struct context_entry *old_ce = NULL;
struct root_entry re;
@@ -1462,10 +1466,9 @@ static int copy_context_table(struct intel_iommu *iommu,
if (idx == 0) {
/* First save what we may have and clean up */
if (new_ce) {
- tbl[tbl_idx] = new_ce;
+ tbl[tbl_idx + tbl_slot] = new_ce;
__iommu_flush_cache(iommu, new_ce,
VTD_PAGE_SIZE);
- pos = 1;
}
if (old_ce)
@@ -1487,6 +1490,9 @@ static int copy_context_table(struct intel_iommu *iommu,
}
}
+ /* Track if saving UCTP or LCTP entries in scalable mode */
+ tbl_slot = ext && devfn >= 0x80 ? 1 : 0;
+
ret = -ENOMEM;
old_ce = memremap(old_ce_phys, PAGE_SIZE,
MEMREMAP_WB);
@@ -1515,7 +1521,7 @@ static int copy_context_table(struct intel_iommu *iommu,
new_ce[idx] = ce;
}
- tbl[tbl_idx + pos] = new_ce;
+ tbl[tbl_idx + tbl_slot] = new_ce;
__iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
@@ -3135,13 +3141,13 @@ static int intel_iommu_attach_device(struct iommu_domain *domain,
if (ret)
return ret;
- ret = iopf_for_domain_set(domain, dev);
+ ret = iopf_for_domain_replace(domain, old, dev);
if (ret)
return ret;
ret = dmar_domain_attach_device(to_dmar_domain(domain), dev);
if (ret)
- iopf_for_domain_remove(domain, dev);
+ iopf_for_domain_replace(old, domain, dev);
return ret;
}
@@ -3316,6 +3322,7 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)
return &iommu->iommu;
free_table:
+ intel_pasid_teardown_sm_context(dev);
intel_pasid_free_table(dev);
clear_rbtree:
device_rbtree_remove(info);
@@ -3844,10 +3851,13 @@ static int identity_domain_attach_dev(struct iommu_domain *domain,
return 0;
/*
- * No PRI support with the global identity domain. No need to enable or
- * disable PRI in this path as the iommu has been put in the blocking
- * state.
+ * The identity domain has no iopf_handler, so no IOPF reference is
+ * taken for it. The reference held by the old domain must still be
+ * released here; putting the device in the blocking state above does
+ * not affect the IOPF reference count.
*/
+ iopf_for_domain_remove(old, dev);
+
if (sm_supported(iommu))
ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID);
else
diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h
index 2cee36138d6e..2df59d999715 100644
--- a/drivers/iommu/intel/iommu.h
+++ b/drivers/iommu/intel/iommu.h
@@ -1240,7 +1240,7 @@ void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start,
unsigned long end);
void intel_context_flush_no_pasid(struct device_domain_info *info,
- struct context_entry *context, u16 did);
+ struct context_entry *context, u16 did, u16 sid);
int intel_iommu_enable_prq(struct intel_iommu *iommu);
int intel_iommu_finish_prq(struct intel_iommu *iommu);
diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c
index 81353fd46b37..e4f24d3f19a6 100644
--- a/drivers/iommu/intel/pasid.c
+++ b/drivers/iommu/intel/pasid.c
@@ -751,7 +751,7 @@ static void device_pasid_table_teardown(struct device *dev, u8 bus, u8 devfn)
context_clear_present(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
spin_unlock(&iommu->lock);
- intel_context_flush_no_pasid(info, context, did);
+ intel_context_flush_no_pasid(info, context, did, PCI_DEVID(bus, devfn));
context_clear_entry(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
}
@@ -955,9 +955,12 @@ static void __context_flush_dev_iotlb(struct device_domain_info *info)
* This helper can only be used when IOMMU is working in the legacy mode or
* IOMMU is in scalable mode but all PASID table entries of the device are
* non-present.
+ *
+ * @sid identifies the context entry that was modified, which may be a DMA
+ * alias of @info->dev rather than its own requester ID.
*/
void intel_context_flush_no_pasid(struct device_domain_info *info,
- struct context_entry *context, u16 did)
+ struct context_entry *context, u16 did, u16 sid)
{
struct intel_iommu *iommu = info->iommu;
@@ -967,7 +970,7 @@ void intel_context_flush_no_pasid(struct device_domain_info *info,
* when operating in scalable mode. Therefore the @did value doesn't
* matter in scalable mode.
*/
- iommu->flush.flush_context(iommu, did, PCI_DEVID(info->bus, info->devfn),
+ iommu->flush.flush_context(iommu, did, sid,
DMA_CCMD_MASK_NOBIT, DMA_CCMD_DEVICE_INVL);
/*
diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c
index d488c23fd353..c5d122f33ba2 100644
--- a/drivers/iommu/iommufd/device.c
+++ b/drivers/iommu/iommufd/device.c
@@ -148,29 +148,22 @@ static void iommufd_device_remove_vdev(struct iommufd_device *idev)
if (!idev->vdev)
goto out_unlock;
- vdev = iommufd_get_vdevice(idev->ictx, idev->vdev->obj.id);
+ vdev = idev->vdev;
+
/*
* An ongoing vdev destroy ioctl has removed the vdev from the object
* xarray, but has not finished iommufd_vdevice_destroy() yet as it
* needs the same mutex. We exit the locking then wait on wait_cnt
* reference for the vdev destruction.
*/
- if (IS_ERR(vdev))
- goto out_unlock;
-
- /* Should never happen */
- if (WARN_ON(vdev != idev->vdev)) {
- iommufd_put_object(idev->ictx, &vdev->obj);
+ if (iommufd_try_inc_users(idev->ictx, &vdev->obj))
goto out_unlock;
- }
/*
* vdev is still alive. Hold a users refcount to prevent racing with
* userspace destruction, then use iommufd_object_tombstone_user() to
* destroy it and leave a tombstone.
*/
- refcount_inc(&vdev->obj.users);
- iommufd_put_object(idev->ictx, &vdev->obj);
mutex_unlock(&idev->igroup->lock);
iommufd_object_tombstone_user(idev->ictx, &vdev->obj);
return;
diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h
index 43fbc5bed8de..421d0cc7c1bc 100644
--- a/drivers/iommu/iommufd/iommufd_private.h
+++ b/drivers/iommu/iommufd/iommufd_private.h
@@ -182,6 +182,7 @@ static inline bool iommufd_lock_obj(struct iommufd_object *obj)
return true;
}
+int iommufd_try_inc_users(struct iommufd_ctx *ictx, struct iommufd_object *obj);
struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id,
enum iommufd_object_type type);
static inline void iommufd_put_object(struct iommufd_ctx *ictx,
@@ -698,14 +699,6 @@ void iommufd_vdevice_abort(struct iommufd_object *obj);
int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd);
void iommufd_hw_queue_destroy(struct iommufd_object *obj);
-static inline struct iommufd_vdevice *
-iommufd_get_vdevice(struct iommufd_ctx *ictx, u32 id)
-{
- return container_of(iommufd_get_object(ictx, id,
- IOMMUFD_OBJ_VDEVICE),
- struct iommufd_vdevice, obj);
-}
-
#ifdef CONFIG_IOMMUFD_TEST
int iommufd_test(struct iommufd_ucmd *ucmd);
void iommufd_selftest_destroy(struct iommufd_object *obj);
diff --git a/drivers/iommu/iommufd/main.c b/drivers/iommu/iommufd/main.c
index 8c6d43601afb..e1097a1db21a 100644
--- a/drivers/iommu/iommufd/main.c
+++ b/drivers/iommu/iommufd/main.c
@@ -180,6 +180,26 @@ struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id,
return obj;
}
+/*
+ * Increment the users count of an object outside the context of an ioctl that
+ * has already locked it. The users refcount cannot be increased on an already
+ * created object unless the object is installed in the xarray, otherwise things
+ * are racing with a parallel destruction.
+ */
+int iommufd_try_inc_users(struct iommufd_ctx *ictx, struct iommufd_object *obj)
+{
+ struct iommufd_object *cur;
+
+ xa_lock(&ictx->objects);
+ cur = xa_load(&ictx->objects, obj->id);
+ if (cur == obj)
+ refcount_inc(&obj->users);
+ xa_unlock(&ictx->objects);
+ if (cur != obj)
+ return -EBUSY;
+ return 0;
+}
+
static int iommufd_object_dec_wait(struct iommufd_ctx *ictx,
struct iommufd_object *to_destroy)
{
diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c
index 727b59799d5f..55f21c799d8e 100644
--- a/drivers/iommu/iommufd/selftest.c
+++ b/drivers/iommu/iommufd/selftest.c
@@ -12,6 +12,7 @@
#include <linux/iommu.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
+#include <linux/sizes.h>
#include <linux/xarray.h>
#include <uapi/linux/iommufd.h>
#include <linux/generic_pt/iommu.h>
@@ -1705,6 +1706,9 @@ static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id,
if (!page_size || !length || iova % page_size || length % page_size ||
!uptr)
return -EINVAL;
+ max = length / page_size;
+ if (max > SZ_16M * BITS_PER_BYTE)
+ return -EOVERFLOW;
hwpt = get_md_pagetable(ucmd, mockpt_id, &mock);
if (IS_ERR(hwpt))
@@ -1715,7 +1719,6 @@ static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id,
goto out_put;
}
- max = length / page_size;
tmp = kvzalloc(DIV_ROUND_UP(max, BITS_PER_LONG) * sizeof(unsigned long),
GFP_KERNEL_ACCOUNT);
if (!tmp) {
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index d0d926be7495..9a3ed70563b1 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -720,7 +720,7 @@ static int msm_iommu_probe(struct platform_device *pdev)
iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);
if (!iommu)
- return -ENODEV;
+ return -ENOMEM;
iommu->dev = &pdev->dev;
INIT_LIST_HEAD(&iommu->ctx_list);
diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c
index ac97dd2868d4..e907c9953142 100644
--- a/drivers/iommu/mtk_iommu_v1.c
+++ b/drivers/iommu/mtk_iommu_v1.c
@@ -88,7 +88,7 @@ struct dma_iommu_mapping {
/* MTK generation one iommu HW only support 4K size mapping */
#define MT2701_IOMMU_PAGE_SHIFT 12
#define MT2701_IOMMU_PAGE_SIZE (1UL << MT2701_IOMMU_PAGE_SHIFT)
-#define MT2701_LARB_NR_MAX 3
+#define MT2701_LARB_NR_MAX 4
/*
* MTK m4u support 4GB iova address space, and only support 4K page
generated by cgit v1.2.3 (git 2.39.1) at 2026-09-15 21:58:51 +0000