summaryrefslogtreecommitdiff
path: root/drivers/iommu
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/iommu')
-rw-r--r--drivers/iommu/amd/debugfs.c12
-rw-r--r--drivers/iommu/amd/init.c58
-rw-r--r--drivers/iommu/amd/ppr.c2
-rw-r--r--drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c9
-rw-r--r--drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c119
-rw-r--r--drivers/iommu/arm/arm-smmu/qcom_iommu.c6
-rw-r--r--drivers/iommu/dma-iommu.c26
-rw-r--r--drivers/iommu/intel/iommu.c31
-rw-r--r--drivers/iommu/intel/iommu.h2
-rw-r--r--drivers/iommu/intel/pasid.c9
-rw-r--r--drivers/iommu/iommufd/device.c13
-rw-r--r--drivers/iommu/iommufd/iommufd_private.h9
-rw-r--r--drivers/iommu/iommufd/main.c20
-rw-r--r--drivers/iommu/iommufd/selftest.c5
-rw-r--r--drivers/iommu/msm_iommu.c2
-rw-r--r--drivers/iommu/mtk_iommu_v1.c2
16 files changed, 226 insertions, 99 deletions
diff --git a/drivers/iommu/amd/debugfs.c b/drivers/iommu/amd/debugfs.c
index 3909a1fb218e..f7bb551d285f 100644
--- a/drivers/iommu/amd/debugfs.c
+++ b/drivers/iommu/amd/debugfs.c
@@ -176,19 +176,13 @@ static ssize_t devid_write(struct file *filp, const char __user *ubuf,
kfree(srcid_ptr);
return -ENODEV;
}
- break;
- }
-
- if (pci_seg->id != seg) {
+ sbdf = PCI_SEG_DEVID_TO_SBDF(seg, devid);
kfree(srcid_ptr);
- return -EINVAL;
+ return cnt;
}
- sbdf = PCI_SEG_DEVID_TO_SBDF(seg, devid);
-
kfree(srcid_ptr);
-
- return cnt;
+ return -EINVAL;
}
static int devid_show(struct seq_file *m, void *unused)
diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index 69170146d442..6bb9d3661bae 100644
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -3059,7 +3059,7 @@ static void disable_iommus(void)
* disable suspend until real resume implemented
*/
-static void amd_iommu_resume(void)
+static void amd_iommu_resume(void *data)
{
struct amd_iommu *iommu;
@@ -3074,7 +3074,7 @@ static void amd_iommu_resume(void)
amd_iommu_enable_interrupts();
}
-static int amd_iommu_suspend(void)
+static int amd_iommu_suspend(void *data)
{
/* disable IOMMUs to go out of the way for BIOS */
disable_iommus();
@@ -3082,11 +3082,15 @@ static int amd_iommu_suspend(void)
return 0;
}
-static struct syscore_ops amd_iommu_syscore_ops = {
+static const struct syscore_ops amd_iommu_syscore_ops = {
.suspend = amd_iommu_suspend,
.resume = amd_iommu_resume,
};
+static struct syscore amd_iommu_syscore = {
+ .ops = &amd_iommu_syscore_ops,
+};
+
static void __init free_iommu_resources(void)
{
free_iommu_all();
@@ -3096,14 +3100,31 @@ static void __init free_iommu_resources(void)
/* SB IOAPIC is always on this device in AMD systems */
#define IOAPIC_SB_DEVID ((0x00 << 8) | PCI_DEVFN(0x14, 0))
+/* SB IOAPIC for Hygon family 18h model 4h is on the device 0xb */
+#define IOAPIC_SB_DEVID_FAM18H_M4H ((0x00 << 8) | PCI_DEVFN(0xb, 0))
+
+/*
+ * The Southbridge IOAPIC is assigned a GSI Base of 0 (handling interrupts
+ * 0 through 23).
+ */
+static int __init get_sb_ioapic_id(void)
+{
+ int idx = mp_find_ioapic(0);
+
+ if (idx < 0)
+ return -ENODEV;
+
+ return mpc_ioapic_id(idx);
+}
+
static bool __init check_ioapic_information(void)
{
const char *fw_bug = FW_BUG;
bool ret, has_sb_ioapic;
- int idx;
+ int idx, sb_apicid;
has_sb_ioapic = false;
- ret = false;
+ ret = true;
/*
* If we have map overrides on the kernel command line the
@@ -3113,6 +3134,16 @@ static bool __init check_ioapic_information(void)
if (cmdline_maps)
fw_bug = "";
+ sb_apicid = get_sb_ioapic_id();
+ if (sb_apicid < 0) {
+ /*
+ * Lack of SB IOAPIC registration is not a firmware bug,
+ * e.g. kernel booted with noapic or noacpi.
+ */
+ fw_bug = "";
+ goto out;
+ }
+
for (idx = 0; idx < nr_ioapics; idx++) {
int devid, id = mpc_ioapic_id(idx);
@@ -3121,12 +3152,16 @@ static bool __init check_ioapic_information(void)
pr_err("%s: IOAPIC[%d] not in IVRS table\n",
fw_bug, id);
ret = false;
- } else if (devid == IOAPIC_SB_DEVID) {
+ } else if (id == sb_apicid && (devid == IOAPIC_SB_DEVID ||
+ (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON &&
+ boot_cpu_data.x86 == 0x18 &&
+ boot_cpu_data.x86_model >= 0x4 &&
+ boot_cpu_data.x86_model <= 0xf &&
+ devid == IOAPIC_SB_DEVID_FAM18H_M4H))) {
has_sb_ioapic = true;
- ret = true;
}
}
-
+out:
if (!has_sb_ioapic) {
/*
* We expect the SB IOAPIC to be listed in the IVRS
@@ -3137,6 +3172,7 @@ static bool __init check_ioapic_information(void)
* device id for the IOAPIC in the system.
*/
pr_err("%s: No southbridge IOAPIC found\n", fw_bug);
+ ret = false;
}
if (!ret)
@@ -3458,7 +3494,7 @@ static int __init state_next(void)
init_state = IOMMU_ENABLED;
break;
case IOMMU_ENABLED:
- register_syscore_ops(&amd_iommu_syscore_ops);
+ register_syscore(&amd_iommu_syscore);
iommu_snp_enable();
amd_iommu_apply_erratum_snp();
@@ -3576,12 +3612,12 @@ int __init amd_iommu_enable(void)
void amd_iommu_disable(void)
{
- amd_iommu_suspend();
+ amd_iommu_suspend(NULL);
}
int amd_iommu_reenable(int mode)
{
- amd_iommu_resume();
+ amd_iommu_resume(NULL);
return 0;
}
diff --git a/drivers/iommu/amd/ppr.c b/drivers/iommu/amd/ppr.c
index 80369ca1e316..1dde19a79e80 100644
--- a/drivers/iommu/amd/ppr.c
+++ b/drivers/iommu/amd/ppr.c
@@ -140,7 +140,7 @@ static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw)
if (event.fault.prm.pasid == 0 ||
event.fault.prm.pasid >= dev_data->max_pasids) {
pr_info_ratelimited("Invalid PASID : 0x%x, device : 0x%x\n",
- event.fault.prm.pasid, pdev->dev.id);
+ event.fault.prm.pasid, dev_data->devid);
goto out;
}
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 01f448acd8dc..b677390e95bb 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -1097,9 +1097,9 @@ VISIBLE_IF_KUNIT
void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target,
__le64 *safe_bits)
{
- const __le64 eats_s1chk =
+ const u64 eats_s1chk =
FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_S1CHK);
- const __le64 eats_trans =
+ const u64 eats_trans =
FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_TRANS);
/*
@@ -3992,8 +3992,9 @@ static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
u32 reg;
writel_relaxed(val, smmu->base + reg_off);
- return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
- 1, ARM_SMMU_POLL_TIMEOUT_US);
+ return readl_relaxed_poll_timeout_atomic(smmu->base + ack_off, reg,
+ reg == val, 1,
+ ARM_SMMU_POLL_TIMEOUT_US);
}
/* GBPA is "special" */
diff --git a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
index cee489855311..2b55a221c35a 100644
--- a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
+++ b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
@@ -322,16 +322,30 @@ static void tegra241_vintf0_handle_error(struct tegra241_vintf *vintf)
while (map) {
unsigned long lidx = __ffs64(map);
- struct tegra241_vcmdq *vcmdq = vintf->lvcmdqs[lidx];
- u32 gerror = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR));
+ struct tegra241_vcmdq *vcmdq;
+ u32 gerror;
+ map &= ~BIT_ULL(lidx);
+
+ /* Pairs with smp_store_release() publishing it */
+ vcmdq = smp_load_acquire(&vintf->lvcmdqs[lidx]);
+ if (!vcmdq)
+ continue;
+
+ gerror = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR));
__arm_smmu_cmdq_skip_err(&vintf->cmdqv->smmu, &vcmdq->cmdq);
writel(gerror, REG_VCMDQ_PAGE0(vcmdq, GERRORN));
- map &= ~BIT_ULL(lidx);
}
}
}
+/*
+ * The CMDQV error interrupt is edge-triggered, so a pending VINTF error fires
+ * this ISR once and does not re-assert. An unacked guest therefore cannot
+ * storm the host. The HW latches and forwards each new error event on its
+ * own, so an already-set ERR_MAP bit does not suppress the interrupt for a
+ * new error.
+ */
static irqreturn_t tegra241_cmdqv_isr(int irq, void *devid)
{
struct tegra241_cmdqv *cmdqv = (struct tegra241_cmdqv *)devid;
@@ -354,16 +368,27 @@ static irqreturn_t tegra241_cmdqv_isr(int irq, void *devid)
/* Handle VINTF0 and its LVCMDQs */
if (vintf_map & BIT_ULL(0)) {
- tegra241_vintf0_handle_error(cmdqv->vintfs[0]);
+ struct tegra241_vintf *vintf0;
+
vintf_map &= ~BIT_ULL(0);
+
+ /* NULL until tegra241_cmdqv_init_structures() publishes it */
+ vintf0 = smp_load_acquire(&cmdqv->vintfs[0]);
+ if (vintf0)
+ tegra241_vintf0_handle_error(vintf0);
}
/* Handle other user VINTFs and their LVCMDQs */
while (vintf_map) {
unsigned long idx = __ffs64(vintf_map);
+ struct tegra241_vintf *vintf;
- tegra241_vintf_user_handle_error(cmdqv->vintfs[idx]);
vintf_map &= ~BIT_ULL(idx);
+
+ /* The slot may be published or torn down (NULL'd) concurrently */
+ vintf = smp_load_acquire(&cmdqv->vintfs[idx]);
+ if (vintf)
+ tegra241_vintf_user_handle_error(vintf);
}
return IRQ_HANDLED;
@@ -670,7 +695,6 @@ static int tegra241_vintf_init_lvcmdq(struct tegra241_vintf *vintf, u16 lidx,
vcmdq->page0 = cmdqv->base + TEGRA241_VINTFi_LVCMDQ_PAGE0(idx, lidx);
vcmdq->page1 = cmdqv->base + TEGRA241_VINTFi_LVCMDQ_PAGE1(idx, lidx);
- vintf->lvcmdqs[lidx] = vcmdq;
return 0;
}
@@ -686,7 +710,7 @@ static void tegra241_vintf_free_lvcmdq(struct tegra241_vintf *vintf, u16 lidx)
dev_dbg(vintf->cmdqv->dev,
"%sdeallocated\n", lvcmdq_error_header(vcmdq, header, 64));
/* Guest-owned VCMDQ is free-ed with hw_queue by iommufd core */
- if (vcmdq->vintf->hyp_own)
+ if (!vcmdq->vintf->idx)
kfree(vcmdq);
}
@@ -709,14 +733,15 @@ tegra241_vintf_alloc_lvcmdq(struct tegra241_vintf *vintf, u16 lidx)
/* Build an arm_smmu_cmdq for each LVCMDQ */
ret = tegra241_vcmdq_alloc_smmu_cmdq(vcmdq);
if (ret)
- goto deinit_lvcmdq;
+ goto free_vcmdq;
+
+ /* Pairs with the smp_load_acquire() in the error ISR */
+ smp_store_release(&vintf->lvcmdqs[lidx], vcmdq);
dev_dbg(cmdqv->dev,
"%sallocated\n", lvcmdq_error_header(vcmdq, header, 64));
return vcmdq;
-deinit_lvcmdq:
- tegra241_vintf_deinit_lvcmdq(vintf, lidx);
free_vcmdq:
kfree(vcmdq);
return ERR_PTR(ret);
@@ -727,8 +752,18 @@ free_vcmdq:
static void tegra241_cmdqv_deinit_vintf(struct tegra241_cmdqv *cmdqv, u16 idx)
{
kfree(cmdqv->vintfs[idx]->lvcmdqs);
+ /*
+ * Clear the slot and drain any in-flight ISR before returning idx to
+ * the IDA, so a concurrent create that reuses idx cannot have its
+ * freshly published VINTF erased here. A plain WRITE_ONCE() suffices
+ * since clearing the slot publishes no data. This also covers the
+ * init-failure unwind, which reaches deinit_vintf() without the
+ * destroy callback.
+ */
+ WRITE_ONCE(cmdqv->vintfs[idx], NULL);
+ if (cmdqv->irq > 0)
+ synchronize_irq(cmdqv->irq);
ida_free(&cmdqv->vintf_ids, idx);
- cmdqv->vintfs[idx] = NULL;
}
static int tegra241_cmdqv_init_vintf(struct tegra241_cmdqv *cmdqv, u16 max_idx,
@@ -754,7 +789,8 @@ static int tegra241_cmdqv_init_vintf(struct tegra241_cmdqv *cmdqv, u16 max_idx,
return -ENOMEM;
}
- cmdqv->vintfs[idx] = vintf;
+ /* Pairs with the smp_load_acquire() in tegra241_cmdqv_isr() */
+ smp_store_release(&cmdqv->vintfs[idx], vintf);
return ret;
}
@@ -772,7 +808,7 @@ static void tegra241_cmdqv_remove_vintf(struct tegra241_cmdqv *cmdqv, u16 idx)
dev_dbg(cmdqv->dev, "VINTF%u: deallocated\n", vintf->idx);
tegra241_cmdqv_deinit_vintf(cmdqv, idx);
- if (!vintf->hyp_own) {
+ if (vintf->idx) {
mutex_destroy(&vintf->lvcmdq_mutex);
ida_destroy(&vintf->sids);
/* Guest-owned VINTF is free-ed with viommu by iommufd core */
@@ -798,6 +834,14 @@ static void tegra241_cmdqv_remove(struct arm_smmu_device *smmu)
container_of(smmu, struct tegra241_cmdqv, smmu);
u16 idx;
+ /*
+ * Free the IRQ before tearing down the VINTFs. free_irq() waits for any
+ * in-flight tegra241_cmdqv_isr() to finish and blocks new ones, so the
+ * ISR cannot dereference a VINTF that is freed by the loop below.
+ */
+ if (cmdqv->irq > 0)
+ free_irq(cmdqv->irq, cmdqv);
+
/* Remove VINTF resources */
for (idx = 0; idx < cmdqv->num_vintfs; idx++) {
if (cmdqv->vintfs[idx]) {
@@ -810,8 +854,6 @@ static void tegra241_cmdqv_remove(struct arm_smmu_device *smmu)
/* Remove cmdqv resources */
ida_destroy(&cmdqv->vintf_ids);
- if (cmdqv->irq > 0)
- free_irq(cmdqv->irq, cmdqv);
iounmap(cmdqv->base);
kfree(cmdqv->vintfs);
put_device(cmdqv->dev); /* smmu->impl_dev */
@@ -946,6 +988,12 @@ static int tegra241_cmdqv_init_structures(struct arm_smmu_device *smmu)
ret = tegra241_cmdqv_init_vintf(cmdqv, 0, vintf);
if (ret) {
dev_err(cmdqv->dev, "failed to init vintf0: %d\n", ret);
+ /*
+ * tegra241_cmdqv_init_vintf() failed to publish the vintf0 to
+ * cmdqv->vintfs[], so the probe unwind path that goes through
+ * cmdqv->vintfs[] would miss it. Free it here.
+ */
+ kfree(vintf);
return ret;
}
@@ -1006,17 +1054,6 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res,
cmdqv->dev = smmu->impl_dev;
cmdqv->base_phys = res->start;
- if (cmdqv->irq > 0) {
- ret = request_threaded_irq(irq, NULL, tegra241_cmdqv_isr,
- IRQF_ONESHOT, "tegra241-cmdqv",
- cmdqv);
- if (ret) {
- dev_err(cmdqv->dev, "failed to request irq (%d): %d\n",
- cmdqv->irq, ret);
- goto iounmap;
- }
- }
-
regval = readl_relaxed(REG_CMDQV(cmdqv, PARAM));
cmdqv->num_vintfs = 1 << FIELD_GET(CMDQV_NUM_VINTF_LOG2, regval);
cmdqv->num_vcmdqs = 1 << FIELD_GET(CMDQV_NUM_VCMDQ_LOG2, regval);
@@ -1027,10 +1064,25 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res,
cmdqv->vintfs =
kcalloc(cmdqv->num_vintfs, sizeof(*cmdqv->vintfs), GFP_KERNEL);
if (!cmdqv->vintfs)
- goto free_irq;
+ goto iounmap;
ida_init(&cmdqv->vintf_ids);
+ /*
+ * Request the IRQ only after cmdqv->vintfs is allocated and zeroed, so
+ * the ISR would not walk an uninitialized array.
+ */
+ if (cmdqv->irq > 0) {
+ ret = request_threaded_irq(irq, NULL, tegra241_cmdqv_isr,
+ IRQF_ONESHOT, "tegra241-cmdqv",
+ cmdqv);
+ if (ret) {
+ dev_err(cmdqv->dev, "failed to request irq (%d): %d\n",
+ cmdqv->irq, ret);
+ goto free_vintfs;
+ }
+ }
+
#ifdef CONFIG_IOMMU_DEBUGFS
if (!cmdqv_debugfs_dir) {
cmdqv_debugfs_dir =
@@ -1045,9 +1097,9 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res,
return new_smmu;
-free_irq:
- if (cmdqv->irq > 0)
- free_irq(cmdqv->irq, cmdqv);
+free_vintfs:
+ ida_destroy(&cmdqv->vintf_ids);
+ kfree(cmdqv->vintfs);
iounmap:
iounmap(base);
return NULL;
@@ -1202,13 +1254,15 @@ static int tegra241_vintf_alloc_lvcmdq_user(struct iommufd_hw_queue *hw_queue,
if (ret)
goto unmap_lvcmdq;
+ /* No lockless reader of a user VINTF's lvcmdqs[]; mutex-serialized */
+ vintf->lvcmdqs[lidx] = vcmdq;
+
hw_queue->destroy = &tegra241_vintf_destroy_lvcmdq_user;
mutex_unlock(&vintf->lvcmdq_mutex);
return 0;
unmap_lvcmdq:
tegra241_vcmdq_unmap_lvcmdq(vcmdq);
- tegra241_vintf_deinit_lvcmdq(vintf, lidx);
undepend_vcmdq:
if (vcmdq->prev)
iommufd_hw_queue_undepend(vcmdq, vcmdq->prev, core);
@@ -1254,7 +1308,8 @@ static int tegra241_vintf_init_vsid(struct iommufd_vdevice *vdev)
if (virt_sid > FIELD_MAX(VINTF_SID_MATCH_VIRT_SID))
return -EINVAL;
- WARN_ON_ONCE(master->num_streams != 1);
+ if (master->num_streams != 1)
+ return -EOPNOTSUPP;
/* Find an empty pair of SID_REPLACE and SID_MATCH */
sidx = ida_alloc_max(&vintf->sids, vintf->cmdqv->num_sids_per_vintf - 1,
diff --git a/drivers/iommu/arm/arm-smmu/qcom_iommu.c b/drivers/iommu/arm/arm-smmu/qcom_iommu.c
index 9c1166a3af6c..82c3ffe0c783 100644
--- a/drivers/iommu/arm/arm-smmu/qcom_iommu.c
+++ b/drivers/iommu/arm/arm-smmu/qcom_iommu.c
@@ -200,7 +200,7 @@ static irqreturn_t qcom_iommu_fault(int irq, void *dev)
fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0);
iova = iommu_readq(ctx, ARM_SMMU_CB_FAR);
- if (!report_iommu_fault(ctx->domain, ctx->dev, iova, 0)) {
+ if (report_iommu_fault(ctx->domain, ctx->dev, iova, 0)) {
dev_err_ratelimited(ctx->dev,
"Unhandled context fault: fsr=0x%x, "
"iova=0x%016llx, fsynr=0x%x, cb=%d\n",
@@ -858,7 +858,7 @@ static int qcom_iommu_device_probe(struct platform_device *pdev)
ret = iommu_device_register(&qcom_iommu->iommu, &qcom_iommu_ops, dev);
if (ret) {
dev_err(dev, "Failed to register iommu\n");
- goto err_pm_disable;
+ goto err_sysfs_remove;
}
if (qcom_iommu->local_base) {
@@ -869,6 +869,8 @@ static int qcom_iommu_device_probe(struct platform_device *pdev)
return 0;
+err_sysfs_remove:
+ iommu_device_sysfs_remove(&qcom_iommu->iommu);
err_pm_disable:
pm_runtime_disable(dev);
return ret;
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index b0dca7e7429a..039e84a452d7 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -1647,13 +1647,16 @@ void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
}
if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
- !gfpflags_allow_blocking(gfp) && !coherent)
+ !gfpflags_allow_blocking(gfp) && !coherent) {
page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr,
- gfp, NULL);
- else
+ gfp, NULL);
+ if (!page)
+ return NULL;
+ } else {
cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);
- if (!cpu_addr)
- return NULL;
+ if (!cpu_addr)
+ return NULL;
+ }
*handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot,
dev->coherent_dma_mask);
@@ -2160,6 +2163,19 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
dma_addr_t iova;
int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
size_t size = cookie_msi_granule(domain);
+ static DEFINE_MUTEX(msi_prepare_lock);
+
+ /*
+ * Normally a device's default domain is only ever attached to that
+ * device's own group, and the group mutex held by
+ * iommu_group_mutex_assert()'s callers is enough on its own. A VFIO
+ * type1 container is the one case that breaks that assumption: it
+ * can merge devices from different groups onto one domain, so two
+ * devices' group mutexes don't serialize each other here. A static
+ * lock is sufficient due to the expectation that this is a corner
+ * case that will never be contended in practice.
+ */
+ guard(mutex)(&msi_prepare_lock);
msi_addr &= ~(phys_addr_t)(size - 1);
list_for_each_entry(msi_page, msi_page_list, list)
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index aec1a05d48d0..77e51712040b 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -1421,7 +1421,8 @@ static void copied_context_tear_down(struct intel_iommu *iommu,
assert_spin_locked(&iommu->lock);
did_old = context_domain_id(context);
- context_clear_entry(context);
+ context_clear_present(context);
+ __iommu_flush_cache(iommu, context, sizeof(*context));
if (did_old < cap_ndoms(iommu->cap)) {
iommu->flush.flush_context(iommu, did_old,
@@ -1432,6 +1433,9 @@ static void copied_context_tear_down(struct intel_iommu *iommu,
DMA_TLB_DSI_FLUSH);
}
+ context_clear_entry(context);
+ __iommu_flush_cache(iommu, context, sizeof(*context));
+
clear_context_copied(iommu, bus, devfn);
}
@@ -1728,7 +1732,7 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8
context_clear_present(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
spin_unlock(&iommu->lock);
- intel_context_flush_no_pasid(info, context, did);
+ intel_context_flush_no_pasid(info, context, did, PCI_DEVID(bus, devfn));
context_clear_entry(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
}
@@ -1924,7 +1928,7 @@ static int copy_context_table(struct intel_iommu *iommu,
struct context_entry **tbl,
int bus, bool ext)
{
- int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
+ int tbl_idx, tbl_slot = 0, idx, devfn, ret = 0, did;
struct context_entry *new_ce = NULL, ce;
struct context_entry *old_ce = NULL;
struct root_entry re;
@@ -1940,10 +1944,9 @@ static int copy_context_table(struct intel_iommu *iommu,
if (idx == 0) {
/* First save what we may have and clean up */
if (new_ce) {
- tbl[tbl_idx] = new_ce;
+ tbl[tbl_idx + tbl_slot] = new_ce;
__iommu_flush_cache(iommu, new_ce,
VTD_PAGE_SIZE);
- pos = 1;
}
if (old_ce)
@@ -1965,6 +1968,9 @@ static int copy_context_table(struct intel_iommu *iommu,
}
}
+ /* Track if saving UCTP or LCTP entries in scalable mode */
+ tbl_slot = ext && devfn >= 0x80 ? 1 : 0;
+
ret = -ENOMEM;
old_ce = memremap(old_ce_phys, PAGE_SIZE,
MEMREMAP_WB);
@@ -1993,7 +1999,7 @@ static int copy_context_table(struct intel_iommu *iommu,
new_ce[idx] = ce;
}
- tbl[tbl_idx + pos] = new_ce;
+ tbl[tbl_idx + tbl_slot] = new_ce;
__iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
@@ -2303,7 +2309,7 @@ static void iommu_flush_all(void)
}
}
-static int iommu_suspend(void)
+static int iommu_suspend(void *data)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu = NULL;
@@ -2330,7 +2336,7 @@ static int iommu_suspend(void)
return 0;
}
-static void iommu_resume(void)
+static void iommu_resume(void *data)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu = NULL;
@@ -2361,14 +2367,18 @@ static void iommu_resume(void)
}
}
-static struct syscore_ops iommu_syscore_ops = {
+static const struct syscore_ops iommu_syscore_ops = {
.resume = iommu_resume,
.suspend = iommu_suspend,
};
+static struct syscore iommu_syscore = {
+ .ops = &iommu_syscore_ops,
+};
+
static void __init init_iommu_pm_ops(void)
{
- register_syscore_ops(&iommu_syscore_ops);
+ register_syscore(&iommu_syscore);
}
#else
@@ -3858,6 +3868,7 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)
return &iommu->iommu;
free_table:
+ intel_pasid_teardown_sm_context(dev);
intel_pasid_free_table(dev);
clear_rbtree:
device_rbtree_remove(info);
diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h
index 8e2df00e881c..e696e019e969 100644
--- a/drivers/iommu/intel/iommu.h
+++ b/drivers/iommu/intel/iommu.h
@@ -1320,7 +1320,7 @@ void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start,
unsigned long end);
void intel_context_flush_no_pasid(struct device_domain_info *info,
- struct context_entry *context, u16 did);
+ struct context_entry *context, u16 did, u16 sid);
int intel_iommu_enable_prq(struct intel_iommu *iommu);
int intel_iommu_finish_prq(struct intel_iommu *iommu);
diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c
index 85c3116e351c..48e21743f207 100644
--- a/drivers/iommu/intel/pasid.c
+++ b/drivers/iommu/intel/pasid.c
@@ -749,7 +749,7 @@ static void device_pasid_table_teardown(struct device *dev, u8 bus, u8 devfn)
context_clear_present(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
spin_unlock(&iommu->lock);
- intel_context_flush_no_pasid(info, context, did);
+ intel_context_flush_no_pasid(info, context, did, PCI_DEVID(bus, devfn));
context_clear_entry(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
}
@@ -953,9 +953,12 @@ static void __context_flush_dev_iotlb(struct device_domain_info *info)
* This helper can only be used when IOMMU is working in the legacy mode or
* IOMMU is in scalable mode but all PASID table entries of the device are
* non-present.
+ *
+ * @sid identifies the context entry that was modified, which may be a DMA
+ * alias of @info->dev rather than its own requester ID.
*/
void intel_context_flush_no_pasid(struct device_domain_info *info,
- struct context_entry *context, u16 did)
+ struct context_entry *context, u16 did, u16 sid)
{
struct intel_iommu *iommu = info->iommu;
@@ -965,7 +968,7 @@ void intel_context_flush_no_pasid(struct device_domain_info *info,
* when operating in scalable mode. Therefore the @did value doesn't
* matter in scalable mode.
*/
- iommu->flush.flush_context(iommu, did, PCI_DEVID(info->bus, info->devfn),
+ iommu->flush.flush_context(iommu, did, sid,
DMA_CCMD_MASK_NOBIT, DMA_CCMD_DEVICE_INVL);
/*
diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c
index c40515bf5017..3216deab98e3 100644
--- a/drivers/iommu/iommufd/device.c
+++ b/drivers/iommu/iommufd/device.c
@@ -148,29 +148,22 @@ static void iommufd_device_remove_vdev(struct iommufd_device *idev)
if (!idev->vdev)
goto out_unlock;
- vdev = iommufd_get_vdevice(idev->ictx, idev->vdev->obj.id);
+ vdev = idev->vdev;
+
/*
* An ongoing vdev destroy ioctl has removed the vdev from the object
* xarray, but has not finished iommufd_vdevice_destroy() yet as it
* needs the same mutex. We exit the locking then wait on wait_cnt
* reference for the vdev destruction.
*/
- if (IS_ERR(vdev))
- goto out_unlock;
-
- /* Should never happen */
- if (WARN_ON(vdev != idev->vdev)) {
- iommufd_put_object(idev->ictx, &vdev->obj);
+ if (iommufd_try_inc_users(idev->ictx, &vdev->obj))
goto out_unlock;
- }
/*
* vdev is still alive. Hold a users refcount to prevent racing with
* userspace destruction, then use iommufd_object_tombstone_user() to
* destroy it and leave a tombstone.
*/
- refcount_inc(&vdev->obj.users);
- iommufd_put_object(idev->ictx, &vdev->obj);
mutex_unlock(&idev->igroup->lock);
iommufd_object_tombstone_user(idev->ictx, &vdev->obj);
return;
diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h
index 53d37e77c4b9..c208563eb367 100644
--- a/drivers/iommu/iommufd/iommufd_private.h
+++ b/drivers/iommu/iommufd/iommufd_private.h
@@ -181,6 +181,7 @@ static inline bool iommufd_lock_obj(struct iommufd_object *obj)
return true;
}
+int iommufd_try_inc_users(struct iommufd_ctx *ictx, struct iommufd_object *obj);
struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id,
enum iommufd_object_type type);
static inline void iommufd_put_object(struct iommufd_ctx *ictx,
@@ -695,14 +696,6 @@ void iommufd_vdevice_abort(struct iommufd_object *obj);
int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd);
void iommufd_hw_queue_destroy(struct iommufd_object *obj);
-static inline struct iommufd_vdevice *
-iommufd_get_vdevice(struct iommufd_ctx *ictx, u32 id)
-{
- return container_of(iommufd_get_object(ictx, id,
- IOMMUFD_OBJ_VDEVICE),
- struct iommufd_vdevice, obj);
-}
-
#ifdef CONFIG_IOMMUFD_TEST
int iommufd_test(struct iommufd_ucmd *ucmd);
void iommufd_selftest_destroy(struct iommufd_object *obj);
diff --git a/drivers/iommu/iommufd/main.c b/drivers/iommu/iommufd/main.c
index ce775fbbae94..02d06a02d5e6 100644
--- a/drivers/iommu/iommufd/main.c
+++ b/drivers/iommu/iommufd/main.c
@@ -180,6 +180,26 @@ struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id,
return obj;
}
+/*
+ * Increment the users count of an object outside the context of an ioctl that
+ * has already locked it. The users refcount cannot be increased on an already
+ * created object unless the object is installed in the xarray, otherwise things
+ * are racing with a parallel destruction.
+ */
+int iommufd_try_inc_users(struct iommufd_ctx *ictx, struct iommufd_object *obj)
+{
+ struct iommufd_object *cur;
+
+ xa_lock(&ictx->objects);
+ cur = xa_load(&ictx->objects, obj->id);
+ if (cur == obj)
+ refcount_inc(&obj->users);
+ xa_unlock(&ictx->objects);
+ if (cur != obj)
+ return -EBUSY;
+ return 0;
+}
+
static int iommufd_object_dec_wait(struct iommufd_ctx *ictx,
struct iommufd_object *to_destroy)
{
diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c
index 35c42ff4355a..cd3d1380cf6c 100644
--- a/drivers/iommu/iommufd/selftest.c
+++ b/drivers/iommu/iommufd/selftest.c
@@ -10,6 +10,7 @@
#include <linux/iommu.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
+#include <linux/sizes.h>
#include <linux/xarray.h>
#include <uapi/linux/iommufd.h>
@@ -1806,6 +1807,9 @@ static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id,
if (!page_size || !length || iova % page_size || length % page_size ||
!uptr)
return -EINVAL;
+ max = length / page_size;
+ if (max > SZ_16M * BITS_PER_BYTE)
+ return -EOVERFLOW;
hwpt = get_md_pagetable(ucmd, mockpt_id, &mock);
if (IS_ERR(hwpt))
@@ -1816,7 +1820,6 @@ static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id,
goto out_put;
}
- max = length / page_size;
tmp = kvzalloc(DIV_ROUND_UP(max, BITS_PER_LONG) * sizeof(unsigned long),
GFP_KERNEL_ACCOUNT);
if (!tmp) {
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index d5e2fc9b0111..1dfa6f69c28f 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -719,7 +719,7 @@ static int msm_iommu_probe(struct platform_device *pdev)
iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);
if (!iommu)
- return -ENODEV;
+ return -ENOMEM;
iommu->dev = &pdev->dev;
INIT_LIST_HEAD(&iommu->ctx_list);
diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c
index 44b965a2db92..ad2dbd29fef8 100644
--- a/drivers/iommu/mtk_iommu_v1.c
+++ b/drivers/iommu/mtk_iommu_v1.c
@@ -88,7 +88,7 @@ struct dma_iommu_mapping {
/* MTK generation one iommu HW only support 4K size mapping */
#define MT2701_IOMMU_PAGE_SHIFT 12
#define MT2701_IOMMU_PAGE_SIZE (1UL << MT2701_IOMMU_PAGE_SHIFT)
-#define MT2701_LARB_NR_MAX 3
+#define MT2701_LARB_NR_MAX 4
/*
* MTK m4u support 4GB iova address space, and only support 4K page
generated by cgit v1.2.3 (git 2.39.1) at 2026-09-17 20:22:52 +0000