Skip to content

Commit b6fd468

Browse files
jgunthorpejoergroedel
jgunthorpe
authored and
joergroedel
committed
iommu/vt-d: Split piotlb invalidation into range and all
Currently these call chains are muddled up by using npages=-1, but only one caller has the possibility to do both options. Simplify qi_flush_piotlb() to qi_flush_piotlb_all() since all callers pass npages=-1. Split qi_batch_add_piotlb() into qi_batch_add_piotlb_all() and related helpers. Signed-off-by: Jason Gunthorpe <jgg@nvidia.com> Link: https://lore.kernel.org/r/1-v1-f175e27af136+11647-iommupt_inv_vtd_jgg@nvidia.com Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com> Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>
1 parent 51234c4 commit b6fd468

5 files changed

Lines changed: 41 additions & 49 deletions

File tree

‎drivers/iommu/intel/cache.c‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -330,15 +330,17 @@ static void qi_batch_add_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid
330330
qi_batch_increment_index(iommu, batch);
331331
}
332332

333+
static void qi_batch_add_piotlb_all(struct intel_iommu *iommu, u16 did,
334+
u32 pasid, struct qi_batch *batch)
335+
{
336+
qi_desc_piotlb_all(did, pasid, &batch->descs[batch->index]);
337+
qi_batch_increment_index(iommu, batch);
338+
}
339+
333340
static void qi_batch_add_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid,
334341
u64 addr, unsigned long npages, bool ih,
335342
struct qi_batch *batch)
336343
{
337-
/*
338-
* npages == -1 means a PASID-selective invalidation, otherwise,
339-
* a positive value for Page-selective-within-PASID invalidation.
340-
* 0 is not a valid input.
341-
*/
342344
if (!npages)
343345
return;
344346

@@ -378,8 +380,12 @@ static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag *
378380
u64 type = DMA_TLB_PSI_FLUSH;
379381

380382
if (intel_domain_use_piotlb(domain)) {
381-
qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid, addr,
382-
pages, ih, domain->qi_batch);
383+
if (pages == -1)
384+
qi_batch_add_piotlb_all(iommu, tag->domain_id,
385+
tag->pasid, domain->qi_batch);
386+
else
387+
qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid,
388+
addr, pages, ih, domain->qi_batch);
383389
return;
384390
}
385391

‎drivers/iommu/intel/dmar.c‎

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1551,23 +1551,12 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
15511551
qi_submit_sync(iommu, &desc, 1, 0);
15521552
}
15531553

1554-
/* PASID-based IOTLB invalidation */
1555-
void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
1556-
unsigned long npages, bool ih)
1554+
/* PASID-selective IOTLB invalidation */
1555+
void qi_flush_piotlb_all(struct intel_iommu *iommu, u16 did, u32 pasid)
15571556
{
1558-
struct qi_desc desc = {.qw2 = 0, .qw3 = 0};
1557+
struct qi_desc desc = {};
15591558

1560-
/*
1561-
* npages == -1 means a PASID-selective invalidation, otherwise,
1562-
* a positive value for Page-selective-within-PASID invalidation.
1563-
* 0 is not a valid input.
1564-
*/
1565-
if (WARN_ON(!npages)) {
1566-
pr_err("Invalid input npages = %ld\n", npages);
1567-
return;
1568-
}
1569-
1570-
qi_desc_piotlb(did, pasid, addr, npages, ih, &desc);
1559+
qi_desc_piotlb_all(did, pasid, &desc);
15711560
qi_submit_sync(iommu, &desc, 1, 0);
15721561
}
15731562

‎drivers/iommu/intel/iommu.h‎

Lines changed: 20 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1077,31 +1077,29 @@ static inline void qi_desc_dev_iotlb(u16 sid, u16 pfsid, u16 qdep, u64 addr,
10771077
desc->qw3 = 0;
10781078
}
10791079

1080+
/* PASID-selective IOTLB invalidation */
1081+
static inline void qi_desc_piotlb_all(u16 did, u32 pasid, struct qi_desc *desc)
1082+
{
1083+
desc->qw0 = QI_EIOTLB_PASID(pasid) | QI_EIOTLB_DID(did) |
1084+
QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | QI_EIOTLB_TYPE;
1085+
desc->qw1 = 0;
1086+
}
1087+
1088+
/* Page-selective-within-PASID IOTLB invalidation */
10801089
static inline void qi_desc_piotlb(u16 did, u32 pasid, u64 addr,
10811090
unsigned long npages, bool ih,
10821091
struct qi_desc *desc)
10831092
{
1084-
if (npages == -1) {
1085-
desc->qw0 = QI_EIOTLB_PASID(pasid) |
1086-
QI_EIOTLB_DID(did) |
1087-
QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
1088-
QI_EIOTLB_TYPE;
1089-
desc->qw1 = 0;
1090-
} else {
1091-
int mask = ilog2(__roundup_pow_of_two(npages));
1092-
unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));
1093-
1094-
if (WARN_ON_ONCE(!IS_ALIGNED(addr, align)))
1095-
addr = ALIGN_DOWN(addr, align);
1096-
1097-
desc->qw0 = QI_EIOTLB_PASID(pasid) |
1098-
QI_EIOTLB_DID(did) |
1099-
QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |
1100-
QI_EIOTLB_TYPE;
1101-
desc->qw1 = QI_EIOTLB_ADDR(addr) |
1102-
QI_EIOTLB_IH(ih) |
1103-
QI_EIOTLB_AM(mask);
1104-
}
1093+
int mask = ilog2(__roundup_pow_of_two(npages));
1094+
unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));
1095+
1096+
if (WARN_ON_ONCE(!IS_ALIGNED(addr, align)))
1097+
addr = ALIGN_DOWN(addr, align);
1098+
1099+
desc->qw0 = QI_EIOTLB_PASID(pasid) | QI_EIOTLB_DID(did) |
1100+
QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) | QI_EIOTLB_TYPE;
1101+
desc->qw1 = QI_EIOTLB_ADDR(addr) | QI_EIOTLB_IH(ih) |
1102+
QI_EIOTLB_AM(mask);
11051103
}
11061104

11071105
static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid,
@@ -1163,8 +1161,7 @@ void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
11631161
void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
11641162
u16 qdep, u64 addr, unsigned mask);
11651163

1166-
void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
1167-
unsigned long npages, bool ih);
1164+
void qi_flush_piotlb_all(struct intel_iommu *iommu, u16 did, u32 pasid);
11681165

11691166
void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,
11701167
u32 pasid, u16 qdep, u64 addr,

‎drivers/iommu/intel/pasid.c‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,7 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,
282282
pasid_cache_invalidation_with_pasid(iommu, did, pasid);
283283

284284
if (pgtt == PASID_ENTRY_PGTT_PT || pgtt == PASID_ENTRY_PGTT_FL_ONLY)
285-
qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);
285+
qi_flush_piotlb_all(iommu, did, pasid);
286286
else
287287
iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
288288

@@ -308,7 +308,7 @@ static void pasid_flush_caches(struct intel_iommu *iommu,
308308

309309
if (cap_caching_mode(iommu->cap)) {
310310
pasid_cache_invalidation_with_pasid(iommu, did, pasid);
311-
qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);
311+
qi_flush_piotlb_all(iommu, did, pasid);
312312
} else {
313313
iommu_flush_write_buffer(iommu);
314314
}
@@ -342,7 +342,7 @@ static void intel_pasid_flush_present(struct intel_iommu *iommu,
342342
* Addr[63:12]=0x7FFFFFFF_FFFFF) to affected functions
343343
*/
344344
pasid_cache_invalidation_with_pasid(iommu, did, pasid);
345-
qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);
345+
qi_flush_piotlb_all(iommu, did, pasid);
346346

347347
devtlb_invalidation_with_pasid(iommu, dev, pasid);
348348
}

‎drivers/iommu/intel/prq.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ void intel_iommu_drain_pasid_prq(struct device *dev, u32 pasid)
113113
qi_desc_dev_iotlb(sid, info->pfsid, info->ats_qdep, 0,
114114
MAX_AGAW_PFN_WIDTH, &desc[2]);
115115
} else {
116-
qi_desc_piotlb(did, pasid, 0, -1, 0, &desc[1]);
116+
qi_desc_piotlb_all(did, pasid, &desc[1]);
117117
qi_desc_dev_iotlb_pasid(sid, info->pfsid, pasid, info->ats_qdep,
118118
0, MAX_AGAW_PFN_WIDTH, &desc[2]);
119119
}

0 commit comments

Comments
 (0)