diff --git a/drivers/vdpa/mlx5/core/mlx5_vdpa.h b/drivers/vdpa/mlx5/core/mlx5_vdpa.h index ca56242972b3af646b6808b33d7f7c4947c45b6c..869219529ff12180cd1e7b01d26844a2c45d2620 100644 --- a/drivers/vdpa/mlx5/core/mlx5_vdpa.h +++ b/drivers/vdpa/mlx5/core/mlx5_vdpa.h @@ -11,6 +11,8 @@ #define MLX5V_ETH_HARD_MTU (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN) +extern int mlx5_vdpa_max_iotlb_entries; + struct mlx5_vdpa_direct_mr { u64 start; u64 end; diff --git a/drivers/vdpa/mlx5/core/mr.c b/drivers/vdpa/mlx5/core/mr.c index 165730656934528df6e824e5f69a223d558bf7c4..d99bc889a2824ab2b6908c2d7aa48b0573cd6f27 100644 --- a/drivers/vdpa/mlx5/core/mr.c +++ b/drivers/vdpa/mlx5/core/mr.c @@ -553,6 +553,9 @@ static int _mlx5_vdpa_create_dvq_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr = &mvdev->mr; int err; + if (mlx5_vdpa_max_iotlb_entries < 2) + return -EINVAL; + if (mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP] != asid) return 0; diff --git a/drivers/vdpa/mlx5/core/resources.c b/drivers/vdpa/mlx5/core/resources.c index d5a59c9035fbed9af6d762467e8e517d4c8e3ecc..00b66438ab530664d6e91f4d72e19d8be908a453 100644 --- a/drivers/vdpa/mlx5/core/resources.c +++ b/drivers/vdpa/mlx5/core/resources.c @@ -3,8 +3,14 @@ #include #include +#include #include "mlx5_vdpa.h" +int mlx5_vdpa_max_iotlb_entries = 2048; +module_param_named(max_iotlb_entries, mlx5_vdpa_max_iotlb_entries, int, 0444); +MODULE_PARM_DESC(max_iotlb_entries, + "Maximum number of iotlb entries. (default: 2048)"); + static int alloc_pd(struct mlx5_vdpa_dev *dev, u32 *pdn, u16 uid) { struct mlx5_core_dev *mdev = dev->mdev; @@ -229,7 +235,10 @@ int mlx5_vdpa_destroy_mkey(struct mlx5_vdpa_dev *mvdev, u32 mkey) static int init_ctrl_vq(struct mlx5_vdpa_dev *mvdev) { - mvdev->cvq.iotlb = vhost_iotlb_alloc(0, 0); + if (mlx5_vdpa_max_iotlb_entries < 2) + return -EINVAL; + + mvdev->cvq.iotlb = vhost_iotlb_alloc(mlx5_vdpa_max_iotlb_entries, 0); if (!mvdev->cvq.iotlb) return -ENOMEM; diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c index 421ab01ef06ba50038b486d6bda6379d53abd4c1..0ddec4199f7e4ec445f7b33d4875fde3c130becd 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c @@ -34,7 +34,7 @@ MODULE_PARM_DESC(batch_mapping, "Batched mapping 1 -Enable; 0 - Disable"); static int max_iotlb_entries = 2048; module_param(max_iotlb_entries, int, 0444); MODULE_PARM_DESC(max_iotlb_entries, - "Maximum number of iotlb entries for each address space. 0 means unlimited. (default: 2048)"); + "Maximum number of iotlb entries for each address space. (default: 2048)"); static bool use_va = true; module_param(use_va, bool, 0444); @@ -199,6 +199,8 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr, if (!dev_attr->alloc_size) return ERR_PTR(-EINVAL); + if (max_iotlb_entries < 2) + return ERR_PTR(-EINVAL); if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) { if (config->device_features & @@ -259,8 +261,14 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr, if (!vdpasim->iommu_pt) goto err_iommu; - for (i = 0; i < vdpasim->dev_attr.nas; i++) + for (i = 0; i < vdpasim->dev_attr.nas; i++) { vhost_iotlb_init(&vdpasim->iommu[i], max_iotlb_entries, 0); + ret = vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX, + 0, VHOST_MAP_RW); + if (ret) + goto err_iommu; + vdpasim->iommu_pt[i] = true; + } for (i = 0; i < dev_attr->nvqs; i++) vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0], diff --git a/drivers/vdpa/vdpa_user/iova_domain.c b/drivers/vdpa/vdpa_user/iova_domain.c index 5e4a77b9bae6bea6402626b3ad00134d67aa31d2..7d62b0c59cd4b55f9c405639a71b915c4053a562 100644 --- a/drivers/vdpa/vdpa_user/iova_domain.c +++ b/drivers/vdpa/vdpa_user/iova_domain.c @@ -12,11 +12,17 @@ #include #include #include +#include #include #include #include "iova_domain.h" +static int max_iotlb_entries = 2048; +module_param(max_iotlb_entries, int, 0444); +MODULE_PARM_DESC(max_iotlb_entries, + "Maximum number of iotlb entries. (default: 2048)"); + static int vduse_iotlb_add_range(struct vduse_iova_domain *domain, u64 start, u64 last, u64 addr, unsigned int perm, @@ -561,11 +567,14 @@ vduse_domain_create(unsigned long iova_limit, size_t bounce_size) if (iova_limit <= bounce_size) return NULL; + if (max_iotlb_entries <= 0) + return NULL; + domain = kzalloc(sizeof(*domain), GFP_KERNEL); if (!domain) return NULL; - domain->iotlb = vhost_iotlb_alloc(0, 0); + domain->iotlb = vhost_iotlb_alloc(max_iotlb_entries, 0); if (!domain->iotlb) goto err_iotlb; diff --git a/drivers/vhost/iotlb.c b/drivers/vhost/iotlb.c index ea61330a34310226134411a20881cecfcbd374e5..a6228dd24931670df792d887816fdd37f839b6e5 100644 --- a/drivers/vhost/iotlb.c +++ b/drivers/vhost/iotlb.c @@ -20,6 +20,14 @@ INTERVAL_TREE_DEFINE(struct vhost_iotlb_map, rb, __u64, __subtree_last, START, LAST, static inline, vhost_iotlb_itree); +static void vhost_iotlb_map_unlink(struct vhost_iotlb *iotlb, + struct vhost_iotlb_map *map) +{ + vhost_iotlb_itree_remove(map, &iotlb->root); + list_del(&map->link); + iotlb->nmaps--; +} + /** * vhost_iotlb_map_free - remove a map node and free it * @iotlb: the IOTLB @@ -28,10 +36,8 @@ INTERVAL_TREE_DEFINE(struct vhost_iotlb_map, void vhost_iotlb_map_free(struct vhost_iotlb *iotlb, struct vhost_iotlb_map *map) { - vhost_iotlb_itree_remove(map, &iotlb->root); - list_del(&map->link); + vhost_iotlb_map_unlink(iotlb, map); kfree(map); - iotlb->nmaps--; } EXPORT_SYMBOL_GPL(vhost_iotlb_map_free); @@ -57,14 +63,25 @@ int vhost_iotlb_add_range_ctx(struct vhost_iotlb *iotlb, if (last < start) return -EFAULT; + if (!iotlb->limit) + return -EINVAL; + /* If the range being mapped is [0, ULONG_MAX], split it into two entries * otherwise its size would overflow u64. */ if (start == 0 && last == ULONG_MAX) { u64 mid = last / 2; - int err = vhost_iotlb_add_range_ctx(iotlb, start, mid, addr, - perm, opaque); + int err; + + if (iotlb->limit < 2) + return -ENOSPC; + if (!(iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) && + iotlb->nmaps > iotlb->limit - 2) + return -ENOSPC; + + err = vhost_iotlb_add_range_ctx(iotlb, start, mid, addr, + perm, opaque); if (err) return err; @@ -72,17 +89,19 @@ int vhost_iotlb_add_range_ctx(struct vhost_iotlb *iotlb, start = mid + 1; } - if (iotlb->limit && - iotlb->nmaps == iotlb->limit && - iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) { - map = list_first_entry(&iotlb->list, typeof(*map), link); - vhost_iotlb_map_free(iotlb, map); + if (iotlb->nmaps >= iotlb->limit) { + if (iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) { + map = list_first_entry(&iotlb->list, typeof(*map), link); + vhost_iotlb_map_unlink(iotlb, map); + } else { + return -ENOSPC; + } + } else { + map = kmalloc(sizeof(*map), GFP_ATOMIC); + if (!map) + return -ENOMEM; } - map = kmalloc(sizeof(*map), GFP_ATOMIC); - if (!map) - return -ENOMEM; - map->start = start; map->size = last - start + 1; map->last = last; diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 809815617dcddca15ba31e18112020d1e9f53af1..24da2b9cb541fec53413051ed8d70a7125923ea1 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -34,6 +34,11 @@ enum { #define VHOST_VDPA_DEV_MAX (1U << MINORBITS) +static int max_iotlb_entries = 2048; +module_param(max_iotlb_entries, int, 0444); +MODULE_PARM_DESC(max_iotlb_entries, + "Maximum number of iotlb entries. (default: 2048)"); + #define VHOST_VDPA_IOTLB_BUCKETS 16 struct vhost_vdpa_as { @@ -108,12 +113,14 @@ static struct vhost_vdpa_as *vhost_vdpa_alloc_as(struct vhost_vdpa *v, u32 asid) if (asid >= v->vdpa->nas) return NULL; + if (max_iotlb_entries <= 0) + return NULL; as = kmalloc(sizeof(*as), GFP_KERNEL); if (!as) return NULL; - vhost_iotlb_init(&as->iotlb, 0, 0); + vhost_iotlb_init(&as->iotlb, max_iotlb_entries, 0); as->id = asid; hlist_add_head(&as->hash_link, head); diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 147cfb64bba2d5e5d3cd69bef5b1a6d331cf51ae..b6ef569d5969873bbeec10fb84226f11dc76726c 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -293,7 +293,7 @@ static void __vhost_worker_flush(struct vhost_worker *worker) vhost_worker_queue(worker, &flush.work); /* - * Drop mutex in case our worker is killed and it needs to take the + * Drop mutex in case our worker is killed and cleanup needs to take the * mutex to force cleanup. */ mutex_unlock(&worker->mutex); @@ -977,6 +977,9 @@ EXPORT_SYMBOL_GPL(vhost_dev_set_owner); static struct vhost_iotlb *iotlb_alloc(void) { + if (max_iotlb_entries <= 0) + return NULL; + return vhost_iotlb_alloc(max_iotlb_entries, VHOST_IOTLB_FLAG_RETIRE); } @@ -995,6 +998,7 @@ void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *umem) vhost_dev_cleanup(dev); dev->umem = umem; + /* We don't need VQ locks below since vhost_dev_cleanup makes sure * VQs aren't running. */ @@ -1268,7 +1272,7 @@ static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq, int type) { void __user *uaddr = vhost_vq_meta_fetch(vq, - (u64)(uintptr_t)addr, size, type); + (u64)(uintptr_t)addr, size, type); if (uaddr) return uaddr; @@ -1283,7 +1287,7 @@ static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq, } else { \ __typeof__(ptr) to = \ (__typeof__(ptr)) __vhost_get_user(vq, ptr, \ - sizeof(*ptr), VHOST_ADDR_USED); \ + sizeof(*ptr), VHOST_ADDR_USED); \ if (to != NULL) \ ret = __put_user(x, to); \ else \ @@ -1548,7 +1552,7 @@ ssize_t vhost_chr_write_iter(struct vhost_dev *dev, EXPORT_SYMBOL(vhost_chr_write_iter); __poll_t vhost_chr_poll(struct file *file, struct vhost_dev *dev, - poll_table *wait) + poll_table *wait) { __poll_t mask = 0; @@ -1791,6 +1795,8 @@ static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) return -EOPNOTSUPP; if (mem.nregions > max_mem_regions) return -E2BIG; + if (max_iotlb_entries <= 0) + return -EINVAL; newmem = kvzalloc(struct_size(newmem, regions, mem.nregions), GFP_KERNEL); if (!newmem) @@ -2085,7 +2091,11 @@ int vhost_init_device_iotlb(struct vhost_dev *d) struct vhost_iotlb *niotlb, *oiotlb; int i; + if (max_iotlb_entries <= 0) + return -EINVAL; + niotlb = iotlb_alloc(); + if (!niotlb) return -ENOMEM; @@ -2180,7 +2190,7 @@ long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp) EXPORT_SYMBOL_GPL(vhost_dev_ioctl); /* TODO: This is really inefficient. We need something like get_user() - * (instruction directly accesses the data, with an exception table entry + * (instruction directly accessing the data, with an exception table entry * returning -EFAULT). See Documentation/arch/x86/exception-tables.rst. */ static int set_bit_to_user(int nr, void __user *addr)