From 3fb1ff24956738e0b88d62aa706b475814d0607a Mon Sep 17 00:00:00 2001 From: Minwoo Im Date: Wed, 14 Aug 2024 13:51:58 +0900 Subject: [PATCH] vfio: Reset vfio_container in detaching last group If application follows the calling sequence, it fails in getting device fd as the following logs, Assumming that there's only one vfio group and container with a single device attached. nvme_ctrl_init (e.g., unvme add ) nvme_close (e.g., unvme del ) nvme_ctrl_init I nvme_pci_init (src/nvme/core.c:510): nvme/core: pci class code is 0x010802 I vfio_get_device_fd (src/iommu/vfio.c:430): iommu/vfio: vfio iommu group is /dev/vfio/5 I vfio_group_set_container (src/iommu/vfio.c:290): iommu/vfio: adding group '/dev/vfio/5' to container D vfio_get_device_fd (src/iommu/vfio.c:438): iommu/vfio: failed to get device fd D vfio_pci_open (src/vfio/pci.c:151): vfio/pci: failed to get device fd This is because kernel removes `container->iommu_driver` by setting it to NULL if this is the last vfio group to be detached from the current container. This will fail in the following vfio kernel code: int vfio_group_use_container(struct vfio_group *group) { lockdep_assert_held(&group->group_lock); /* * The container fd has been assigned with VFIO_GROUP_SET_CONTAINER but * VFIO_SET_IOMMU hasn't been done yet. */ if (!group->container->iommu_driver) return -EINVAL; ... VFIO_SET_IOMMU ioctl is required to re-initailize `container->iommu` instance. But, current `libvfn` is skipping vfio_iommu_type1_init in case vfio->iommu_set is set. This patch resets the vfio_container instance when the last vfio group is detached from the current container just like kernel does. Signed-off-by: Minwoo Im --- src/iommu/vfio.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/iommu/vfio.c b/src/iommu/vfio.c index e5de4e1..5616662 100644 --- a/src/iommu/vfio.c +++ b/src/iommu/vfio.c @@ -61,6 +61,7 @@ struct vfio_container { #define VFN_MAX_VFIO_GROUPS 64 struct vfio_group groups[VFN_MAX_VFIO_GROUPS]; + int nr_groups; pthread_mutex_t lock; uint64_t next, next_ephemeral, nephemerals; @@ -72,6 +73,7 @@ struct vfio_container { static struct vfio_container vfio_default_container = { .fd = -1, .name = "default", + .nr_groups = 0, }; #ifdef VFIO_IOMMU_INFO_CAPS @@ -352,6 +354,7 @@ static int vfio_get_group_fd(struct vfio_container *vfio, goto free_group_path; } + atomic_inc(&vfio->nr_groups); return group->fd; free_group_path: @@ -455,9 +458,24 @@ static int vfio_put_device_fd(struct iommu_ctx *ctx, const char *bdf) return -1; } - if (atomic_dec_fetch(&group->nr_devs) == 0) + if (atomic_dec_fetch(&group->nr_devs) == 0) { close(group->fd); + /* + * Kernel vfio driver will remove IOMMU driver and data from + * the container if the current detached group is the last one + * of the container. We should reset the container instance + * for cases re-opening vfio container only if it's + * `vfio_default_container`. + */ + if (atomic_dec_fetch(&vfio->nr_groups) == 0) { + close(vfio->fd); + + vfio->fd = -1; + vfio->iommu_set = false; + } + } + return 0; }