vfio: Reset vfio_container in detaching last group

If application follows the calling sequence, it fails in getting device
fd as the following logs, Assumming that there's only one vfio group and
container with a single device attached.

 nvme_ctrl_init (e.g., unvme add <bdf>)
 nvme_close     (e.g., unvme del <bdf>)
 nvme_ctrl_init

 I nvme_pci_init (src/nvme/core.c:510): nvme/core: pci class code is 0x010802
 I vfio_get_device_fd (src/iommu/vfio.c:430): iommu/vfio: vfio iommu group is /dev/vfio/5
 I vfio_group_set_container (src/iommu/vfio.c:290): iommu/vfio: adding group '/dev/vfio/5' to container
 D vfio_get_device_fd (src/iommu/vfio.c:438): iommu/vfio: failed to get device fd
 D vfio_pci_open (src/vfio/pci.c:151): vfio/pci: failed to get device fd

This is because kernel removes `container->iommu_driver` by setting it
to NULL if this is the last vfio group to be detached from the current
container.  This will fail in the following vfio kernel code:

int vfio_group_use_container(struct vfio_group *group)
{
        lockdep_assert_held(&group->group_lock);

        /*
         * The container fd has been assigned with VFIO_GROUP_SET_CONTAINER but
         * VFIO_SET_IOMMU hasn't been done yet.
         */
        if (!group->container->iommu_driver)
                return -EINVAL;

	...

VFIO_SET_IOMMU ioctl is required to re-initailize `container->iommu`
instance.  But, current `libvfn` is skipping vfio_iommu_type1_init in
case vfio->iommu_set is set.

This patch resets the vfio_container instance when the last vfio group
is detached from the current container just like kernel does.

Signed-off-by: Minwoo Im <[email protected]>

src/iommu/vfio.c

@@ -61,6 +61,7 @@ struct vfio_container {
 
 #define VFN_MAX_VFIO_GROUPS 64
 	struct vfio_group groups[VFN_MAX_VFIO_GROUPS];
+	int nr_groups;
 
 	pthread_mutex_t lock;
 	uint64_t next, next_ephemeral, nephemerals;
@@ -72,6 +73,7 @@ struct vfio_container {
 static struct vfio_container vfio_default_container = {
 	.fd = -1,
 	.name = "default",
+	.nr_groups = 0,
 };
 
 #ifdef VFIO_IOMMU_INFO_CAPS
@@ -352,6 +354,7 @@ static int vfio_get_group_fd(struct vfio_container *vfio,
 		goto free_group_path;
 	}
 
+	atomic_inc(&vfio->nr_groups);
 	return group->fd;
 
 free_group_path:
@@ -455,9 +458,24 @@ static int vfio_put_device_fd(struct iommu_ctx *ctx, const char *bdf)
 		return -1;
 	}
 
-	if (atomic_dec_fetch(&group->nr_devs) == 0)
+	if (atomic_dec_fetch(&group->nr_devs) == 0) {
 		close(group->fd);
 
+		/*
+		 * Kernel vfio driver will remove IOMMU driver and data from
+		 * the container if the current detached group is the last one
+		 * of the container.  We should reset the container instance
+		 * for cases re-opening vfio container only if it's
+		 * `vfio_default_container`.
+		 */
+		if (atomic_dec_fetch(&vfio->nr_groups) == 0) {
+			close(vfio->fd);
+
+			vfio->fd = -1;
+			vfio->iommu_set = false;
+		}
+	}
+
 	return 0;
 }