From ecab35e22ea4b82566c337adfded1ab64ac1b8c7 Mon Sep 17 00:00:00 2001 From: Minwoo Im Date: Tue, 14 Jan 2025 11:20:36 +0900 Subject: [PATCH] nvme: export nvme_configure_[sq|cq] to public They have been used when creating admin queue and I/O queues in nvme_configure_adminq() and nvme_create_io[sq|cq]() to create an instance of SQ and CQ to be managed in libvfn with ``ctrl->sq[qid]`` and ``ctrl->cq[qid]``. The problem happens when an application handles admin CQ only when an interrupt (e.g., MSI-X) comes with its own CQ handling behavior on top of it. In this case, nvme_create_iosq() and nvme_create_iocq() high-level APIs issue create I/O queue admin command by __admin() where the command is handled in a sync way by reaping the CQ entry right after the command submission. It goes to collision of the CQ between application CQ reaper class and libvfn __admin() callstack. To solve this issue, export nvme_configure_[sq|cq]() to public to let application manages admin command submission for creating queues. Signed-off-by: Minwoo Im --- include/vfn/nvme/ctrl.h | 43 +++++++++++++++++++++++++++++++++++++++++ src/nvme/core.c | 6 +++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/include/vfn/nvme/ctrl.h b/include/vfn/nvme/ctrl.h index f0df449..0702ace 100644 --- a/include/vfn/nvme/ctrl.h +++ b/include/vfn/nvme/ctrl.h @@ -215,6 +215,49 @@ int nvme_del_ctrl(struct nvme_ctrl *ctrl); */ int nvme_reset(struct nvme_ctrl *ctrl); +/** + * nvme_configure_sq - Configure a submission queue instance + * @ctrl: Controller to configure a submission queue instance + * @qid: Queue identifier + * @qsize: Queue size + * @cq: Corresponding completion queue instance + * @flags: Submission queue configuration flags + * + * Create a submission queue instance for the given @qid. This allocates + * memory spaces for the submission queue and map it to IOMMU page table for + * the DMA operations. + * + * The newly created SQ instance can be referred by ``ctrl->sq[qid]``. + * + * **Note** This helper does not issue a Create I/O Submission Queue admin + * command to the admin submission queue. Caller should prepare command + * submission with the instance to be configured. + * + * Return: ``0`` on success, ``-1`` on error and set ``errno``. + */ +int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize, + struct nvme_cq *cq, unsigned long flags); + +/** + * nvme_configure_cq - Configure a completion queue instance + * @ctrl: Controller to configure a completion queue instance + * @qid: Queue identifier + * @qsize: Queue size + * @vector: interrupt vector + * + * Create a completion queue instance for the given @qid. This allocates + * memory spaces for the completion queue and map it to IOMMU page table for + * the DMA operations. + * + * The newly created CQ instance can be referred by ``ctrl->cq[qid]``. + * + * **Note** This helper does not issue a Create I/O Completion Queue admin + * command to the admin submission queue. Caller should prepare command + * submission with the instance to be configured. + * + * Return: ``0`` on success, ``-1`` on error and set ``errno``. + */ +int nvme_configure_cq(struct nvme_ctrl *ctrl, int qid, int qsize, int vector); /** * nvme_configure_adminq - Configure admin sq/cq pair diff --git a/src/nvme/core.c b/src/nvme/core.c index 449c1f2..18cff72 100644 --- a/src/nvme/core.c +++ b/src/nvme/core.c @@ -110,7 +110,7 @@ int nvme_del_ctrl(struct nvme_ctrl *ctrl) return 0; } -static int nvme_configure_cq(struct nvme_ctrl *ctrl, int qid, int qsize, int vector) +int nvme_configure_cq(struct nvme_ctrl *ctrl, int qid, int qsize, int vector) { struct nvme_cq *cq = &ctrl->cq[qid]; uint64_t cap; @@ -181,8 +181,8 @@ void nvme_discard_cq(struct nvme_ctrl *ctrl, struct nvme_cq *cq) memset(cq, 0x0, sizeof(*cq)); } -static int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize, - struct nvme_cq *cq, unsigned long UNUSED flags) +int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize, + struct nvme_cq *cq, unsigned long UNUSED flags) { struct nvme_sq *sq = &ctrl->sq[qid]; uint64_t cap;