nvme: add support for CMB (Controller Memory Buffer)

CMB provides device memory buffer for driver context such as SQ/CQ and
data buffer.  nvme_configure_cmb() initializes CMB registers in the nvme
controller and nvme_discard_cmb() will disable it and destroy the cmb
attributes in @ctrl->cmb.

Signed-off-by: Minwoo Im <[email protected]>

examples/cmb-p2p.c

@@ -54,70 +54,11 @@ static const struct nvme_ctrl_opts ctrl_opts = {
 	.ncqr = 63,
 };
 
-static inline off_t bar_offset(int bar)
-{
-	return PCI_BASE_ADDRESS_0 + bar * 4;
-}
-
-static void *nvme_configure_cmb(struct nvme_ctrl *ctrl, uint64_t *iova, size_t *len)
-{
-	uint64_t cap, szu, ofst, hwaddr;
-	uint32_t cmbloc, cmbsz, v32;
-	int bir;
-	void *cmb;
-
-	cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP));
-
-	if (NVME_GET(cap, CAP_CMBS))
-		mmio_hl_write64(ctrl->regs + NVME_REG_CMBMSC, cpu_to_le64(0x1));
-
-	cmbsz = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CMBSZ));
-	cmbloc = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CMBLOC));
-
-	szu = 1 << (12 + 4 * NVME_GET(cmbsz, CMBSZ_SZU));
-	*len = szu * NVME_GET(cmbsz, CMBSZ_SZ);
-
-	ofst = szu * NVME_GET(cmbloc, CMBLOC_OFST);
-	bir = NVME_GET(cmbloc, CMBLOC_BIR);
-
-	printf("cmb bir is %d\n", bir);
-
-	if (vfio_pci_read_config(&ctrl->pci, &v32, sizeof(v32), bar_offset(bir)) < 0)
-		err(1, "failed to read pci config");
-
-	hwaddr = v32 & PCI_BASE_ADDRESS_MEM_MASK;
-
-	if (vfio_pci_read_config(&ctrl->pci, &v32, sizeof(v32), bar_offset(bir + 1)) < 0)
-		err(1, "failed to read pci config");
-
-	hwaddr |= (uint64_t)v32 << 32;
-
-	printf("cmb bar is mapped at physical address 0x%lx\n", hwaddr);
-
-	/* map the cmb */
-	cmb = vfio_pci_map_bar(&ctrl->pci, bir, *len, ofst, PROT_READ | PROT_WRITE);
-	if (!cmb)
-		err(1, "failed to map cmb");
-
-	if (iommu_map_vaddr(__iommu_ctx(ctrl), cmb, *len, iova, 0x0))
-		err(1, "failed to map cmb in iommu (try VFN_IOMMU_FORCE_VFIO=1)");
-
-	if (NVME_GET(cap, CAP_CMBS)) {
-		printf("assigned cmb base address is 0x%lx\n", *iova);
-
-		/* set the base address and enable the memory space */
-		mmio_hl_write64(ctrl->regs + NVME_REG_CMBMSC, cpu_to_le64(*iova | 0x3));
-	}
-
-	return cmb;
-}
-
 int main(int argc, char **argv)
 {
 	struct nvme_ctrl src = {}, dst = {};
 
 	uint64_t iova;
-	size_t len;
 	void *cmb;
 
 	union nvme_cmd cmd = {};
@@ -141,7 +82,11 @@ int main(int argc, char **argv)
 	if (nvme_init(&dst, bdfs[1], &ctrl_opts))
 		err(1, "failed to initialize destination nvme controller");
 
-	cmb = nvme_configure_cmb(&dst, &iova, &len);
+	if (nvme_configure_cmb(&dst))
+		err(1, "failed to initialize cmb to destination nvme controller");
+
+	iova = dst.cmb.iova;
+	cmb = dst.cmb.vaddr;
 
 	cmd.identify = (struct nvme_cmd_identify) {
 		.opcode = nvme_admin_identify,

include/vfn/nvme/ctrl.h

@@ -106,6 +106,16 @@ struct nvme_ctrl {
 	 * See &enum nvme_ctrl_feature_flags.
 	 */
 	unsigned long flags;
+
+	/**
+	 * @cmb: CMB atribute initialized by nvme_configure_cmb()
+	 */
+	struct {
+		int bar;
+		void *vaddr;
+		uint64_t iova;
+		size_t size;
+	} cmb;
 };
 
 /**
@@ -340,4 +350,28 @@ void nvme_discard_cq(struct nvme_ctrl *ctrl, struct nvme_cq *cq);
  */
 void nvme_discard_sq(struct nvme_ctrl *ctrl, struct nvme_sq *sq);
 
+/**
+ * nvme_configure_cmb - Initialize Controller Memory Buffer
+ * @ctrl: See &struct nvme_ctrl
+ *
+ * Enable CMB in the @ctrl and initialize @ctrl->cmb members.  It first maps
+ * a vaddr to the CMB BAR memory region and map it to IOMMU.  And the mapped
+ * iova will be configured as CBA(CMB Base Address) to the register.
+ *
+ * **Note** iommufd-backed IOMMU does not support BAR vaddr to be mapped to the
+ * IOMMU page table.  vfio-backed IOMMU should be used.
+ *
+ * Return: On success, returns ``0``. On error, returns ``-1`` and sets
+ * ``errno``.
+ */
+int nvme_configure_cmb(struct nvme_ctrl *ctrl);
+
+/**
+ * nvme_discard_cmb - Disable CMB and discard cmb instance @ctrl->cmb
+ * @ctrl: See &struct nvme_ctrl
+ *
+ * Disable CMB in the @ctrl and discard @ctrl->cmb members.
+ */
+void nvme_discard_cmb(struct nvme_ctrl *ctrl);
+
 #endif /* LIBVFN_NVME_CTRL_H */

src/nvme/core.c

@@ -735,6 +735,8 @@ void nvme_close(struct nvme_ctrl *ctrl)
 
 	free(ctrl->cq);
 
+	nvme_discard_cmb(ctrl);
+
 	if (ctrl->dbbuf.doorbells) {
 		struct iommu_ctx *ctx = __iommu_ctx(ctrl);
 		size_t len;
@@ -755,3 +757,77 @@ void nvme_close(struct nvme_ctrl *ctrl)
 
 	memset(ctrl, 0x0, sizeof(*ctrl));
 }
+
+int nvme_configure_cmb(struct nvme_ctrl *ctrl)
+{
+	uint64_t cmbmsc;
+	uint32_t cmbloc;
+	uint32_t cmbsz;
+	uint64_t cap;
+	int bar;
+
+	cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP));
+	if (!NVME_FIELD_GET(cap, CAP_CMBS))
+		return 0;
+
+	cmbmsc = NVME_FIELD_SET(1, CMBMSC_CRE);
+	mmio_hl_write64(ctrl->regs + NVME_REG_CMBMSC, cpu_to_le64(cmbmsc));
+
+	/*
+	 * Read CMBMSC back here to guarantee that the previous CMBMSC write to
+	 * be flushed before reading CMBLOC register avoiding reordering.
+	 */
+	cmbmsc = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CMBMSC));
+	cmbloc = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CMBLOC));
+	bar = NVME_FIELD_GET(cmbloc, CMBLOC_BIR);
+
+	cmbsz = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CMBSZ));
+
+	ctrl->cmb.bar = bar;
+	ctrl->cmb.size = nvme_cmb_size(cmbsz);
+	ctrl->cmb.vaddr = vfio_pci_map_bar(&ctrl->pci, bar,
+			ctrl->cmb.size, NVME_FIELD_GET(cmbloc, CMBLOC_OFST),
+			PROT_READ | PROT_WRITE);
+	if (!ctrl->cmb.vaddr) {
+		log_debug("could not map bar %d vaddr\n", bar);
+		return -1;
+	}
+
+	if (iommu_map_vaddr(__iommu_ctx(ctrl), ctrl->cmb.vaddr,
+				ctrl->cmb.size, &ctrl->cmb.iova, 0x0)) {
+		log_debug("could not map bar vaddr to iommu\n");
+		return -1;
+	}
+
+	cmbmsc |= NVME_FIELD_SET(1, CMBMSC_CMSE) |
+		(ctrl->cmb.iova >> NVME_CMBMSC_CBA_SHIFT) <<
+		NVME_CMBMSC_CBA_SHIFT;
+
+	mmio_hl_write64(ctrl->regs + NVME_REG_CMBMSC, cpu_to_le64(cmbmsc));
+
+	log_debug("cmb initialized (bar=%d, iova=%#lx, vaddr=%p, size=%#lx)\n",
+			bar, ctrl->cmb.iova, ctrl->cmb.vaddr, ctrl->cmb.size);
+	return 0;
+}
+
+void nvme_discard_cmb(struct nvme_ctrl *ctrl)
+{
+	uint64_t cmbmsc;
+	uint32_t cmbloc;
+
+	if (!ctrl->cmb.vaddr)
+		return;
+
+	cmbmsc = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CMBMSC));
+	cmbmsc &= ~(1 << NVME_CMBMSC_CMSE_SHIFT);
+	cmbmsc &= ~(NVME_CMBMSC_CBA_MASK << NVME_CMBMSC_CBA_SHIFT);
+	mmio_hl_write64(ctrl->regs + NVME_REG_CMBMSC, cpu_to_le64(cmbmsc));
+
+	iommu_unmap_vaddr(__iommu_ctx(ctrl), ctrl->cmb.vaddr, NULL);
+
+	cmbloc = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CMBLOC));
+	vfio_pci_unmap_bar(&ctrl->pci, ctrl->cmb.bar, ctrl->cmb.vaddr,
+			ctrl->cmb.size, NVME_FIELD_GET(cmbloc, CMBLOC_OFST));
+
+	memset(&ctrl->cmb, 0, sizeof(ctrl->cmb));
+}

src/nvme/types.h

@@ -27,6 +27,9 @@ enum nvme_reg {
 	NVME_REG_AQA			= 0x0024,
 	NVME_REG_ASQ			= 0x0028,
 	NVME_REG_ACQ			= 0x0030,
+	NVME_REG_CMBLOC			= 0x0038,
+	NVME_REG_CMBSZ			= 0x003c,
+	NVME_REG_CMBMSC			= 0x0050,
 };
 
 enum nvme_cap {
@@ -42,6 +45,8 @@ enum nvme_cap {
 	NVME_CAP_MPSMIN_MASK		= 0xf,
 	NVME_CAP_MPSMAX_SHIFT		= 52,
 	NVME_CAP_MPSMAX_MASK		= 0xf,
+	NVME_CAP_CMBS_SHIFT		= 57,
+	NVME_CAP_CMBS_MASK		= 0x1,
 
 	NVME_CAP_CSS_CSI		= 1 << 6,
 	NVME_CAP_CSS_ADMIN		= 1 << 7,
@@ -75,6 +80,42 @@ enum nvme_csts {
 	NVME_CSTS_RDY_MASK		= 0x1,
 };
 
+enum nvme_cmbloc {
+	NVME_CMBLOC_BIR_SHIFT		= 0,
+	NVME_CMBLOC_BIR_MASK		= 0x7,
+	NVME_CMBLOC_OFST_SHIFT		= 12,
+	NVME_CMBLOC_OFST_MASK		= 0xfffff,
+};
+
+enum nvme_cmbsz {
+	NVME_CMBSZ_SZU_SHIFT	= 8,
+	NVME_CMBSZ_SZU_MASK	= 0xf,
+	NVME_CMBSZ_SZ_SHIFT	= 12,
+	NVME_CMBSZ_SZ_MASK	= 0xfffff,
+	NVME_CMBSZ_SZU_4K	= 0,
+	NVME_CMBSZ_SZU_64K	= 1,
+	NVME_CMBSZ_SZU_1M	= 2,
+	NVME_CMBSZ_SZU_16M	= 3,
+	NVME_CMBSZ_SZU_256M	= 4,
+	NVME_CMBSZ_SZU_4G	= 5,
+	NVME_CMBSZ_SZU_64G	= 6,
+};
+
+static inline __u64 nvme_cmb_size(__u32 cmbsz)
+{
+	return ((__u64)NVME_FIELD_GET(cmbsz, CMBSZ_SZ)) *
+		(1ULL << (12 + 4 * NVME_FIELD_GET(cmbsz, CMBSZ_SZU)));
+}
+
+enum nvme_cmbmsc {
+	NVME_CMBMSC_CRE_SHIFT		= 0,
+	NVME_CMBMSC_CRE_MASK		= 0x1,
+	NVME_CMBMSC_CMSE_SHIFT		= 1,
+	NVME_CMBMSC_CMSE_MASK		= 0x1,
+	NVME_CMBMSC_CBA_SHIFT		= 12,
+};
+static const __u64 NVME_CMBMSC_CBA_MASK = 0xfffffffffffffull;
+
 enum nvme_feat {
 	NVME_FEAT_NRQS_NSQR_SHIFT	= 0,
 	NVME_FEAT_NRQS_NSQR_MASK	= 0xffff,