nvme: Expose MDTS value with basic logic

While the interaction between the block backends and frontends like NVMe
could be improved to include negotiation of parameters such as MDTS, it
would be nice to define a reasonable value in the mean time.

Author: pfmooney

bin/propolis-server/src/lib/initializer.rs

@@ -540,8 +540,11 @@ impl<'a> MachineInitializer<'a> {
                     chipset.pci_attach(bdf, vioblk);
                 }
                 DeviceInterface::Nvme => {
+                    // Limit data transfers to 1MiB (2^8 * 4k) in size
+                    let mdts = Some(8);
                     let nvme = nvme::PciNvme::create(
                         name.to_string(),
+                        mdts,
                         self.log.new(
                             slog::o!("component" => format!("nvme-{}", name)),
                         ),

bin/propolis-standalone/src/main.rs

@@ -990,7 +990,9 @@ fn setup_instance(
                         .to_string();
                     let log =
                         log.new(slog::o!("dev" => format!("nvme-{}", name)));
-                    let nvme = hw::nvme::PciNvme::create(dev_serial, log);
+                    // Limit data transfers to 1MiB (2^8 * 4k) in size
+                    let mdts = Some(8);
+                    let nvme = hw::nvme::PciNvme::create(dev_serial, mdts, log);
 
                     guard.inventory.register_instance(&nvme, &bdf.to_string());
                     guard.inventory.register_block(&backend, name);

lib/propolis/src/hw/nvme/bits.rs

@@ -252,6 +252,12 @@ bitstruct! {
         reserved4: u8 = 56..64;
     }
 }
+impl Capabilities {
+    /// Size in bytes represented by the MPSMIN value
+    pub fn mpsmin_sz(&self) -> usize {
+        1 << (12 + self.mpsmin())
+    }
+}
 
 bitstruct! {
     /// Representation of the Controller Configuration (CC) register.

lib/propolis/src/hw/nvme/mod.rs

@@ -411,6 +411,20 @@ impl NvmeCtrl {
         b << (self.ns_ident.lbaf[(self.ns_ident.flbas & 0xF) as usize]).lbads
     }
 
+    /// Check a given request transfer size against any MDTS configured on the
+    /// controller.
+    ///
+    /// Returns `true` if the size is acceptable for the MDTS.
+    fn valid_for_mdts(&self, sz: u64) -> bool {
+        match self.ctrl_ident.mdts {
+            0 => true,
+            mdts => {
+                let limit = (self.ctrl.cap.mpsmin_sz() as u64) << mdts;
+                sz <= limit
+            }
+        }
+    }
+
     fn update_block_info(&mut self, info: block::DeviceInfo) {
         let nsze = info.total_size;
         self.ns_ident = bits::IdentifyNamespace {
@@ -496,7 +510,11 @@ pub struct PciNvme {
 
 impl PciNvme {
     /// Create a new pci-nvme device with the given values
-    pub fn create(serial_number: String, log: slog::Logger) -> Arc<Self> {
+    pub fn create(
+        serial_number: String,
+        mdts: Option<u8>,
+        log: slog::Logger,
+    ) -> Arc<Self> {
         let builder = pci::Builder::new(pci::Ident {
             vendor_id: VENDOR_OXIDE,
             device_id: PROPOLIS_NVME_DEV_ID,
@@ -526,6 +544,7 @@ impl PciNvme {
             ssvid: VENDOR_OXIDE,
             sn,
             ieee: OXIDE_OUI,
+            mdts: mdts.unwrap_or(0),
             // We use standard Completion/Submission Queue Entry structures with no extra
             // data, so required (minimum) == maximum
             sqes: NvmQueueEntrySize(0).with_maximum(sqes).with_required(sqes),

lib/propolis/src/hw/nvme/requests.rs

@@ -6,6 +6,7 @@ use crate::{
     accessors::MemAccessor,
     block::{self, Operation, Request, Result as BlockResult},
     hw::nvme::{bits, cmds::Completion},
+    vmm::mem::MemCtx,
 };
 
 use super::{cmds::NvmCmd, queue::Permit, PciNvme};
@@ -80,10 +81,23 @@ impl PciNvme {
                 let cid = sub.cid();
                 let cmd = NvmCmd::parse(sub);
 
+                fn fail_mdts(permit: Permit, mem: &MemCtx) {
+                    permit.complete(
+                        Completion::generic_err(bits::STS_INVAL_FIELD).dnr(),
+                        Some(&mem),
+                    );
+                }
+
                 match cmd {
                     Ok(NvmCmd::Write(cmd)) => {
                         let off = state.nlb_to_size(cmd.slba as usize) as u64;
                         let size = state.nlb_to_size(cmd.nlb as usize) as u64;
+
+                        if !state.valid_for_mdts(size) {
+                            fail_mdts(permit, &mem);
+                            continue;
+                        }
+
                         probes::nvme_write_enqueue!(|| (
                             qid, idx, cid, off, size
                         ));
@@ -100,6 +114,11 @@ impl PciNvme {
                         let off = state.nlb_to_size(cmd.slba as usize) as u64;
                         let size = state.nlb_to_size(cmd.nlb as usize) as u64;
 
+                        if !state.valid_for_mdts(size) {
+                            fail_mdts(permit, &mem);
+                            continue;
+                        }
+
                         probes::nvme_read_enqueue!(|| (
                             qid, idx, cid, off, size
                         ));