nvme: Expose MDTS value with basic logic

bin/propolis-server/src/lib/initializer.rs

@@ -540,8 +540,11 @@ impl<'a> MachineInitializer<'a> {
                     chipset.pci_attach(bdf, vioblk);
                 }
                 DeviceInterface::Nvme => {
+                    // Limit data transfers to 1MiB (2^8 * 4k) in size
+                    let mdts = Some(8);
                     let nvme = nvme::PciNvme::create(
                         name.to_string(),
+                        mdts,
                         self.log.new(
                             slog::o!("component" => format!("nvme-{}", name)),
                         ),

bin/propolis-standalone/src/main.rs

@@ -990,7 +990,9 @@ fn setup_instance(
                         .to_string();
                     let log =
                         log.new(slog::o!("dev" => format!("nvme-{}", name)));
-                    let nvme = hw::nvme::PciNvme::create(dev_serial, log);
+                    // Limit data transfers to 1MiB (2^8 * 4k) in size
+                    let mdts = Some(8);
+                    let nvme = hw::nvme::PciNvme::create(dev_serial, mdts, log);
 
                     guard.inventory.register_instance(&nvme, &bdf.to_string());
                     guard.inventory.register_block(&backend, name);

lib/propolis/src/hw/nvme/bits.rs

@@ -252,6 +252,12 @@ bitstruct! {
         reserved4: u8 = 56..64;
     }
 }
+impl Capabilities {
+    /// Size in bytes represented by the MPSMIN value
+    pub fn mpsmin_sz(&self) -> usize {
+        1 << (12 + self.mpsmin())
+    }
+}
 
 bitstruct! {
     /// Representation of the Controller Configuration (CC) register.

lib/propolis/src/hw/nvme/mod.rs

@@ -411,6 +411,20 @@ impl NvmeCtrl {
         b << (self.ns_ident.lbaf[(self.ns_ident.flbas & 0xF) as usize]).lbads
     }
 
+    /// Check a given request transfer size against any MDTS configured on the
+    /// controller.
+    ///
+    /// Returns `true` if the size is acceptable for the MDTS.
+    fn valid_for_mdts(&self, sz: u64) -> bool {
+        match self.ctrl_ident.mdts {
+            0 => true,
+            mdts => {
+                let limit = (self.ctrl.cap.mpsmin_sz() as u64) << mdts;
+                sz <= limit
+            }
+        }
+    }
+
     fn update_block_info(&mut self, info: block::DeviceInfo) {
         let nsze = info.total_size;
         self.ns_ident = bits::IdentifyNamespace {
@@ -496,7 +510,11 @@ pub struct PciNvme {
 
 impl PciNvme {
     /// Create a new pci-nvme device with the given values
-    pub fn create(serial_number: String, log: slog::Logger) -> Arc<Self> {
+    pub fn create(
+        serial_number: String,
+        mdts: Option<u8>,
+        log: slog::Logger,
+    ) -> Arc<Self> {
         let builder = pci::Builder::new(pci::Ident {
             vendor_id: VENDOR_OXIDE,
             device_id: PROPOLIS_NVME_DEV_ID,
@@ -526,6 +544,7 @@ impl PciNvme {
             ssvid: VENDOR_OXIDE,
             sn,
             ieee: OXIDE_OUI,
+            mdts: mdts.unwrap_or(0),
             // We use standard Completion/Submission Queue Entry structures with no extra
             // data, so required (minimum) == maximum
             sqes: NvmQueueEntrySize(0).with_maximum(sqes).with_required(sqes),

lib/propolis/src/hw/nvme/requests.rs

@@ -6,6 +6,7 @@ use crate::{
     accessors::MemAccessor,
     block::{self, Operation, Request, Result as BlockResult},
     hw::nvme::{bits, cmds::Completion},
+    vmm::mem::MemCtx,
 };
 
 use super::{cmds::NvmCmd, queue::Permit, PciNvme};
@@ -80,10 +81,23 @@ impl PciNvme {
                 let cid = sub.cid();
                 let cmd = NvmCmd::parse(sub);
 
+                fn fail_mdts(permit: Permit, mem: &MemCtx) {
+                    permit.complete(
+                        Completion::generic_err(bits::STS_INVAL_FIELD).dnr(),
+                        Some(&mem),
+                    );
+                }
+
                 match cmd {
                     Ok(NvmCmd::Write(cmd)) => {
                         let off = state.nlb_to_size(cmd.slba as usize) as u64;
                         let size = state.nlb_to_size(cmd.nlb as usize) as u64;
+
+                        if !state.valid_for_mdts(size) {
+                            fail_mdts(permit, &mem);
+                            continue;
+                        }
+
                         probes::nvme_write_enqueue!(|| (
                             qid, idx, cid, off, size
                         ));
@@ -100,6 +114,11 @@ impl PciNvme {
                         let off = state.nlb_to_size(cmd.slba as usize) as u64;
                         let size = state.nlb_to_size(cmd.nlb as usize) as u64;
 
+                        if !state.valid_for_mdts(size) {
+                            fail_mdts(permit, &mem);
+                            continue;
+                        }
+
                         probes::nvme_read_enqueue!(|| (
                             qid, idx, cid, off, size
                         ));