Misc nvme/block fixes

lib/propolis/src/block/backend.rs

@@ -182,6 +182,7 @@ impl Attachment {
     pub fn notify(&self) {
         // TODO: avoid thundering herd?
         self.0.req_notifier.notify_waiters();
+        let _guard = self.0.state.lock().unwrap();
         self.0.cv.notify_all();
     }
 }

lib/propolis/src/hw/nvme/mod.rs

@@ -618,7 +618,7 @@ impl PciNvme {
 
         let cur = state.ctrl.cc;
         if new.enabled() && !cur.enabled() {
-            slog::info!(self.log, "Enabling controller");
+            slog::debug!(self.log, "Enabling controller");
 
             let mem = self.mem_access();
             let mem = mem.ok_or(NvmeError::MemoryInaccessible)?;
@@ -635,7 +635,7 @@ impl PciNvme {
                 state.ctrl.csts.set_ready(true);
             }
         } else if !new.enabled() && cur.enabled() {
-            slog::info!(self.log, "Disabling controller");
+            slog::debug!(self.log, "Disabling controller");
 
             // Reset controller state which will set CC.EN=0 and CSTS.RDY=0
             state.reset();
@@ -840,20 +840,23 @@ impl PciNvme {
                     // Completion Queue y Head Doorbell
                     let cq = state.get_cq(qid)?;
                     cq.notify_head(val)?;
-
-                    // We may have skipped pulling entries off some SQ due to this
-                    // CQ having no available entry slots. Since we've just freed
-                    // up some slots, notify any attached block backend that
-                    // there may be new requests available.
-                    self.block_attach.notify();
                 } else {
                     // Submission Queue y Tail Doorbell
                     let sq = state.get_sq(qid)?;
                     sq.notify_tail(val)?;
-
-                    // Poke block backend to service new requests
-                    self.block_attach.notify();
                 }
+
+                // Poke block backend to service new requests
+                //
+                // This is done for CQs in addition to SQs, since we may have
+                // skipped pulling entries off some SQ due to CQs having no
+                // available entry slots.
+                //
+                // The state lock must be released before issuing the
+                // notification to avoid deadlocking with the block backend
+                // attempting to fetch new requests.
+                drop(state);
+                self.block_attach.notify();
             }
         }