Remove the Active → Faulted transition (#1260)

This PR removes the Active → Faulted transition when too many IO
operations are in flight.

See #1252 for details on why this is desirable!

There are a bunch of changes that come along for the ride:

- The backpressure equation is changed from quadratic to the form `1 /
(1 - f)`, i.e. it now goes to infinity at our desired maximum value.
These maximum values are set at 2x the fault for the Offline → Faulted
transition.
- Computing backpressure is now a standalone function on
`BackpressureConfig`, so it can be unit tested
- Unit tests are added to confirm backpressure properties
- The `Offline` → `Faulted` transition happens at a reasonable timescale
(> 1 sec, < 2 minutes)
  - Backpressure goes to ∞ as we approach these limits
- Backpressure settings are tweaked based on these new requirements

Author: mkeeter

Cargo.lock

@@ -53,6 +53,7 @@ hex = "0.4"
 http = "0.2.12"
 httptest = "0.15.5"
 human_bytes = "0.4.3"
+humantime = "2.1.0"
 hyper = { version = "0.14", features = [ "full" ] }
 hyper-staticfile = "0.9"
 indicatif = { version = "0.17.8", features = ["rayon"] }

Cargo.toml

@@ -1836,9 +1836,7 @@ async fn generic_workload(
 }
 
 // Make use of dsc to stop and start a downstairs while sending IO.  This
-// should trigger the replay code path.  The IO sent to the downstairs should
-// be below the threshold of gone_too_long() so we don't end up faulting the
-// downstairs and doing a live repair
+// should trigger the replay code path.
 async fn replay_workload(
     guest: &Arc<Guest>,
     wtq: &mut WhenToQuit,

crutest/src/main.rs

@@ -1140,13 +1140,13 @@ where
      *        │                          │
      *        │         ┌────────────────┴────────────────────┐
      *        │         │         framed_write_task           │
-     *        │         └─▲─────▲──────────────────▲──────────┘
-     *        │           │     │                  │
-     *        │       ping│     │invalid           │
-     *        │  ┌────────┘     │frame             │responses
-     *        │  │              │errors            │
-     *        │  │              │                  │
-     *   ┌────▼──┴─┐ message   ┌┴──────┐  job     ┌┴────────┐
+     *        │         └───────▲──────────────────▲──────────┘
+     *        │                 │                  │
+     *        │                 │invalid frame     │
+     *        │                 │errors, pings     │responses
+     *        │                 │                  │
+     *        │                 │                  │
+     *   ┌────▼────┐ message   ┌┴──────┐  job     ┌┴────────┐
      *   │resp_loop├──────────►│pf_task├─────────►│ dw_task │
      *   └─────────┘ channel   └──┬────┘ channel  └▲────────┘
      *                            │                │
@@ -1308,12 +1308,7 @@ where
                         return Ok(());
                     }
                     Some(Ok(msg)) => {
-                        if matches!(msg, Message::Ruok) {
-                            // Respond instantly to pings, don't wait.
-                            if let Err(e) = resp_channel_tx.send(Message::Imok) {
-                                bail!("Failed sending Imok: {}", e);
-                            }
-                        } else if let Err(e) = message_channel_tx.send(msg) {
+                        if let Err(e) = message_channel_tx.send(msg) {
                             bail!("Failed sending message to proc_frame: {}", e);
                         }
                     }
@@ -2660,6 +2655,10 @@ impl Downstairs {
                 resp_tx.send(msg)?;
                 None
             }
+            Message::Ruok => {
+                resp_tx.send(Message::Imok)?;
+                None
+            }
             x => bail!("unexpected frame {:?}", x),
         };
         Ok(r)

downstairs/src/lib.rs

@@ -62,12 +62,13 @@ http = { workspace = true, optional = true }
 
 [dev-dependencies]
 expectorate.workspace = true
-openapiv3.workspace = true
+humantime.workspace = true
 openapi-lint.workspace = true
-tokio-test.workspace = true
+openapiv3.workspace = true
+proptest.workspace = true
 tempfile.workspace = true
 test-strategy.workspace = true
-proptest.workspace = true
+tokio-test.workspace = true
 
 [build-dependencies]
 version_check = "0.9.4"

upstairs/Cargo.toml

@@ -2506,6 +2506,10 @@ impl ClientIoTask {
         // If we're reconnecting, then add a short delay to avoid constantly
         // spinning (e.g. if something is fundamentally wrong with the
         // Downstairs)
+        //
+        // The upstairs can still stop us here, e.g. if we need to transition
+        // from Offline -> Faulted because we hit a job limit, that bounces the
+        // IO task (whether it *should* is debatable).
         if self.delay {
             tokio::select! {
                 s = &mut self.stop => {