crutest cleaning; adding rand-read/write tests (#1233)

This is a grab-bag of `crutest` improvements and features, with the goal
of making benchmarking and performance evaluation easier.

# New features
Adding `rand-read` and `rand-write` workloads. This is meant to mimic
the `fio` workload, but running outside of the Propolis VM to remove one
layer of abstraction.

# Medium optimization
Refactor `fill_workload` and `verify_volume` to use 8x worker tasks and
run in parallel. This is a significant speedup: for a 32 GiB region,
running fill + verify goes from 7:47 down to 1:35.

# Minor tweaks
- Use `&Guest` instead of `Arc<&Guest>` in function arguments where
possible
- Use `default_value_t` instead of `default_value` so that default
arguments are checked at compile-time
- Make some functions take `&self` instead of `&mut self`
- Delete unused functions (`RegionInfo::set_wc_min`)
- Replace `Vec<Future<..>>` with `FuturesOrdered` in a few places, and
remove our custom `join_all` function

Author: mkeeter

Cargo.lock

@@ -51,6 +51,7 @@ futures-core = "0.3"
 hex = "0.4"
 http = "0.2.12"
 httptest = "0.15.5"
+human_bytes = "0.4.3"
 hyper = { version = "0.14", features = [ "full" ] }
 hyper-staticfile = "0.9"
 indicatif = { version = "0.17.8", features = ["rayon"] }

Cargo.toml

@@ -333,6 +333,11 @@ impl std::fmt::Display for BuildInfo {
  * A common logger setup for all to use.
  */
 pub fn build_logger() -> slog::Logger {
+    build_logger_with_level(slog::Level::Info)
+}
+
+/// Build a logger with the specific log level
+pub fn build_logger_with_level(level: slog::Level) -> slog::Logger {
     let main_drain = if atty::is(atty::Stream::Stdout) {
         let decorator = slog_term::TermDecorator::new().build();
         let drain = slog_term::FullFormat::new(decorator).build().fuse();
@@ -350,7 +355,7 @@ pub fn build_logger() -> slog::Logger {
 
     let (dtrace_drain, probe_reg) = slog_dtrace::Dtrace::new();
 
-    let filtered_main = slog::LevelFilter::new(main_drain, slog::Level::Info);
+    let filtered_main = slog::LevelFilter::new(main_drain, level);
 
     let log = slog::Logger::root(
         slog::Duplicate::new(filtered_main.fuse(), dtrace_drain.fuse()).fuse(),

common/src/lib.rs

@@ -17,6 +17,7 @@ crucible.workspace = true
 csv.workspace = true
 dropshot.workspace = true
 dsc-client.workspace = true
+human_bytes.workspace = true
 futures-core.workspace = true
 futures.workspace = true
 indicatif.workspace = true
@@ -30,6 +31,7 @@ ringbuffer.workspace = true
 schemars.workspace = true
 serde.workspace = true
 serde_json.workspace = true
+slog.workspace = true
 signal-hook-tokio.workspace = true
 signal-hook.workspace = true
 statistical.workspace = true

crutest/Cargo.toml

@@ -887,7 +887,7 @@ async fn process_cli_command(
                 match perf_workload(
                     guest,
                     ri,
-                    &mut None,
+                    None,
                     count,
                     io_size,
                     io_depth,