diff --git a/common/src/lib.rs b/common/src/lib.rs index b3a16a675..943cbd5e1 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -16,9 +16,10 @@ use tokio::time::{Duration, Instant}; mod region; pub use region::{ - config_path, Block, BlockIndex, BlockOffset, ExtentId, RegionDefinition, - RegionOptions, DATABASE_READ_VERSION, DATABASE_WRITE_VERSION, - MAX_BLOCK_SIZE, MAX_SHIFT, MIN_BLOCK_SIZE, MIN_SHIFT, + config_path, Block, BlockIndex, BlockOffset, ExtentId, ExtentInfo, + RegionDefinition, RegionOptions, DATABASE_READ_VERSION, + DATABASE_WRITE_VERSION, MAX_BLOCK_SIZE, MAX_SHIFT, MIN_BLOCK_SIZE, + MIN_SHIFT, }; pub mod impacted_blocks; diff --git a/common/src/region.rs b/common/src/region.rs index f2a384089..a09d549b7 100644 --- a/common/src/region.rs +++ b/common/src/region.rs @@ -406,6 +406,13 @@ impl Default for RegionOptions { } } +#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)] +/// Info about extents in a region. +pub struct ExtentInfo { + pub extent_size: Block, + pub extent_count: u32, +} + /// Append the region description file to the end of a provided path. pub fn config_path>(dir: P) -> PathBuf { let mut out = dir.as_ref().to_path_buf(); diff --git a/crucible-client-types/src/lib.rs b/crucible-client-types/src/lib.rs index fe2af200a..589b9021d 100644 --- a/crucible-client-types/src/lib.rs +++ b/crucible-client-types/src/lib.rs @@ -38,6 +38,41 @@ pub enum VolumeConstructionRequest { }, } +impl VolumeConstructionRequest { + /// Return all the targets that are part of this VCR. + pub fn targets(&self) -> Vec { + let mut targets = Vec::new(); + match self { + VolumeConstructionRequest::Volume { + id: _, + block_size: _, + sub_volumes, + read_only_parent: _, + } => { + for subreq in sub_volumes { + let new_targets = subreq.targets(); + for nt in new_targets { + targets.push(nt); + } + } + } + VolumeConstructionRequest::Region { + block_size: _, + blocks_per_extent: _, + extent_count: _, + opts, + gen: _, + } => { + for nt in &opts.target { + targets.push(*nt); + } + } + _ => {} + } + targets + } +} + #[allow(clippy::derive_partial_eq_without_eq)] #[derive( Debug, Clone, Default, Serialize, Deserialize, JsonSchema, PartialEq, diff --git a/crutest/src/cli.rs b/crutest/src/cli.rs index dcbf76c20..46e9f5de5 100644 --- a/crutest/src/cli.rs +++ b/crutest/src/cli.rs @@ -128,6 +128,8 @@ enum CliCommand { #[clap(long, action)] skip_verify: bool, }, + /// Run the sparse fill workload, no verify + FillSparse, /// Flush Flush, /// Run Generic workload @@ -499,6 +501,9 @@ async fn cmd_to_msg( CliCommand::Fill { skip_verify } => { fw.send(CliMessage::Fill(skip_verify)).await?; } + CliCommand::FillSparse => { + fw.send(CliMessage::FillSparse).await?; + } CliCommand::Flush => { fw.send(CliMessage::Flush).await?; } @@ -571,8 +576,8 @@ async fn cmd_to_msg( Some(CliMessage::MyUuid(uuid)) => { println!("uuid: {}", uuid); } - Some(CliMessage::Info(es, bs, bl)) => { - println!("Got info: {} {} {}", es, bs, bl); + Some(CliMessage::Info(ri)) => { + println!("Got info: {:?}", ri); } Some(CliMessage::DoneOk) => { println!("Ok"); @@ -871,6 +876,23 @@ async fn process_cli_command( } } } + CliMessage::FillSparse => { + if ri.write_log.is_empty() { + fw.send(CliMessage::Error(CrucibleError::GenericError( + "Info not initialized".to_string(), + ))) + .await + } else { + match fill_sparse_workload(block_io.as_ref(), ri).await { + Ok(_) => fw.send(CliMessage::DoneOk).await, + Err(e) => { + let msg = format!("FillSparse failed with {}", e); + let e = CrucibleError::GenericError(msg); + fw.send(CliMessage::Error(e)).await + } + } + } + } CliMessage::Flush => { println!("Flush"); match block_io.flush(None).await { @@ -883,13 +905,10 @@ async fn process_cli_command( Err(e) => fw.send(CliMessage::Error(e)).await, }, CliMessage::InfoPlease => { - let new_ri = get_region_info(block_io).await; + let new_ri = get_region_info(block_io, false).await; match new_ri { Ok(new_ri) => { - let bs = new_ri.block_size; - let es = new_ri.extent_size.value; - let ts = new_ri.total_size; - *ri = new_ri; + *ri = new_ri.clone(); /* * We may only want to read input from the file once. * Maybe make a command to specifically do it, but it @@ -902,7 +921,7 @@ async fn process_cli_command( *wc_filled = true; } } - fw.send(CliMessage::Info(bs, es, ts)).await + fw.send(CliMessage::Info(new_ri)).await } Err(e) => fw.send(CliMessage::Error(e)).await, } @@ -1074,7 +1093,7 @@ pub async fn start_cli_server( */ let mut ri: RegionInfo = RegionInfo { block_size: 0, - extent_size: Block::new_512(0), + sub_volume_info: Vec::new(), total_size: 0, total_blocks: 0, write_log: WriteLog::new(0), diff --git a/crutest/src/main.rs b/crutest/src/main.rs index b33092b18..87ff0b5c2 100644 --- a/crutest/src/main.rs +++ b/crutest/src/main.rs @@ -135,20 +135,12 @@ enum Workload { }, /// Test that we can replace a downstairs when the upstairs is not active. ReplaceBeforeActive { - /// URL location of the running dsc server - #[clap(long, default_value = "http://127.0.0.1:9998", action)] - dsc_str: String, - /// The address:port of a running downstairs for replacement #[clap(long, action)] replacement: SocketAddr, }, /// Test replacement of a downstairs while doing the initial reconciliation. ReplaceReconcile { - /// URL location of the running dsc server - #[clap(long, default_value = "http://127.0.0.1:9998", action)] - dsc_str: String, - /// The address:port of a running downstairs for replacement #[clap(long, action)] replacement: SocketAddr, @@ -359,7 +351,6 @@ enum RandReadWriteMode { #[derive(Copy, Clone, Debug)] struct RandReadWriteConfig { mode: RandReadWriteMode, - encrypted: bool, io_depth: usize, blocks_per_io: usize, @@ -375,13 +366,8 @@ struct RandReadWriteConfig { } impl RandReadWriteConfig { - fn new( - cfg: RandReadWriteWorkload, - encrypted: bool, - mode: RandReadWriteMode, - ) -> Self { + fn new(cfg: RandReadWriteWorkload, mode: RandReadWriteMode) -> Self { RandReadWriteConfig { - encrypted, io_depth: cfg.io_depth, blocks_per_io: cfg.io_size, time_secs: cfg.time, @@ -397,16 +383,14 @@ impl RandReadWriteConfig { /// Configuration for `bufferbloat_workload` #[derive(Copy, Clone, Debug)] struct BufferbloatConfig { - encrypted: bool, io_depth: usize, blocks_per_io: usize, time_secs: u64, } impl BufferbloatConfig { - fn new(cfg: BufferbloatWorkload, encrypted: bool) -> Self { + fn new(cfg: BufferbloatWorkload) -> Self { BufferbloatConfig { - encrypted, io_depth: cfg.io_depth, blocks_per_io: cfg.io_size, time_secs: cfg.time, @@ -422,7 +406,7 @@ impl BufferbloatConfig { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct RegionInfo { block_size: u64, - extent_size: Block, + sub_volume_info: Vec, total_size: u64, total_blocks: usize, write_log: WriteLog, @@ -434,13 +418,14 @@ pub struct RegionInfo { */ async fn get_region_info( block_io: &Arc, + is_encrypted: bool, ) -> Result { /* * These query requests have the side effect of preventing the test from * starting before the upstairs is ready. */ let block_size = block_io.get_block_size().await?; - let extent_size = block_io.query_extent_size().await?; + let extent_info_vec = block_io.query_extent_info().await?; let total_size = block_io.total_size().await?; let total_blocks = (total_size / block_size) as usize; @@ -454,31 +439,23 @@ async fn get_region_info( max_block_io = total_blocks; } - println!( - "Region: es:{} ec:{} bs:{} ts:{} tb:{} max_io:{} or {}", - extent_size.value, - total_blocks as u64 / extent_size.value, - block_size, - total_size, - total_blocks, - max_block_io, - (max_block_io as u64 * block_size), - ); - /* * Create the write log that tracks the number of writes to each block, * so we can know what to expect for reads. */ let write_log = WriteLog::new(total_blocks); - Ok(RegionInfo { + let ri = RegionInfo { block_size, - extent_size, + sub_volume_info: extent_info_vec, total_size, total_blocks, write_log, max_block_io, - }) + }; + print_region_description(&ri, is_encrypted); + + Ok(ri) } /** @@ -733,7 +710,7 @@ async fn handle_signals( } } -// Construct a volume for use by the tests. +// Construct a volume and a list of targets for use by the tests. // Our choice of how to construct the volume depends on what options we // have been given. // @@ -750,12 +727,17 @@ async fn handle_signals( // CrucibleOpts. This will work as long as one of the downstairs is up // already. If we have a test that requires no downstairs to be running on // startup, then we need to provide a VCR file, or use the dsc server. +// +// While making our volume, we also record the targets that become part of +// the volume in a separate Vec. In some cases we no longer have access to +// the target information after the volume is constructed, and some tests also +// want the specific targets, so we make and return that list here. async fn make_a_volume( opt: &Opt, block_io_logger: Logger, test_log: &Logger, pr: Option, -) -> Result> { +) -> Result<(Arc, Vec)> { let up_uuid = opt.uuid.unwrap_or_else(Uuid::new_v4); let mut crucible_opts = CrucibleOpts { id: up_uuid, @@ -785,8 +767,10 @@ async fn make_a_volume( if !opt.target.is_empty() { warn!(test_log, "targets are ignored when VCR is provided"); } + let targets = vcr.targets(); + let volume = Volume::construct(vcr, pr, block_io_logger).await.unwrap(); - Ok(Arc::new(volume)) + Ok((Arc::new(volume), targets)) } else if opt.dsc.is_some() { // We were given a dsc endpoint, use that to create a VCR that // represents our Volume. @@ -834,20 +818,22 @@ async fn make_a_volume( // Now, loop over regions we found from dsc and make a // sub_volume at every three. + let mut targets = Vec::new(); let mut cid = 0; for sv in 0..sv_count { - let mut targets = Vec::new(); + let mut sv_targets = Vec::new(); for _ in 0..3 { let port = dsc_client.dsc_get_port(cid).await.unwrap(); let tar: SocketAddr = format!("{}:{}", dsc.ip(), port.into_inner()) .parse() .unwrap(); + sv_targets.push(tar); targets.push(tar); cid += 1; } - info!(test_log, "SV {:?} has targets: {:?}", sv, targets); - crucible_opts.target = targets; + info!(test_log, "SV {:?} has targets: {:?}", sv, sv_targets); + crucible_opts.target = sv_targets; volume .add_subvolume_create_guest( @@ -860,7 +846,7 @@ async fn make_a_volume( .unwrap(); } - Ok(Arc::new(volume)) + Ok((Arc::new(volume), targets)) } else { // We were not provided a VCR, so, we have to make one by using // the repair port on a downstairs to get region information that @@ -902,6 +888,7 @@ async fn make_a_volume( bail!("Can't determine extent info to build a Volume"); } }; + let targets = crucible_opts.target.clone().into_iter().collect(); let mut volume = Volume::new(extent_info.block_size, block_io_logger); volume @@ -914,7 +901,7 @@ async fn make_a_volume( .await .unwrap(); - Ok(Arc::new(volume)) + Ok((Arc::new(volume), targets)) } } @@ -1001,7 +988,10 @@ async fn main() -> Result<()> { } // Build a Volume for all the tests to use. - let block_io = + // While we build the volume, collect the list of targets that some tests + // need to use. We won't have the list of targets available after we have + // constructed a volume from a dsc endpoint or a VCR file. + let (block_io, mut targets) = make_a_volume(&opt, block_io_logger.clone(), &test_log, pr).await?; if let Workload::CliServer { listen, port } = opt.workload { @@ -1045,7 +1035,7 @@ async fn main() -> Result<()> { * Build the region info struct that all the tests will use. * This includes importing and verifying from a write log, if requested. */ - let mut region_info = match get_region_info(&block_io).await { + let mut region_info = match get_region_info(&block_io, is_encrypted).await { Ok(region_info) => region_info, Err(e) => bail!("failed to get region info: {:?}", e), }; @@ -1095,6 +1085,17 @@ async fn main() -> Result<()> { println!("Run biggest IO test"); biggest_io_workload(&block_io, &mut region_info).await?; } + Workload::Bufferbloat { cfg } => { + bufferbloat_workload( + &block_io, + &mut region_info, + BufferbloatConfig::new(cfg), + ) + .await?; + if opt.quit { + return Ok(()); + } + } Workload::Burst => { println!("Run burst test (demo in a loop)"); burst_workload( @@ -1233,11 +1234,7 @@ async fn main() -> Result<()> { rand_read_write_workload( &block_io, &mut region_info, - RandReadWriteConfig::new( - cfg, - is_encrypted, - RandReadWriteMode::Read, - ), + RandReadWriteConfig::new(cfg, RandReadWriteMode::Read), ) .await?; if opt.quit { @@ -1248,22 +1245,7 @@ async fn main() -> Result<()> { rand_read_write_workload( &block_io, &mut region_info, - RandReadWriteConfig::new( - cfg, - is_encrypted, - RandReadWriteMode::Write, - ), - ) - .await?; - if opt.quit { - return Ok(()); - } - } - Workload::Bufferbloat { cfg } => { - bufferbloat_workload( - &block_io, - &mut region_info, - BufferbloatConfig::new(cfg, is_encrypted), + RandReadWriteConfig::new(cfg, RandReadWriteMode::Write), ) .await?; if opt.quit { @@ -1330,10 +1312,8 @@ async fn main() -> Result<()> { } }; - // Build the list of targets we use during the replace test. - // The first three are provided in the crucible opts, and the - // final one (the replacement) is a test specific arg. - let mut targets = opt.target.clone(); + // Add to the list of targets for our volume the replacement + // target provided on the command line targets.push(replacement); replace_workload( &block_io, @@ -1344,11 +1324,16 @@ async fn main() -> Result<()> { ) .await?; } - Workload::ReplaceBeforeActive { - dsc_str, - replacement, - } => { - let dsc_client = Client::new(&dsc_str); + Workload::ReplaceBeforeActive { replacement } => { + let dsc_client = match opt.dsc { + Some(dsc_addr) => { + let dsc_url = format!("http://{}", dsc_addr); + Client::new(&dsc_url) + } + None => { + bail!("Replace before active requires a dsc endpoint"); + } + }; // Either we have a count, or we run until we get a signal. let wtq = { if opt.continuous { @@ -1359,10 +1344,8 @@ async fn main() -> Result<()> { } }; - // Build the list of targets we use during the replace test. - // The first three are provided in the crucible opts, and the - // final one (the replacement) is a test specific arg. - let mut targets = opt.target.clone(); + // Add to the list of targets for our volume the replacement + // target provided on the command line targets.push(replacement); replace_before_active( &block_io, @@ -1375,11 +1358,16 @@ async fn main() -> Result<()> { ) .await?; } - Workload::ReplaceReconcile { - dsc_str, - replacement, - } => { - let dsc_client = Client::new(&dsc_str); + Workload::ReplaceReconcile { replacement } => { + let dsc_client = match opt.dsc { + Some(dsc_addr) => { + let dsc_url = format!("http://{}", dsc_addr); + Client::new(&dsc_url) + } + None => { + bail!("Replace reconcile requires a dsc endpoint"); + } + }; // Either we have a count, or we run until we get a signal. let wtq = { if opt.continuous { @@ -1390,10 +1378,8 @@ async fn main() -> Result<()> { } }; - // Build the list of targets we use during the replace test. - // The first three are provided in the crucible opts, and the - // final one (the replacement) is a test specific arg. - let mut targets = opt.target.clone(); + // Add to the list of targets for our volume the replacement + // target provided on the command line targets.push(replacement); replace_while_reconcile( &block_io, @@ -2005,24 +1991,32 @@ async fn fill_sparse_workload( ) -> Result<()> { let mut rng = rand_chacha::ChaCha8Rng::from_entropy(); - // Figure out how many extents we have - let extents = ri.total_blocks / (ri.extent_size.value as usize); - let extent_size = ri.extent_size.value as usize; + // It's possible that sub-volumes have a different extent size + // and extent count. + let mut sv_base = 0; + for (index, sv) in ri.sub_volume_info.iter().enumerate() { + // Figure out how many extents we have in this sub_volume + let extents = sv.extent_count as usize; + let extent_size = sv.extent_size.value as usize; - // Do one write to each extent. - for extent in 0..extents { - let mut block_index: usize = extent * extent_size; - let random_offset: usize = rng.gen_range(0..extent_size); - block_index += random_offset; + // Do one write to each extent. + for extent in 0..extents { + let mut block_index: usize = sv_base + (extent * extent_size); + let random_offset: usize = rng.gen_range(0..extent_size); + block_index += random_offset; - let offset = BlockIndex(block_index as u64); + let offset = BlockIndex(block_index as u64); - ri.write_log.update_wc(block_index); + ri.write_log.update_wc(block_index); - let data = fill_vec(block_index, 1, &ri.write_log, ri.block_size); + let data = fill_vec(block_index, 1, &ri.write_log, ri.block_size); - println!("[{extent}/{extents}] Write to block {}", block_index); - block_io.write(offset, data).await?; + println!( + "[{index}/{extent}/{extents}] Write to block {block_index}" + ); + block_io.write(offset, data).await?; + } + sv_base += extents * extent_size; } block_io.flush(None).await?; @@ -3017,6 +3011,11 @@ async fn perf_workload( write_loop: usize, read_loop: usize, ) -> Result<()> { + // If we have more than one sub-volume, this performance test will + // not compute extent size properly. + if ri.sub_volume_info.len() > 1 { + println!("WARNING: Multiple Sub_Volumes seen in perf test"); + } // Before we start, make sure the work queues are empty. loop { let wc = block_io.query_work_queue().await?; @@ -3044,7 +3043,7 @@ async fn perf_workload( .map(|_| Buffer::new(blocks_per_io, ri.block_size as usize)) .collect(); - let es = ri.extent_size.value; + let es = ri.sub_volume_info[0].extent_size.value; let ec = ri.total_blocks as u64 / es; // To make a random block offset, we take the total block count and subtract @@ -3167,20 +3166,29 @@ async fn perf_workload( /// Prints a pleasant summary of the given region fn print_region_description(ri: &RegionInfo, encrypted: bool) { println!("region info:"); - println!(" block size: {} bytes", ri.block_size); - println!(" blocks / extent: {}", ri.extent_size.value); - println!( - " extent size: {}", - human_bytes((ri.block_size * ri.extent_size.value) as f64) - ); + println!(" block size: {} bytes", ri.block_size); + for (index, sv) in ri.sub_volume_info.iter().enumerate() { + println!( + " sub_volume {} blocks / extent: {}", + index, sv.extent_size.value + ); + println!( + " sub_volume {} extent size: {}", + index, + human_bytes((ri.block_size * sv.extent_size.value) as f64) + ); + println!( + " sub_volume {} extent count: {}", + index, sv.extent_count + ); + } + println!(" total blocks: {}", ri.total_blocks); println!( - " extent count: {}", - ri.total_blocks as u64 / ri.extent_size.value + " total size: {}", + human_bytes(ri.total_size as f64) ); - println!(" total blocks: {}", ri.total_blocks); - println!(" total size: {}", human_bytes(ri.total_size as f64)); println!( - " encryption: {}", + " encryption: {}", if encrypted { "yes" } else { "no" } ); } @@ -3220,7 +3228,6 @@ async fn rand_read_write_workload( cfg.blocks_per_io, if cfg.blocks_per_io > 1 { "s" } else { "" }, ); - print_region_description(ri, cfg.encrypted); println!("----------------------------------------------"); let stop = Arc::new(AtomicBool::new(false)); @@ -3386,7 +3393,6 @@ async fn bufferbloat_workload( cfg.blocks_per_io, if cfg.blocks_per_io > 1 { "s" } else { "" }, ); - print_region_description(ri, cfg.encrypted); println!("----------------------------------------------"); let stop = Arc::new(AtomicBool::new(false)); @@ -3954,38 +3960,43 @@ async fn span_workload( block_io: &Arc, ri: &mut RegionInfo, ) -> Result<()> { - /* - * Pick the last block in the first extent - */ - let block_index = (ri.extent_size.value - 1) as usize; + for (index, sv) in ri.sub_volume_info.iter().enumerate() { + // Pick the last block in the first extent + let block_index = (sv.extent_size.value - 1) as usize; - /* - * Update the counter for the blocks we are about to write. - */ - ri.write_log.update_wc(block_index); - ri.write_log.update_wc(block_index + 1); + /* + * Update the counter for the blocks we are about to write. + */ + ri.write_log.update_wc(block_index); + ri.write_log.update_wc(block_index + 1); - let offset = BlockIndex(block_index as u64); - let data = fill_vec(block_index, 2, &ri.write_log, ri.block_size); + let offset = BlockIndex(block_index as u64); + let data = fill_vec(block_index, 2, &ri.write_log, ri.block_size); - println!("Sending a write spanning two extents"); - block_io.write(offset, data).await?; + println!("sub_volume {index} Sending a write spanning two extents"); + block_io.write(offset, data).await?; - println!("Sending a flush"); - block_io.flush(None).await?; + println!("sub_volume {index} Sending a flush"); + block_io.flush(None).await?; - let mut data = crucible::Buffer::repeat(99, 2, ri.block_size as usize); + let mut data = crucible::Buffer::repeat(99, 2, ri.block_size as usize); - println!("Sending a read spanning two extents"); - block_io.read(offset, &mut data).await?; + println!("sub_volume {index} Sending a read spanning two extents"); + block_io.read(offset, &mut data).await?; - let dl = data.into_bytes(); - match validate_vec(dl, block_index, &mut ri.write_log, ri.block_size, false) - { - ValidateStatus::Bad | ValidateStatus::InRange => { - bail!("Span read verify failed"); + let dl = data.into_bytes(); + match validate_vec( + dl, + block_index, + &mut ri.write_log, + ri.block_size, + false, + ) { + ValidateStatus::Bad | ValidateStatus::InRange => { + bail!("sub_volume {index} Span read verify failed"); + } + ValidateStatus::Good => {} } - ValidateStatus::Good => {} } Ok(()) } diff --git a/crutest/src/protocol.rs b/crutest/src/protocol.rs index 291b07320..1dc50a3fb 100644 --- a/crutest/src/protocol.rs +++ b/crutest/src/protocol.rs @@ -34,9 +34,11 @@ pub enum CliMessage { Export, // Run the fill test. Fill(bool), + // Run the sparse fill workload. + FillSparse, Flush, Generic(usize, bool), - Info(u64, u64, u64), + Info(RegionInfo), InfoPlease, IsActive, MyUuid(Uuid), @@ -219,7 +221,15 @@ mod tests { #[test] fn rt_info() -> Result<()> { - let input = CliMessage::Info(1, 2, 99); + let ri = RegionInfo { + block_size: 512, + sub_volume_info: Vec::new(), + total_size: 100, + total_blocks: 5, + write_log: WriteLog::new(0), + max_block_io: 45, + }; + let input = CliMessage::Info(ri); assert_eq!(input, round_trip(&input)?); Ok(()) } diff --git a/tools/test_live_repair.sh b/tools/test_live_repair.sh index d73d84535..468cad9d6 100755 --- a/tools/test_live_repair.sh +++ b/tools/test_live_repair.sh @@ -66,6 +66,11 @@ echo "" > "$test_log" echo "starting $(date)" | tee "$loop_log" echo "Tail $test_log for test output" +# NOTE: we are creating a single region set here plus one more region to be +# used by the replacement, and with the assumption that # the default ports +# will be used (8810, 8820, 8830). The test relies on that because we use the +# fourth region-dir for our "replacement". If you change # the number of +# regions, you must also adjust the replacement below. if ! ${dsc} create --cleanup \ --region-dir "$REGION_ROOT" \ --region-count 4 \ @@ -85,14 +90,9 @@ if ! ps -p $dsc_pid > /dev/null; then exit 1 fi -args=() -args+=( -t "127.0.0.1:8810" ) -args+=( -t "127.0.0.1:8820" ) -args+=( -t "127.0.0.1:8830" ) - gen=1 # Initial seed for verify file -if ! "$crucible_test" fill "${args[@]}" -q -g "$gen"\ +if ! "$crucible_test" fill --dsc 127.0.0.1:9998 -q -g "$gen"\ --verify-out "$verify_log" >> "$test_log" 2>&1 ; then echo Failed on initial verify seed, check "$test_log" ${dsc} cmd shutdown @@ -107,7 +107,8 @@ while [[ $count -le $loops ]]; do cp "$test_log" "$test_log".last echo "" > "$test_log" echo "New loop, $count starts now $(date)" >> "$test_log" - "$crucible_test" replace "${args[@]}" -c 5 \ + "$crucible_test" replace -c 5 \ + --dsc 127.0.0.1:9998 \ --replacement 127.0.0.1:8840 \ --stable -g "$gen" --verify-out "$verify_log" \ --verify-at-start \ diff --git a/tools/test_replace_special.sh b/tools/test_replace_special.sh index fee0b4d3a..128ef7712 100755 --- a/tools/test_replace_special.sh +++ b/tools/test_replace_special.sh @@ -63,6 +63,11 @@ echo "" > "$test_log" echo "starting $(date)" | tee "$loop_log" echo "Tail $test_log for test output" +# NOTE: we are creating a single region set here plus one more region to be +# used by the replacement, and with the assumption that # the default ports +# will be used (8810, 8820, 8830). The test relies on that # because we use +# the fourth region-dir for our "replacement". If you change # the number of +# regions, you must also adjust the replacement below. if ! ${dsc} create --cleanup \ --region-dir "$REGION_ROOT" \ --region-count 4 \ @@ -82,14 +87,9 @@ if ! ps -p $dsc_pid > /dev/null; then exit 1 fi -args=() -args+=( -t "127.0.0.1:8810" ) -args+=( -t "127.0.0.1:8820" ) -args+=( -t "127.0.0.1:8830" ) - gen=1 # Initial seed for verify file -if ! "$crucible_test" fill "${args[@]}" -q -g "$gen"\ +if ! "$crucible_test" fill --dsc 127.0.0.1:9998 -q -g "$gen"\ --verify-out "$verify_log" >> "$test_log" 2>&1 ; then echo Failed on initial verify seed, check "$test_log" ${dsc} cmd shutdown @@ -104,7 +104,8 @@ while [[ $count -le $loops ]]; do cp "$test_log" "$test_log".last echo "" > "$test_log" echo "New loop, $count starts now $(date)" >> "$test_log" - "$crucible_test" replace-reconcile "${args[@]}" -c 5 \ + "$crucible_test" replace-reconcile -c 5 \ + --dsc 127.0.0.1:9998 \ --replacement 127.0.0.1:8840 \ --stable -g "$gen" --verify-out "$verify_log" \ --verify-at-start \ diff --git a/tools/test_up.sh b/tools/test_up.sh index 64389f473..9ee2d5fde 100755 --- a/tools/test_up.sh +++ b/tools/test_up.sh @@ -57,6 +57,7 @@ fail_log="${log_prefix}_fail.txt" rm -f "$fail_log" args=() +hammer_args=() dsc_args=() dsc_create_args=() dump_args=() @@ -86,6 +87,7 @@ case ${1} in upstairs_key=$(openssl rand -base64 32) echo "Upstairs using key: $upstairs_key" args+=( --key "$upstairs_key" ) + hammer_args+=( --key "$upstairs_key" ) dsc_create_args+=( --encrypted ) ;; *) @@ -93,6 +95,8 @@ case ${1} in ;; esac +args+=( --dsc "127.0.0.1:9998" ) + dsc_output_dir="${test_output_dir}/dsc" mkdir -p ${dsc_output_dir} dsc_output="${test_output_dir}/dsc-out.txt" @@ -100,16 +104,6 @@ dsc_output="${test_output_dir}/dsc-out.txt" dsc_create_args+=( --cleanup ) dsc_args+=( --output-dir "$dsc_output_dir" ) dsc_args+=( --ds-bin "$cds" ) - -# Note, this should match the default for DSC -port_base=8810 -# Build the upstairs args -for (( i = 0; i < 3; i++ )); do - (( port_step = i * 10 )) - (( port = port_base + port_step )) - args+=( -t "127.0.0.1:$port" ) -done - dsc_args+=( --region-dir "$testdir" ) echo "dsc output goes to $dsc_output" @@ -176,8 +170,11 @@ done echo "" >> "${log_prefix}_out.txt" echo "Running hammer" | tee -a "${log_prefix}_out.txt" -echo "$ch" -g "$gen" "${args[@]}" >> "${log_prefix}_out.txt" -if ! "$ch" -g "$gen" "${args[@]}" >> "${log_prefix}_out.txt" 2>&1; then +hammer_args+=( -t "127.0.0.1:8810") +hammer_args+=( -t "127.0.0.1:8820") +hammer_args+=( -t "127.0.0.1:8830") +echo "$ch" -g "$gen" "${hammer_args[@]}" >> "${log_prefix}_out.txt" +if ! "$ch" -g "$gen" "${hammer_args[@]}" >> "${log_prefix}_out.txt" 2>&1; then echo "Failed hammer test" echo "Failed hammer test" >> "$fail_log" (( res += 1 )) @@ -208,8 +205,8 @@ else fi (( gen += 1 )) -echo "Copy the $port file" | tee -a "${log_prefix}_out.txt" - +port=8830 +echo "Copy the region for ${testdir}/$port" | tee -a "${log_prefix}_out.txt" echo cp -r "${testdir}/${port}" "${testdir}/previous" cp -r "${testdir}/${port}" "${testdir}/previous" @@ -265,6 +262,9 @@ fi # Put a dump test in the middle of the repair test, so we # can see both a mismatch and that dump works. # The dump args look different than other downstairs commands +# This port base comes from the default for dsc. If that changes, then this +# needs to change as well. +port_base=8810 for (( i = 0; i < 30; i += 10 )); do (( port = port_base + i )) dir="${testdir}/$port" diff --git a/upstairs/src/block_io.rs b/upstairs/src/block_io.rs index 739dcb4c6..c133a2c25 100644 --- a/upstairs/src/block_io.rs +++ b/upstairs/src/block_io.rs @@ -48,8 +48,10 @@ impl BlockIO for FileBlockIO { Ok(()) } - async fn query_extent_size(&self) -> Result { - crucible_bail!(Unsupported, "query_extent_size unsupported",) + async fn query_extent_info( + &self, + ) -> Result, CrucibleError> { + crucible_bail!(Unsupported, "query_extent_info unsupported",) } async fn query_work_queue(&self) -> Result { @@ -216,9 +218,11 @@ impl BlockIO for ReqwestBlockIO { }) } - async fn query_extent_size(&self) -> Result { + async fn query_extent_info( + &self, + ) -> Result, CrucibleError> { Err(CrucibleError::Unsupported( - "query_extent_size unsupported".to_string(), + "query_extent_info unsupported".to_string(), )) } diff --git a/upstairs/src/guest.rs b/upstairs/src/guest.rs index 684926871..d9f0f21ae 100644 --- a/upstairs/src/guest.rs +++ b/upstairs/src/guest.rs @@ -13,7 +13,7 @@ use crate::{ BlockIO, BlockOp, BlockOpWaiter, BlockRes, Buffer, JobId, RawReadResponse, ReplaceResult, UpstairsAction, }; -use crucible_common::{build_logger, Block, BlockIndex, CrucibleError}; +use crucible_common::{build_logger, BlockIndex, CrucibleError, ExtentInfo}; use crucible_protocol::SnapshotDetails; use async_trait::async_trait; @@ -389,9 +389,14 @@ impl Guest { rx.wait().await } - pub async fn query_extent_size(&self) -> Result { - self.send_and_wait(|done| BlockOp::QueryExtentSize { done }) - .await + pub async fn query_extent_info( + &self, + ) -> Result, CrucibleError> { + let block = self + .send_and_wait(|done| BlockOp::QueryExtentInfo { done }) + .await?; + + Ok(vec![block]) } pub async fn query_work_queue(&self) -> Result { @@ -511,9 +516,14 @@ impl BlockIO for Guest { .await } - async fn query_extent_size(&self) -> Result { - self.send_and_wait(|done| BlockOp::QueryExtentSize { done }) - .await + async fn query_extent_info( + &self, + ) -> Result, CrucibleError> { + let block = self + .send_and_wait(|done| BlockOp::QueryExtentInfo { done }) + .await?; + + Ok(vec![block]) } async fn total_size(&self) -> Result { diff --git a/upstairs/src/in_memory.rs b/upstairs/src/in_memory.rs index bdda4bd6f..6e3ef918e 100644 --- a/upstairs/src/in_memory.rs +++ b/upstairs/src/in_memory.rs @@ -41,8 +41,10 @@ impl BlockIO for InMemoryBlockIO { Ok(()) } - async fn query_extent_size(&self) -> Result { - crucible_bail!(Unsupported, "query_extent_size unsupported",) + async fn query_extent_info( + &self, + ) -> Result, CrucibleError> { + crucible_bail!(Unsupported, "query_extent_info unsupported",) } async fn query_work_queue(&self) -> Result { diff --git a/upstairs/src/lib.rs b/upstairs/src/lib.rs index c31a7e699..04714f46d 100644 --- a/upstairs/src/lib.rs +++ b/upstairs/src/lib.rs @@ -110,7 +110,8 @@ pub trait BlockIO: Sync { async fn query_is_active(&self) -> Result; - async fn query_extent_size(&self) -> Result; + async fn query_extent_info(&self) + -> Result, CrucibleError>; async fn query_work_queue(&self) -> Result; // Total bytes of Volume @@ -1596,8 +1597,8 @@ pub(crate) enum BlockOp { done: BlockRes, }, // Begin testing options. - QueryExtentSize { - done: BlockRes, + QueryExtentInfo { + done: BlockRes, }, QueryWorkQueue { done: BlockRes, diff --git a/upstairs/src/upstairs.rs b/upstairs/src/upstairs.rs index 4e5a532ff..9c693574c 100644 --- a/upstairs/src/upstairs.rs +++ b/upstairs/src/upstairs.rs @@ -14,7 +14,7 @@ use crate::{ guest::GuestBlockRes, stats::UpStatOuter, BlockOp, BlockRes, Buffer, ClientId, ClientMap, CrucibleOpts, DsState, - EncryptionContext, GuestIoHandle, Message, RegionDefinition, + EncryptionContext, ExtentInfo, GuestIoHandle, Message, RegionDefinition, RegionDefinitionStatus, SnapshotDetails, WQCounts, }; use crucible_common::{BlockIndex, CrucibleError}; @@ -1049,11 +1049,15 @@ impl Upstairs { }; } // Testing options - BlockOp::QueryExtentSize { done } => { + BlockOp::QueryExtentInfo { done } => { // Yes, test only match self.ddef.get_def() { Some(rd) => { - done.send_ok(rd.extent_size()); + let ei = ExtentInfo { + extent_size: rd.extent_size(), + extent_count: rd.extent_count(), + }; + done.send_ok(ei); } None => { warn!( diff --git a/upstairs/src/volume.rs b/upstairs/src/volume.rs index e70b24ea1..aad7d8e22 100644 --- a/upstairs/src/volume.rs +++ b/upstairs/src/volume.rs @@ -652,22 +652,32 @@ impl BlockIO for Volume { Ok(all_wq) } - // Return a vec of these? - // Return a struct with a vec for SV and Some/None for ROP? - async fn query_extent_size(&self) -> Result { - // ZZZ this needs more info, what if ROP and SV differ? + // Return a single Vec that contains all the extent_info from the + // sub-volumes we have, or if none, then whatever the read only parent + // has. We don't currently support returning extent info if we have both + // sub_volumes and a read_only_parent, but as we are the only consumer of + // this interface, we can add that if it becomes a requirement. + async fn query_extent_info( + &self, + ) -> Result, CrucibleError> { + let mut extent_info = Vec::new(); for sub_volume in &self.sub_volumes { - match sub_volume.query_extent_size().await { - Ok(es) => { - return Ok(es); + match sub_volume.query_extent_info().await { + Ok(ei) => { + for sv_ei in ei.iter() { + extent_info.push(*sv_ei); + } } _ => { continue; } } } + if !extent_info.is_empty() { + return Ok(extent_info); + } if let Some(ref read_only_parent) = &self.read_only_parent { - return read_only_parent.query_extent_size().await; + return read_only_parent.query_extent_info().await; } crucible_bail!(IoError, "Cannot determine extent size"); } @@ -1033,8 +1043,10 @@ impl BlockIO for SubVolume { async fn query_work_queue(&self) -> Result { self.block_io.query_work_queue().await } - async fn query_extent_size(&self) -> Result { - self.block_io.query_extent_size().await + async fn query_extent_info( + &self, + ) -> Result, CrucibleError> { + self.block_io.query_extent_info().await } async fn deactivate(&self) -> Result<(), CrucibleError> {