From 9fbdc14af044044daeb54163571c719187962a01 Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxide.computer>
Date: Thu, 27 Feb 2025 16:25:07 +0000
Subject: [PATCH 01/21] Test improvements for nightly tests

---
 tools/hammer_loop.sh          | 42 ++++++++++++++++++++++-------
 tools/test_live_repair.sh     | 36 ++++++++++++++++++++-----
 tools/test_nightly.sh         | 51 ++++++++++++++++++++---------------
 tools/test_repair.sh          | 24 +++++++++++++----
 tools/test_replace_special.sh | 30 ++++++++++++++++-----
 tools/test_replay.sh          | 20 +++++++++++---
 tools/test_restart_repair.sh  | 39 +++++++++++++++++++++------
 7 files changed, 183 insertions(+), 59 deletions(-)

diff --git a/tools/hammer_loop.sh b/tools/hammer_loop.sh
index 543a360a9..beefcf476 100755
--- a/tools/hammer_loop.sh
+++ b/tools/hammer_loop.sh
@@ -29,6 +29,23 @@ if pgrep -fl -U "$(id -u)" "$cds"; then
     exit 1
 fi
 
+WORK_ROOT=${WORK_ROOT:-/tmp}
+TEST_ROOT="$WORK_ROOT/hammer_loop"
+REGION_ROOT=${REGION_ROOT:-/var/tmp/hammer_loop}
+loop_log="$TEST_ROOT/hammer_loop.log"
+test_log="$TEST_ROOT/hammer_loop_test.log"
+
+if [[ ! -d "$TEST_ROOT" ]]; then
+    mkdir -p "$TEST_ROOT"
+    if [[ $? -ne 0 ]]; then
+        echo "Failed to make test root $TEST_ROOT"
+        exit 1
+    fi
+else
+    # Delete previous test data
+    rm -r "$TEST_ROOT"
+fi
+
 loops=20
 
 usage () {
@@ -37,23 +54,25 @@ usage () {
 }
 
 while getopts 'l:' opt; do
-	case "$opt" in
+    case "$opt" in
         l)  loops=$OPTARG
             ;;
         *)  echo "Invalid option"
             usage
-			exit 1
-			;;
-	esac
+            exit 1
+            ;;
+    esac
 done
 
-if ! "$dsc" create --cleanup --ds-bin "$cds" --extent-count 60 --extent-size 50; then
+if ! "$dsc" create --cleanup --ds-bin "$cds" --extent-count 60 \
+    --extent-size 50 --region-dir "$REGION_ROOT"
+then
     echo "Failed to create region"
     exit 1
 fi
 
 # Start up dsc, verify it really did start.
-"$dsc" start --ds-bin "$cds" &
+"$dsc" start --ds-bin "$cds" --region-dir "$REGION_ROOT" &
 dsc_pid=$!
 sleep 5
 if ! pgrep -P $dsc_pid; then
@@ -78,9 +97,6 @@ function ctrl_c() {
     fi
     exit 1
 }
-
-loop_log=/tmp/hammer_loop.log
-test_log=/tmp/hammer_loop_test.log
 echo "" > ${loop_log}
 echo "starting Hammer test on $(date)" | tee ${loop_log}
 echo "Tail $test_log for test output"
@@ -145,5 +161,11 @@ if pgrep -fl -U "$(id -u)" "$cds" > /dev/null; then
     pkill -f -U "$(id -u)" "$cds"
 fi
 
+if [[ $err -eq 0 ]]; then
+    # No errors, then cleanup all our logs and the region directories.
+    rm -r "$TEST_ROOT"
+    rm -r "$REGION_ROOT"/8810
+    rm -r "$REGION_ROOT"/8820
+    rm -r "$REGION_ROOT"/8830
+fi
 exit "$err"
-
diff --git a/tools/test_live_repair.sh b/tools/test_live_repair.sh
index 10ce4a330..fcce246ca 100755
--- a/tools/test_live_repair.sh
+++ b/tools/test_live_repair.sh
@@ -26,11 +26,22 @@ mkdir -p "$REGION_ROOT"
 
 # Location of logs and working files
 WORK_ROOT=${WORK_ROOT:-/tmp}
-mkdir -p "$WORK_ROOT"
+TEST_ROOT="$WORK_ROOT/test_live_repair"
+if [[ ! -d "$TEST_ROOT" ]]; then
+    mkdir -p "$TEST_ROOT"
+    if [[ $? -ne 0 ]]; then
+        echo "Failed to make test root $TEST_ROOT"
+    exit 1
+    fi  
+else
+    # Delete previous test data
+    rm -r "$TEST_ROOT"
+fi
 
-loop_log="$WORK_ROOT"/test_live_repair_summary.log
-test_log="$WORK_ROOT"/test_live_repair.log
-verify_log="$WORK_ROOT/test_live_repair_verify.log"
+loop_log="$TEST_ROOT"/test_live_repair_summary.log
+test_log="$TEST_ROOT"/test_live_repair.log
+verify_log="$TEST_ROOT/test_live_repair_verify.log"
+dsc_ds_log="$TEST_ROOT/test_live_repair_dsc.log"
 
 ROOT=$(cd "$(dirname "$0")/.." && pwd)
 cd "$ROOT" || (echo failed to cd "$ROOT"; exit 1)
@@ -68,8 +79,8 @@ done
 
 ((region_count=region_sets*3))
 ((region_count+=1))
-echo "" > "$loop_log"
-echo "" > "$test_log"
+rm -f "$loop_log"
+rm -f "$test_log"
 echo "starting $(date)" | tee "$loop_log"
 echo "Tail $test_log for test output"
 
@@ -91,6 +102,7 @@ fi
 if ! ${dsc} create --cleanup \
   --region-dir "$REGION_ROOT" \
   --region-count "$region_count" \
+  --output-dir "$dsc_ds_log" \
   --ds-bin "$downstairs" \
   --extent-size "$extent_size" \
   --extent-count 200 >> "$test_log"; then
@@ -99,6 +111,7 @@ if ! ${dsc} create --cleanup \
 fi
 ${dsc} start --ds-bin "$downstairs" \
   --region-dir "$REGION_ROOT" \
+  --output-dir "$dsc_ds_log" \
   --region-count "$region_count" >> "$test_log" 2>&1 &
 dsc_pid=$!
 sleep 5
@@ -148,4 +161,15 @@ ${dsc} cmd shutdown
 wait "$dsc_pid"
 
 echo "$(date) Test ends with $result" | tee -a "$test_log"
+
+if [[ $result -eq 0 ]]; then
+    echo "DELETE 8840" | tee -a "$test_log"
+    ps -ef | grep crucible-downstairs
+    rm -rf "$REGION_ROOT"/8810
+    rm -rf "$REGION_ROOT"/8820
+    rm -rf "$REGION_ROOT"/8830
+    rm -rf "$REGION_ROOT"/8840
+    rm -rf "$TEST_ROOT"
+fi
+
 exit $result
diff --git a/tools/test_nightly.sh b/tools/test_nightly.sh
index 2af9c37c7..4b6763b28 100755
--- a/tools/test_nightly.sh
+++ b/tools/test_nightly.sh
@@ -10,6 +10,9 @@ err=0
 output_file="/tmp/nightly_results"
 rm -f "$output_file"
 
+REGION_ROOT=/regions/ubuntu/nightly
+echo "using REGION_ROOT=$REGION_ROOT"
+
 ROOT=$(cd "$(dirname "$0")/.." && pwd)
 cd "$ROOT" || (echo failed to cd "$ROOT"; exit 1)
 export BINDIR=${BINDIR:-$ROOT/target/release}
@@ -17,6 +20,7 @@ export BINDIR=${BINDIR:-$ROOT/target/release}
 echo "Nightly starts at $(date)" | tee "$output_file"
 echo "$(date) hammer start" >> "$output_file"
 banner hammer
+banner loop
 ./tools/hammer_loop.sh -l 200
 res=$?
 if [[ "$res" -eq 0 ]]; then
@@ -26,58 +30,63 @@ else
     (( err += 1 ))
 fi
 
+banner test
 banner replay
-echo "$(date) replay start" >> "$output_file"
+echo "$(date) test_replay start" >> "$output_file"
 ./tools/test_replay.sh -l 200
 res=$?
 if [[ "$res" -eq 0 ]]; then
-    echo "$(date) replay pass" >> "$output_file"
+    echo "$(date) test_replay pass" >> "$output_file"
 else
-    echo "$(date) replay fail with: $res" >> "$output_file"
+    echo "$(date) test_replay fail with: $res" >> "$output_file"
     (( err += 1 ))
 fi
 
+banner "test"
 banner repair
-echo "$(date) repair start" >> "$output_file"
+echo "$(date) test_repair start" >> "$output_file"
 ./tools/test_repair.sh -l 500
 res=$?
 if [[ "$res" -eq 0 ]]; then
-    echo "$(date) repair pass" >> "$output_file"
+    echo "$(date) test_repair pass" >> "$output_file"
 else
-    echo "$(date) repair fail with: $res" >> "$output_file"
+    echo "$(date) test_repair fail with: $res" >> "$output_file"
     (( err += 1 ))
 fi
 
-banner restart_repair
-echo "$(date) restart_repair start" >> "$output_file"
-./tools/test_restart_repair.sh -l 200
+banner restart
+banner repair
+echo "$(date) test_restart_repair start" >> "$output_file"
+./tools/test_restart_repair.sh -l 50
 res=$?
 if [[ "$res" -eq 0 ]]; then
-    echo "$(date) restart_repair pass" >> "$output_file"
+    echo "$(date) test_restart_repair pass" >> "$output_file"
 else
-    echo "$(date) restart_repair fail with: $res" >> "$output_file"
+    echo "$(date) test_restart_repair fail with: $res" >> "$output_file"
     (( err += 1 ))
 fi
 
-banner live_repair
-echo "$(date) live_repair start" >> "$output_file"
+banner live
+banner repair
+echo "$(date) test_live_repair start" >> "$output_file"
 ./tools/test_live_repair.sh -l 20
 res=$?
 if [[ "$res" -eq 0 ]]; then
-    echo "$(date) live_repair pass" >> "$output_file"
+    echo "$(date) test_live_repair pass" >> "$output_file"
 else
-    echo "$(date) live_repair fail with: $res" >> "$output_file"
+    echo "$(date) test_live_repair fail with: $res" >> "$output_file"
     (( err += 1 ))
 fi
 
-banner replace_reconcile
-echo "$(date) replace_reconcile start" >> "$output_file"
-./tools/test_replace_special.sh -l 20
+banner replace
+banner special
+echo "$(date) test_replace_special start" >> "$output_file"
+./tools/test_replace_special.sh -l 30
 res=$?
 if [[ "$res" -eq 0 ]]; then
-    echo "$(date) replace_reconcile pass" >> "$output_file"
+    echo "$(date) test_replace_special pass" >> "$output_file"
 else
-    echo "$(date) replace_reconcile fail with: $res" >> "$output_file"
+    echo "$(date) test_replace_special fail with: $res" >> "$output_file"
     (( err += 1 ))
 fi
 duration=$SECONDS
@@ -85,5 +94,5 @@ duration=$SECONDS
 banner results
 cat "$output_file"
 printf "Tests took %d:%02d  errors:%d\n" \
-    $((duration / 60)) $((duration % 60)) "$err"
+    $((duration / 60)) $((duration % 60)) "$err" | tee -a "$output_file"
 
diff --git a/tools/test_repair.sh b/tools/test_repair.sh
index d7ec0ba55..03a2a4104 100755
--- a/tools/test_repair.sh
+++ b/tools/test_repair.sh
@@ -48,12 +48,22 @@ fi
 
 # Location of logs and working files
 WORK_ROOT=${WORK_ROOT:-/tmp}
-mkdir -p "$WORK_ROOT"
+TEST_ROOT="$WORK_ROOT/test_live_repair"
+if [[ ! -d "$TEST_ROOT" ]]; then
+    mkdir -p "$TEST_ROOT"
+    if [[ $? -ne 0 ]]; then
+        echo "Failed to make test root $TEST_ROOT"
+        exit 1
+    fi
+else
+    # Delete previous test data
+    rm -r "$TEST_ROOT"
+fi
 
-verify_file="$WORK_ROOT/test_repair_verify.data"
-test_log="$WORK_ROOT/test_repair_out.txt"
-ds_log_prefix="$WORK_ROOT/test_repair_ds"
-dsc_output_dir="$WORK_ROOT/dsc"
+verify_file="$TEST_ROOT/test_repair_verify.data"
+test_log="$TEST_ROOT/test_repair_out.txt"
+ds_log_prefix="$TEST_ROOT/test_repair_ds"
+dsc_output_dir="$TEST_ROOT/dsc"
 loops=100
 
 usage () {
@@ -224,3 +234,7 @@ duration=$SECONDS
 printf "%d:%02d Test duration\n" $((duration / 60)) $((duration % 60))
 echo "Test completed"
 cleanup
+
+# Errors exit directly, so arrival here indicates success.
+# rm -rf "$REGION_ROOT"/8810
+rm -rf "$TEST_ROOT"
diff --git a/tools/test_replace_special.sh b/tools/test_replace_special.sh
index cd9bd7b5d..ecac3afcb 100755
--- a/tools/test_replace_special.sh
+++ b/tools/test_replace_special.sh
@@ -22,12 +22,22 @@ mkdir -p "$REGION_ROOT"
 
 # Location of logs and working files
 WORK_ROOT=${WORK_ROOT:-/tmp}
-mkdir -p "$WORK_ROOT"
-
-loop_log="$WORK_ROOT"/test_replace_special_summary.log
-test_log="$WORK_ROOT"/test_replace_special.log
-verify_log="$WORK_ROOT/test_replace_special_verify.log"
+TEST_ROOT="${WORK_ROOT}/test_replace_special"
+if [[ ! -d "$TEST_ROOT" ]]; then
+    mkdir -p "$TEST_ROOT"
+    if [[ $? -ne 0 ]]; then
+        echo "Failed to make test root $TEST_ROOT"
+        exit 1
+    fi
+else
+    # Delete previous test data
+    rm -r "$TEST_ROOT"
+fi
 
+loop_log="$TEST_ROOT/test_replace_special_summary.log"
+test_log="$TEST_ROOT/test_replace_special.log"
+verify_log="$TEST_ROOT/test_replace_special_verify.log"
+dsc_ds_log="$TEST_ROOT/test_replace_special_dsc.log"
 
 ROOT=$(cd "$(dirname "$0")/.." && pwd)
 cd "$ROOT" || (echo failed to cd "$ROOT"; exit 1)
@@ -76,6 +86,7 @@ echo "Tail $test_log for test output"
 if ! ${dsc} create --cleanup \
   --region-dir "$REGION_ROOT" \
   --region-count "$region_count" \
+  --output-dir "$dsc_ds_log" \
   --ds-bin "$downstairs" \
   --extent-count 400 \
   --block-size 4096 >> "$test_log"; then
@@ -84,6 +95,7 @@ if ! ${dsc} create --cleanup \
 fi
 ${dsc} start --ds-bin "$downstairs" \
   --region-dir "$REGION_ROOT" \
+  --output-dir "$dsc_ds_log" \
   --region-count "$region_count" >> "$test_log" 2>&1 &
 dsc_pid=$!
 sleep 5
@@ -109,7 +121,7 @@ replacement_port=$(${dsc} cmd port -c $last_client)
 
 # Now run the crutest replace-reconcile test
 SECONDS=0
-cp "$test_log" "$test_log".last
+cp "$test_log" "$test_log".fill
 echo "" > "$test_log"
 echo "$(date) replace-reconcile starts now" | tee -a "$test_log"
 "$crucible_test" replace-reconcile -c "$loops" --dsc 127.0.0.1:9998 \
@@ -130,4 +142,10 @@ ${dsc} cmd shutdown
 wait "$dsc_pid"
 
 echo "$(date) Test ends with $result" | tee -a "$test_log"
+
+if [[ $result -eq 0 ]]; then
+    # Cleanup
+    echo "$(date) Cleanup for $0" | tee -a "$test_log"
+    rm -rf "$TEST_ROOT"
+fi
 exit $result
diff --git a/tools/test_replay.sh b/tools/test_replay.sh
index 2a777e844..dbd19ca21 100755
--- a/tools/test_replay.sh
+++ b/tools/test_replay.sh
@@ -16,10 +16,21 @@ function ctrl_c() {
 }
 
 WORK_ROOT=${WORK_ROOT:-/tmp}
-mkdir -p "$WORK_ROOT"
+TEST_ROOT="$WORK_ROOT/test_replay"
 
-test_log="$WORK_ROOT/test_replay.log"
-verify_log="$WORK_ROOT/test_replay_verify.log"
+if [[ ! -d "$TEST_ROOT" ]]; then
+    mkdir -p "$TEST_ROOT"
+    if [[ $? -ne 0 ]]; then
+        echo "Failed to make test root $TEST_ROOT"
+        exit 1
+    fi
+else
+    # Delete previous test data
+    rm -r "$TEST_ROOT"
+fi
+
+test_log="$TEST_ROOT/test_replay.log"
+verify_log="$TEST_ROOT/test_replay_verify.log"
 
 ROOT=$(cd "$(dirname "$0")/.." && pwd)
 cd "$ROOT" || (echo failed to cd "$ROOT"; exit 1)
@@ -122,4 +133,7 @@ wait "$dsc_pid"
 
 sleep 4
 echo "$(date) Test ends with $result" | tee -a "$test_log" 2>&1
+if [[ $result -eq 0 ]]; then
+    rm -r "$TEST_ROOT"
+fi
 exit "$result"
diff --git a/tools/test_restart_repair.sh b/tools/test_restart_repair.sh
index 7cd42bca7..3dfb9d1f8 100755
--- a/tools/test_restart_repair.sh
+++ b/tools/test_restart_repair.sh
@@ -98,28 +98,43 @@ while getopts 'l:' opt; do
 done
 
 WORK_ROOT=${WORK_ROOT:-/tmp}
-export loop_log="$WORK_ROOT/repair_restart.log"
-export test_log="$WORK_ROOT/repair_restart_test.log"
-export dsc_log="$WORK_ROOT/repair_restart_dsc.log"
-export verify_log="$WORK_ROOT/repair_restart_verify.log"
+TEST_ROOT="$WORK_ROOT/test_restart_repair"
 REGION_ROOT=${REGION_ROOT:-/var/tmp/test_restart_repair}
+export loop_log="$TEST_ROOT/test_restart_repair.log"
+export test_log="$TEST_ROOT/test_restart_repair_test.log"
+export verify_log="$TEST_ROOT/test_restart_repair_verify.log"
+export dsc_log="$TEST_ROOT/test_restart_repair_dsc.log"
+export dsc_ds_log="$TEST_ROOT/dsc"
+
+if [[ ! -d "$TEST_ROOT" ]]; then
+    mkdir -p "$TEST_ROOT"
+    if [[ $? -ne 0 ]]; then
+        echo "Failed to make test root $TEST_ROOT"
+        exit 1
+    fi
+else
+    # Delete previous test data
+    rm -r "$TEST_ROOT"
+fi
 
-echo "" > "$loop_log"
 echo "starting $(date)" | tee "$loop_log"
-echo "" > "$test_log"
 echo "Tail $test_log for test output"
 echo "Tail $loop_log for summary output"
 echo "Tail $dsc_log for dsc outout"
 
 echo "Create a new region to test" | tee -a "${loop_log}"
 ulimit -n 65536
-if ! "$dsc" create --cleanup --ds-bin "$cds" --extent-count 61 --extent-size 5120 --region-dir "$REGION_ROOT"; then
+if ! "$dsc" create --cleanup --ds-bin "$cds" --extent-count 61 \
+    --extent-size 5120 --output-dir "$dsc_ds_log" \
+    --region-dir "$REGION_ROOT"
+then
     echo "Failed to create region at $REGION_ROOT"
     exit 1
 fi
 
 echo "Starting the downstairs" | tee -a "${loop_log}"
-"$dsc" start --ds-bin "$cds" --region-dir "$REGION_ROOT" >> "$dsc_log" 2>&1 &
+"$dsc" start --ds-bin "$cds" --region-dir "$REGION_ROOT" \
+    --output-dir "$dsc_ds_log" >> "$dsc_log" 2>&1 &
 dsc_pid=$!
 # Sleep 5 to give the downstairs time to get going.
 sleep 5
@@ -273,4 +288,12 @@ printf "[%03d] %d:%02d  ave:%d:%02d  total:%d:%02d errors:%d last_run_seconds:%d
   $((ave / 60)) $((ave % 60)) \
   $((total / 60)) $((total % 60)) \
   "$err" $duration | tee -a "$loop_log"
+
+if [[ $err -eq 0 ]]; then
+    # No errors, then cleanup all our logs and the region directories.
+    rm -r "$TEST_ROOT"
+    rm -r "$REGION_ROOT"/8810
+    rm -r "$REGION_ROOT"/8820
+    rm -r "$REGION_ROOT"/8830
+fi
 exit "$err"

From 1527e227e57f65cd2b89afa211558f37894a5944 Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxide.computer>
Date: Mon, 3 Mar 2025 16:49:00 +0000
Subject: [PATCH 02/21] Further polish, use dsc when possible

---
 tools/test_live_repair.sh    | 10 +++++-----
 tools/test_nightly.sh        | 24 ++++++++++++++++++++++++
 tools/test_repair.sh         | 28 ++++++++++++++++++----------
 tools/test_restart_repair.sh | 19 +++++++------------
 4 files changed, 54 insertions(+), 27 deletions(-)

diff --git a/tools/test_live_repair.sh b/tools/test_live_repair.sh
index fcce246ca..7ab960d0e 100755
--- a/tools/test_live_repair.sh
+++ b/tools/test_live_repair.sh
@@ -31,7 +31,7 @@ if [[ ! -d "$TEST_ROOT" ]]; then
     mkdir -p "$TEST_ROOT"
     if [[ $? -ne 0 ]]; then
         echo "Failed to make test root $TEST_ROOT"
-    exit 1
+        exit 1
     fi  
 else
     # Delete previous test data
@@ -79,9 +79,8 @@ done
 
 ((region_count=region_sets*3))
 ((region_count+=1))
-rm -f "$loop_log"
-rm -f "$test_log"
-echo "starting $(date)" | tee "$loop_log"
+echo "Starting $(date)" > "$test_log"
+echo "starting $(date)" > "$loop_log"
 echo "Tail $test_log for test output"
 
 # No real data was used to come up with these numbers.  If you have some data
@@ -105,7 +104,8 @@ if ! ${dsc} create --cleanup \
   --output-dir "$dsc_ds_log" \
   --ds-bin "$downstairs" \
   --extent-size "$extent_size" \
-  --extent-count 200 >> "$test_log"; then
+  --extent-count 200 >> "$test_log"
+then
     echo "Failed to create downstairs regions"
     exit 1
 fi
diff --git a/tools/test_nightly.sh b/tools/test_nightly.sh
index 4b6763b28..329fffc83 100755
--- a/tools/test_nightly.sh
+++ b/tools/test_nightly.sh
@@ -30,6 +30,10 @@ else
     (( err += 1 ))
 fi
 
+echo "$(date) Next test"
+ps -ef | egrep "downstairs|dsc"
+echo ""
+
 banner test
 banner replay
 echo "$(date) test_replay start" >> "$output_file"
@@ -42,6 +46,10 @@ else
     (( err += 1 ))
 fi
 
+echo "$(date) Next test"
+ps -ef | egrep "downstairs|dsc"
+echo ""
+
 banner "test"
 banner repair
 echo "$(date) test_repair start" >> "$output_file"
@@ -52,8 +60,13 @@ if [[ "$res" -eq 0 ]]; then
 else
     echo "$(date) test_repair fail with: $res" >> "$output_file"
     (( err += 1 ))
+    exit 1
 fi
 
+echo "$(date) Next test"
+ps -ef | egrep "downstairs|dsc"
+echo ""
+
 banner restart
 banner repair
 echo "$(date) test_restart_repair start" >> "$output_file"
@@ -64,8 +77,13 @@ if [[ "$res" -eq 0 ]]; then
 else
     echo "$(date) test_restart_repair fail with: $res" >> "$output_file"
     (( err += 1 ))
+    exit 1
 fi
 
+echo "$(date) Next test"
+ps -ef | egrep "downstairs|dsc"
+echo ""
+
 banner live
 banner repair
 echo "$(date) test_live_repair start" >> "$output_file"
@@ -76,8 +94,13 @@ if [[ "$res" -eq 0 ]]; then
 else
     echo "$(date) test_live_repair fail with: $res" >> "$output_file"
     (( err += 1 ))
+    exit 1
 fi
 
+echo "$(date) Next test"
+ps -ef | egrep "downstairs|dsc"
+echo ""
+
 banner replace
 banner special
 echo "$(date) test_replace_special start" >> "$output_file"
@@ -88,6 +111,7 @@ if [[ "$res" -eq 0 ]]; then
 else
     echo "$(date) test_replace_special fail with: $res" >> "$output_file"
     (( err += 1 ))
+    exit 1
 fi
 duration=$SECONDS
 
diff --git a/tools/test_repair.sh b/tools/test_repair.sh
index 03a2a4104..358ed8c1d 100755
--- a/tools/test_repair.sh
+++ b/tools/test_repair.sh
@@ -34,7 +34,7 @@ dsc="$BINDIR/dsc"
 
 for bin in $cds $ct $dsc; do
     if [[ ! -f "$bin" ]]; then
-        echo "Can't find crucible binary at $bin" >&2
+        echo "Can't find required binary at $bin" >&2
         exit 1
     fi
 done
@@ -69,25 +69,27 @@ loops=100
 usage () {
     echo "Usage: $0 [-l #] [N]" >&2
     echo " -l loops   Number of test loops to perform (default 100)" >&2
-	echo " -N         Don't dump color output"
+    echo " -N         Don't dump color output"
 }
 
 dump_args=()
 while getopts 'l:N' opt; do
-	case "$opt" in
+    case "$opt" in
         l)  loops=$OPTARG
             ;;
-		N)  echo "Turn off color for downstairs dump"
+        N)  echo "Turn off color for downstairs dump"
             dump_args+=(" --no-color")
             ;;
         *)  echo "Invalid option"
             usage
-			exit 1
-			;;
-	esac
+            exit 1
+            ;;
+    esac
 done
 
-if ! "$dsc" create --cleanup --ds-bin "$cds" --extent-count 30 --extent-size 20 --region-dir "$REGION_ROOT" --output-dir "$dsc_output_dir"; then
+if ! "$dsc" create --cleanup --ds-bin "$cds" --extent-count 30 \
+        --extent-size 20 --region-dir "$REGION_ROOT" \
+        --output-dir "$dsc_output_dir"; then
     echo "Failed to create region"
     exit 1
 fi
@@ -114,6 +116,10 @@ ds1_pid=$!
 ${cds} run -d "${REGION_ROOT}/8830" -p 8830 &> "$ds_log_prefix"8830.txt &
 ds2_pid=$!
 
+# Some programatic way to wait for all the downstairs to start before we
+# continue here.
+sleep 20
+
 os_name=$(uname)
 if [[ "$os_name" == 'Darwin' ]]; then
     # stupid macos needs this to avoid popup hell.
@@ -209,7 +215,7 @@ while [[ $count -lt $loops ]]; do
     then
         echo "Exit on verify fail, loop: $count, choice: $choice"
         echo "Check $test_log for details"
-        cleanup
+	cleanup
         exit 1
     fi
     set +o errexit
@@ -236,5 +242,7 @@ echo "Test completed"
 cleanup
 
 # Errors exit directly, so arrival here indicates success.
-# rm -rf "$REGION_ROOT"/8810
 rm -rf "$TEST_ROOT"
+rm -rf "$REGION_ROOT"/8810
+rm -rf "$REGION_ROOT"/8820
+rm -rf "$REGION_ROOT"/8830
diff --git a/tools/test_restart_repair.sh b/tools/test_restart_repair.sh
index 3dfb9d1f8..01cdd7cb1 100755
--- a/tools/test_restart_repair.sh
+++ b/tools/test_restart_repair.sh
@@ -117,6 +117,7 @@ else
     rm -r "$TEST_ROOT"
 fi
 
+touch "$loop_log"
 echo "starting $(date)" | tee "$loop_log"
 echo "Tail $test_log for test output"
 echo "Tail $loop_log for summary output"
@@ -152,19 +153,12 @@ if [[ "$os_name" == 'Darwin' ]]; then
     codesign -s - -f "$ct"
 fi
 
-args=()
-port_base=8810
-for (( i = 0; i < 30; i += 10 )); do
-    (( port = port_base + i ))
-    args+=( -t "127.0.0.1:$port" )
-done
-
 gen=1
 # Send something to the region so our old region files have data.
 echo "$(date) pre-fill" >> "$test_log"
 echo "$(date) run pre-fill of our region" | tee -a "$loop_log"
-echo "$ct" fill "${args[@]}" --stable -g "$gen" >> "$test_log"
-"$ct" fill "${args[@]}" --stable -g "$gen" >> "$test_log" 2>&1
+echo "$ct" fill --dsc 127.0.0.1:9998 --stable -g "$gen" >> "$test_log"
+"$ct" fill --dsc 127.0.0.1:9998 --stable -g "$gen" >> "$test_log" 2>&1
 if [[ $? -ne 0 ]]; then
     echo "Error in initial pre-fill"
     ctrl_c
@@ -179,6 +173,7 @@ stop_all_downstairs
 # We need to do this before moving the region directory out from
 # under a downstairs, otherwise it can fail and exit and the
 # downstairs daemon will think it is a real failure.
+# Issue oxidecomputer/crucible#1660
 sleep 7
 
 # Create the "old" region files
@@ -195,8 +190,8 @@ bring_all_downstairs_online
 # different data in current vs. old region directories.
 echo "$(date) Run a second fill test" >> "$test_log"
 echo "$(date) Run a second fill test" | tee -a "$loop_log"
-echo "$ct" fill "${args[@]}" --stable -g "$gen" --verify-out "$verify_log" >> "$test_log"
-"$ct" fill "${args[@]}" --stable -g "$gen" --verify-out "$verify_log" >> "$test_log" 2>&1
+echo "$ct" fill --dsc 127.0.0.1:9998 --stable -g "$gen" --verify-out "$verify_log" >> "$test_log"
+"$ct" fill --dsc 127.0.0.1:9998 --stable -g "$gen" --verify-out "$verify_log" >> "$test_log" 2>&1
 if [[ $? -ne 0 ]]; then
     echo "Error in initial fill"
     ctrl_c
@@ -250,7 +245,7 @@ while [[ $count -le $loops ]]; do
     fi
 
     echo "$(date) do one IO" >> "$test_log"
-    "$ct" one "${args[@]}" \
+    "$ct" one --dsc 127.0.0.1:9998 \
             -q -g "$gen" --verify-out "$verify_log" \
             --verify-in "$verify_log" \
             --verify-at-start \

From 1a973fcba5e33a84dfb95e313c136e61ca2b3046 Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxide.computer>
Date: Mon, 3 Mar 2025 19:56:38 +0000
Subject: [PATCH 03/21] test_replay dsc logs to test specific directory

---
 tools/test_replay.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tools/test_replay.sh b/tools/test_replay.sh
index dbd19ca21..f24006181 100755
--- a/tools/test_replay.sh
+++ b/tools/test_replay.sh
@@ -31,6 +31,7 @@ fi
 
 test_log="$TEST_ROOT/test_replay.log"
 verify_log="$TEST_ROOT/test_replay_verify.log"
+dsc_ds_log="$TEST_ROOT/test_live_repair_dsc.log"
 
 ROOT=$(cd "$(dirname "$0")/.." && pwd)
 cd "$ROOT" || (echo failed to cd "$ROOT"; exit 1)
@@ -75,13 +76,15 @@ echo "Tail $test_log for test output"
 
 echo "Creating $region_count downstairs regions" | tee -a "$test_log"
 if ! ${dsc} create --cleanup --ds-bin "$downstairs" \
+        --output-dir "$dsc_ds_log" \
         --extent-count 50 --region-count "$region_count" >> "$test_log"; then
     echo "Failed to create downstairs regions"
     exit 1
 fi
 
 echo "Starting $region_count downstairs" | tee -a "$test_log"
-${dsc} start --ds-bin "$downstairs" --region-count "$region_count" >> "$test_log" 2>&1 &
+${dsc} start --ds-bin "$downstairs" --output-dir "$dsc_ds_log" \
+	--region-count "$region_count" >> "$test_log" 2>&1 &
 dsc_pid=$!
 sleep 5
 if ! ps -p $dsc_pid > /dev/null; then

From e19e8a2795983ee9649baec4dd991dcce72d4bce Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxide.computer>
Date: Tue, 4 Mar 2025 17:20:49 +0000
Subject: [PATCH 04/21] More polish

---
 tools/hammer_loop.sh         |  2 ++
 tools/test_live_repair.sh    |  4 ++--
 tools/test_nightly.sh        | 15 ++++++++++-----
 tools/test_repair.sh         |  6 +++++-
 tools/test_replay.sh         | 10 +++++++++-
 tools/test_restart_repair.sh | 31 ++++++++++++++++++++++---------
 6 files changed, 50 insertions(+), 18 deletions(-)

diff --git a/tools/hammer_loop.sh b/tools/hammer_loop.sh
index beefcf476..d7eb28b17 100755
--- a/tools/hammer_loop.sh
+++ b/tools/hammer_loop.sh
@@ -167,5 +167,7 @@ if [[ $err -eq 0 ]]; then
     rm -r "$REGION_ROOT"/8810
     rm -r "$REGION_ROOT"/8820
     rm -r "$REGION_ROOT"/8830
+    # If empty, remove the region directory
+    rmdir "$REGION_ROOT"
 fi
 exit "$err"
diff --git a/tools/test_live_repair.sh b/tools/test_live_repair.sh
index 7ab960d0e..ec1644443 100755
--- a/tools/test_live_repair.sh
+++ b/tools/test_live_repair.sh
@@ -163,12 +163,12 @@ wait "$dsc_pid"
 echo "$(date) Test ends with $result" | tee -a "$test_log"
 
 if [[ $result -eq 0 ]]; then
-    echo "DELETE 8840" | tee -a "$test_log"
-    ps -ef | grep crucible-downstairs
     rm -rf "$REGION_ROOT"/8810
     rm -rf "$REGION_ROOT"/8820
     rm -rf "$REGION_ROOT"/8830
     rm -rf "$REGION_ROOT"/8840
+    # If empty, remove the region directory
+    rmdir "$REGION_ROOT"
     rm -rf "$TEST_ROOT"
 fi
 
diff --git a/tools/test_nightly.sh b/tools/test_nightly.sh
index 329fffc83..0671b2d3c 100755
--- a/tools/test_nightly.sh
+++ b/tools/test_nightly.sh
@@ -21,7 +21,8 @@ echo "Nightly starts at $(date)" | tee "$output_file"
 echo "$(date) hammer start" >> "$output_file"
 banner hammer
 banner loop
-./tools/hammer_loop.sh -l 200
+#./tools/hammer_loop.sh -l 200
+./tools/hammer_loop.sh -l 10
 res=$?
 if [[ "$res" -eq 0 ]]; then
     echo "$(date) hammer pass" >> "$output_file"
@@ -37,7 +38,8 @@ echo ""
 banner test
 banner replay
 echo "$(date) test_replay start" >> "$output_file"
-./tools/test_replay.sh -l 200
+#./tools/test_replay.sh -l 200
+./tools/test_replay.sh -l 10
 res=$?
 if [[ "$res" -eq 0 ]]; then
     echo "$(date) test_replay pass" >> "$output_file"
@@ -70,7 +72,8 @@ echo ""
 banner restart
 banner repair
 echo "$(date) test_restart_repair start" >> "$output_file"
-./tools/test_restart_repair.sh -l 50
+#./tools/test_restart_repair.sh -l 50
+./tools/test_restart_repair.sh -l 5
 res=$?
 if [[ "$res" -eq 0 ]]; then
     echo "$(date) test_restart_repair pass" >> "$output_file"
@@ -87,7 +90,8 @@ echo ""
 banner live
 banner repair
 echo "$(date) test_live_repair start" >> "$output_file"
-./tools/test_live_repair.sh -l 20
+#./tools/test_live_repair.sh -l 20
+./tools/test_live_repair.sh -l 3
 res=$?
 if [[ "$res" -eq 0 ]]; then
     echo "$(date) test_live_repair pass" >> "$output_file"
@@ -104,7 +108,8 @@ echo ""
 banner replace
 banner special
 echo "$(date) test_replace_special start" >> "$output_file"
-./tools/test_replace_special.sh -l 30
+#./tools/test_replace_special.sh -l 30
+./tools/test_replace_special.sh -l 3
 res=$?
 if [[ "$res" -eq 0 ]]; then
     echo "$(date) test_replace_special pass" >> "$output_file"
diff --git a/tools/test_repair.sh b/tools/test_repair.sh
index 358ed8c1d..c8d455755 100755
--- a/tools/test_repair.sh
+++ b/tools/test_repair.sh
@@ -43,7 +43,9 @@ done
 # For buildomat, the regions should be in /var/tmp
 REGION_ROOT=${REGION_ROOT:-/var/tmp/test_repair}
 if [[ -d ${REGION_ROOT} ]]; then
-    rm -rf ${REGION_ROOT}
+    rm -rf "$REGION_ROOT"/8810
+    rm -rf "$REGION_ROOT"/8820
+    rm -rf "$REGION_ROOT"/8830
 fi
 
 # Location of logs and working files
@@ -246,3 +248,5 @@ rm -rf "$TEST_ROOT"
 rm -rf "$REGION_ROOT"/8810
 rm -rf "$REGION_ROOT"/8820
 rm -rf "$REGION_ROOT"/8830
+# If empty, remove the region directory
+rmdir "$REGION_ROOT"
diff --git a/tools/test_replay.sh b/tools/test_replay.sh
index f24006181..c4adba574 100755
--- a/tools/test_replay.sh
+++ b/tools/test_replay.sh
@@ -15,9 +15,10 @@ function ctrl_c() {
     exit 1
 }
 
+REGION_ROOT=${REGION_ROOT:-/var/tmp/test_replay}
+mkdir -p "$REGION_ROOT"
 WORK_ROOT=${WORK_ROOT:-/tmp}
 TEST_ROOT="$WORK_ROOT/test_replay"
-
 if [[ ! -d "$TEST_ROOT" ]]; then
     mkdir -p "$TEST_ROOT"
     if [[ $? -ne 0 ]]; then
@@ -76,6 +77,7 @@ echo "Tail $test_log for test output"
 
 echo "Creating $region_count downstairs regions" | tee -a "$test_log"
 if ! ${dsc} create --cleanup --ds-bin "$downstairs" \
+	--region-dir "$REGION_ROOT" \
         --output-dir "$dsc_ds_log" \
         --extent-count 50 --region-count "$region_count" >> "$test_log"; then
     echo "Failed to create downstairs regions"
@@ -84,6 +86,7 @@ fi
 
 echo "Starting $region_count downstairs" | tee -a "$test_log"
 ${dsc} start --ds-bin "$downstairs" --output-dir "$dsc_ds_log" \
+	--region-dir "$REGION_ROOT" \
 	--region-count "$region_count" >> "$test_log" 2>&1 &
 dsc_pid=$!
 sleep 5
@@ -137,6 +140,11 @@ wait "$dsc_pid"
 sleep 4
 echo "$(date) Test ends with $result" | tee -a "$test_log" 2>&1
 if [[ $result -eq 0 ]]; then
+    rm -rf "$REGION_ROOT"/8810
+    rm -rf "$REGION_ROOT"/8820
+    rm -rf "$REGION_ROOT"/8830
+    # If empty, remove the region directory
+    rmdir "$REGION_ROOT"
     rm -r "$TEST_ROOT"
 fi
 exit "$result"
diff --git a/tools/test_restart_repair.sh b/tools/test_restart_repair.sh
index 01cdd7cb1..c12e0ffa3 100755
--- a/tools/test_restart_repair.sh
+++ b/tools/test_restart_repair.sh
@@ -97,15 +97,18 @@ while getopts 'l:' opt; do
     esac
 done
 
-WORK_ROOT=${WORK_ROOT:-/tmp}
-TEST_ROOT="$WORK_ROOT/test_restart_repair"
 REGION_ROOT=${REGION_ROOT:-/var/tmp/test_restart_repair}
-export loop_log="$TEST_ROOT/test_restart_repair.log"
-export test_log="$TEST_ROOT/test_restart_repair_test.log"
-export verify_log="$TEST_ROOT/test_restart_repair_verify.log"
-export dsc_log="$TEST_ROOT/test_restart_repair_dsc.log"
-export dsc_ds_log="$TEST_ROOT/dsc"
+if [[ -d "$REGION_ROOT" ]]; then
+    rm -r "$REGION_ROOT"/8810
+    rm -r "$REGION_ROOT"/8810.old
+    rm -r "$REGION_ROOT"/8820
+    rm -r "$REGION_ROOT"/8820.old
+    rm -r "$REGION_ROOT"/8830
+    rm -r "$REGION_ROOT"/8830.old
+fi
 
+WORK_ROOT=${WORK_ROOT:-/tmp}
+TEST_ROOT="$WORK_ROOT/test_restart_repair"
 if [[ ! -d "$TEST_ROOT" ]]; then
     mkdir -p "$TEST_ROOT"
     if [[ $? -ne 0 ]]; then
@@ -117,6 +120,12 @@ else
     rm -r "$TEST_ROOT"
 fi
 
+export loop_log="$TEST_ROOT/test_restart_repair.log"
+export test_log="$TEST_ROOT/test_restart_repair_test.log"
+export verify_log="$TEST_ROOT/test_restart_repair_verify.log"
+export dsc_log="$TEST_ROOT/test_restart_repair_dsc.log"
+export dsc_ds_log="$TEST_ROOT/dsc"
+
 touch "$loop_log"
 echo "starting $(date)" | tee "$loop_log"
 echo "Tail $test_log for test output"
@@ -246,8 +255,7 @@ while [[ $count -le $loops ]]; do
 
     echo "$(date) do one IO" >> "$test_log"
     "$ct" one --dsc 127.0.0.1:9998 \
-            -q -g "$gen" --verify-out "$verify_log" \
-            --verify-in "$verify_log" \
+            -q -g "$gen" --verify-out "$verify_log" \ --verify-in "$verify_log" \
             --verify-at-start \
             --retry-activate >> "$test_log" 2>&1
     result=$?
@@ -288,7 +296,12 @@ if [[ $err -eq 0 ]]; then
     # No errors, then cleanup all our logs and the region directories.
     rm -r "$TEST_ROOT"
     rm -r "$REGION_ROOT"/8810
+    rm -r "$REGION_ROOT"/8810.old
     rm -r "$REGION_ROOT"/8820
+    rm -r "$REGION_ROOT"/8820.old
     rm -r "$REGION_ROOT"/8830
+    rm -r "$REGION_ROOT"/8830.old
+    # If the directory is empty, remove it.
+    rmdir "$REGION_ROOT" || true
 fi
 exit "$err"

From 8f4803c425abc7f25f6529c0084b39ad74ebd124 Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxide.computer>
Date: Tue, 4 Mar 2025 21:40:29 +0000
Subject: [PATCH 05/21] fix typo

---
 tools/test_restart_repair.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/test_restart_repair.sh b/tools/test_restart_repair.sh
index c12e0ffa3..66ae73c3e 100755
--- a/tools/test_restart_repair.sh
+++ b/tools/test_restart_repair.sh
@@ -255,7 +255,8 @@ while [[ $count -le $loops ]]; do
 
     echo "$(date) do one IO" >> "$test_log"
     "$ct" one --dsc 127.0.0.1:9998 \
-            -q -g "$gen" --verify-out "$verify_log" \ --verify-in "$verify_log" \
+            -q -g "$gen" --verify-out "$verify_log" \
+	    --verify-in "$verify_log" \
             --verify-at-start \
             --retry-activate >> "$test_log" 2>&1
     result=$?

From 0dacaf3c7b7a935c59b1044eb5f2e5c3e0d1ccbf Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxide.computer>
Date: Tue, 4 Mar 2025 22:22:41 +0000
Subject: [PATCH 06/21] More more polish

---
 tools/hammer_loop.sh          | 14 +++++++++-----
 tools/test_nightly.sh         |  7 ++++---
 tools/test_replace_special.sh |  5 +++++
 3 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/tools/hammer_loop.sh b/tools/hammer_loop.sh
index d7eb28b17..8ae7d132c 100755
--- a/tools/hammer_loop.sh
+++ b/tools/hammer_loop.sh
@@ -32,9 +32,6 @@ fi
 WORK_ROOT=${WORK_ROOT:-/tmp}
 TEST_ROOT="$WORK_ROOT/hammer_loop"
 REGION_ROOT=${REGION_ROOT:-/var/tmp/hammer_loop}
-loop_log="$TEST_ROOT/hammer_loop.log"
-test_log="$TEST_ROOT/hammer_loop_test.log"
-
 if [[ ! -d "$TEST_ROOT" ]]; then
     mkdir -p "$TEST_ROOT"
     if [[ $? -ne 0 ]]; then
@@ -46,6 +43,10 @@ else
     rm -r "$TEST_ROOT"
 fi
 
+loop_log="$TEST_ROOT/hammer_loop.log"
+test_log="$TEST_ROOT/hammer_loop_test.log"
+dsc_ds_log="$TEST_ROOT/test_live_repair_dsc.log"
+
 loops=20
 
 usage () {
@@ -65,6 +66,7 @@ while getopts 'l:' opt; do
 done
 
 if ! "$dsc" create --cleanup --ds-bin "$cds" --extent-count 60 \
+    --output-dir "$dsc_ds_log" \
     --extent-size 50 --region-dir "$REGION_ROOT"
 then
     echo "Failed to create region"
@@ -72,7 +74,8 @@ then
 fi
 
 # Start up dsc, verify it really did start.
-"$dsc" start --ds-bin "$cds" --region-dir "$REGION_ROOT" &
+"$dsc" start --ds-bin "$cds" --region-dir "$REGION_ROOT" 
+    --output-dir "$dsc_ds_log" &
 dsc_pid=$!
 sleep 5
 if ! pgrep -P $dsc_pid; then
@@ -154,8 +157,9 @@ printf "[%03d] %d:%02d  ave:%d:%02d  total:%d:%02d errors:%d last_run_seconds:%d
   "$err" $duration | tee -a ${loop_log}
 
 echo "Stopping dsc"
-kill $dsc_pid 2> /dev/null
+"$dsc" cmd shutdown
 wait $dsc_pid
+
 # Also remove any leftover downstairs
 if pgrep -fl -U "$(id -u)" "$cds" > /dev/null; then
     pkill -f -U "$(id -u)" "$cds"
diff --git a/tools/test_nightly.sh b/tools/test_nightly.sh
index 0671b2d3c..d3e5bf4a2 100755
--- a/tools/test_nightly.sh
+++ b/tools/test_nightly.sh
@@ -39,7 +39,7 @@ banner test
 banner replay
 echo "$(date) test_replay start" >> "$output_file"
 #./tools/test_replay.sh -l 200
-./tools/test_replay.sh -l 10
+./tools/test_replay.sh -l 3
 res=$?
 if [[ "$res" -eq 0 ]]; then
     echo "$(date) test_replay pass" >> "$output_file"
@@ -55,7 +55,8 @@ echo ""
 banner "test"
 banner repair
 echo "$(date) test_repair start" >> "$output_file"
-./tools/test_repair.sh -l 500
+# ./tools/test_repair.sh -l 500
+./tools/test_repair.sh -l 5
 res=$?
 if [[ "$res" -eq 0 ]]; then
     echo "$(date) test_repair pass" >> "$output_file"
@@ -73,7 +74,7 @@ banner restart
 banner repair
 echo "$(date) test_restart_repair start" >> "$output_file"
 #./tools/test_restart_repair.sh -l 50
-./tools/test_restart_repair.sh -l 5
+./tools/test_restart_repair.sh -l 3
 res=$?
 if [[ "$res" -eq 0 ]]; then
     echo "$(date) test_restart_repair pass" >> "$output_file"
diff --git a/tools/test_replace_special.sh b/tools/test_replace_special.sh
index ecac3afcb..9b41090f4 100755
--- a/tools/test_replace_special.sh
+++ b/tools/test_replace_special.sh
@@ -146,6 +146,11 @@ echo "$(date) Test ends with $result" | tee -a "$test_log"
 if [[ $result -eq 0 ]]; then
     # Cleanup
     echo "$(date) Cleanup for $0" | tee -a "$test_log"
+    rm -rf "$REGION_ROOT"/8810
+    rm -rf "$REGION_ROOT"/8820
+    rm -rf "$REGION_ROOT"/8830
+    rm -rf "$REGION_ROOT"/8840
+    rmdir "$REGION_ROOT"
     rm -rf "$TEST_ROOT"
 fi
 exit $result

From 3d629f0120d54d0465a98e0f4a46dcd413cc19b3 Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxide.computer>
Date: Wed, 5 Mar 2025 03:21:06 +0000
Subject: [PATCH 07/21] bad line break

---
 tools/hammer_loop.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/hammer_loop.sh b/tools/hammer_loop.sh
index 8ae7d132c..8eb104ac2 100755
--- a/tools/hammer_loop.sh
+++ b/tools/hammer_loop.sh
@@ -74,7 +74,7 @@ then
 fi
 
 # Start up dsc, verify it really did start.
-"$dsc" start --ds-bin "$cds" --region-dir "$REGION_ROOT" 
+"$dsc" start --ds-bin "$cds" --region-dir "$REGION_ROOT" \
     --output-dir "$dsc_ds_log" &
 dsc_pid=$!
 sleep 5

From 975734655a6a9aa82194cb5ece6074ae5e088b4c Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxide.computer>
Date: Wed, 5 Mar 2025 18:30:33 +0000
Subject: [PATCH 08/21] the polish continues

---
 tools/test_repair.sh | 2 +-
 tools/test_replay.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/test_repair.sh b/tools/test_repair.sh
index c8d455755..faaa580e8 100755
--- a/tools/test_repair.sh
+++ b/tools/test_repair.sh
@@ -65,7 +65,7 @@ fi
 verify_file="$TEST_ROOT/test_repair_verify.data"
 test_log="$TEST_ROOT/test_repair_out.txt"
 ds_log_prefix="$TEST_ROOT/test_repair_ds"
-dsc_output_dir="$TEST_ROOT/dsc"
+dsc_output_dir="$TEST_ROOT/test_repair_dsc"
 loops=100
 
 usage () {
diff --git a/tools/test_replay.sh b/tools/test_replay.sh
index c4adba574..82ad25f21 100755
--- a/tools/test_replay.sh
+++ b/tools/test_replay.sh
@@ -32,7 +32,7 @@ fi
 
 test_log="$TEST_ROOT/test_replay.log"
 verify_log="$TEST_ROOT/test_replay_verify.log"
-dsc_ds_log="$TEST_ROOT/test_live_repair_dsc.log"
+dsc_ds_log="$TEST_ROOT/test_replay_dsc.log"
 
 ROOT=$(cd "$(dirname "$0")/.." && pwd)
 cd "$ROOT" || (echo failed to cd "$ROOT"; exit 1)

From 06782ac89a2e0f4894788d2a8fd0301419f94228 Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxide.computer>
Date: Wed, 5 Mar 2025 18:32:13 +0000
Subject: [PATCH 09/21] restore test loop counts

---
 tools/test_nightly.sh | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/tools/test_nightly.sh b/tools/test_nightly.sh
index d3e5bf4a2..62b08cd5b 100755
--- a/tools/test_nightly.sh
+++ b/tools/test_nightly.sh
@@ -21,8 +21,7 @@ echo "Nightly starts at $(date)" | tee "$output_file"
 echo "$(date) hammer start" >> "$output_file"
 banner hammer
 banner loop
-#./tools/hammer_loop.sh -l 200
-./tools/hammer_loop.sh -l 10
+./tools/hammer_loop.sh -l 200
 res=$?
 if [[ "$res" -eq 0 ]]; then
     echo "$(date) hammer pass" >> "$output_file"
@@ -35,11 +34,11 @@ echo "$(date) Next test"
 ps -ef | egrep "downstairs|dsc"
 echo ""
 
+sleep 1
 banner test
 banner replay
 echo "$(date) test_replay start" >> "$output_file"
-#./tools/test_replay.sh -l 200
-./tools/test_replay.sh -l 3
+./tools/test_replay.sh -l 200
 res=$?
 if [[ "$res" -eq 0 ]]; then
     echo "$(date) test_replay pass" >> "$output_file"
@@ -52,11 +51,11 @@ echo "$(date) Next test"
 ps -ef | egrep "downstairs|dsc"
 echo ""
 
+sleep 1
 banner "test"
 banner repair
 echo "$(date) test_repair start" >> "$output_file"
-# ./tools/test_repair.sh -l 500
-./tools/test_repair.sh -l 5
+./tools/test_repair.sh -l 500
 res=$?
 if [[ "$res" -eq 0 ]]; then
     echo "$(date) test_repair pass" >> "$output_file"
@@ -70,11 +69,11 @@ echo "$(date) Next test"
 ps -ef | egrep "downstairs|dsc"
 echo ""
 
+sleep 1
 banner restart
 banner repair
 echo "$(date) test_restart_repair start" >> "$output_file"
-#./tools/test_restart_repair.sh -l 50
-./tools/test_restart_repair.sh -l 3
+./tools/test_restart_repair.sh -l 50
 res=$?
 if [[ "$res" -eq 0 ]]; then
     echo "$(date) test_restart_repair pass" >> "$output_file"
@@ -88,11 +87,11 @@ echo "$(date) Next test"
 ps -ef | egrep "downstairs|dsc"
 echo ""
 
+sleep 1
 banner live
 banner repair
 echo "$(date) test_live_repair start" >> "$output_file"
-#./tools/test_live_repair.sh -l 20
-./tools/test_live_repair.sh -l 3
+./tools/test_live_repair.sh -l 20
 res=$?
 if [[ "$res" -eq 0 ]]; then
     echo "$(date) test_live_repair pass" >> "$output_file"
@@ -106,11 +105,11 @@ echo "$(date) Next test"
 ps -ef | egrep "downstairs|dsc"
 echo ""
 
+sleep 1
 banner replace
 banner special
 echo "$(date) test_replace_special start" >> "$output_file"
-#./tools/test_replace_special.sh -l 30
-./tools/test_replace_special.sh -l 3
+./tools/test_replace_special.sh -l 30
 res=$?
 if [[ "$res" -eq 0 ]]; then
     echo "$(date) test_replace_special pass" >> "$output_file"

From a4f22ac9c542d3a41dede0df2f87d6560e4bfc72 Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxide.computer>
Date: Thu, 6 Mar 2025 01:25:13 +0000
Subject: [PATCH 10/21] Final...ish polish

---
 tools/test_nightly.sh | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/tools/test_nightly.sh b/tools/test_nightly.sh
index 62b08cd5b..d10324006 100755
--- a/tools/test_nightly.sh
+++ b/tools/test_nightly.sh
@@ -29,9 +29,6 @@ else
     echo "$(date) hammer fail with: $res" >> "$output_file"
     (( err += 1 ))
 fi
-
-echo "$(date) Next test"
-ps -ef | egrep "downstairs|dsc"
 echo ""
 
 sleep 1
@@ -46,9 +43,6 @@ else
     echo "$(date) test_replay fail with: $res" >> "$output_file"
     (( err += 1 ))
 fi
-
-echo "$(date) Next test"
-ps -ef | egrep "downstairs|dsc"
 echo ""
 
 sleep 1
@@ -64,9 +58,6 @@ else
     (( err += 1 ))
     exit 1
 fi
-
-echo "$(date) Next test"
-ps -ef | egrep "downstairs|dsc"
 echo ""
 
 sleep 1
@@ -82,9 +73,6 @@ else
     (( err += 1 ))
     exit 1
 fi
-
-echo "$(date) Next test"
-ps -ef | egrep "downstairs|dsc"
 echo ""
 
 sleep 1
@@ -100,9 +88,6 @@ else
     (( err += 1 ))
     exit 1
 fi
-
-echo "$(date) Next test"
-ps -ef | egrep "downstairs|dsc"
 echo ""
 
 sleep 1

From ba90e1c898ba939eff59999221654385d8992f46 Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxide.computer>
Date: Thu, 6 Mar 2025 01:51:08 +0000
Subject: [PATCH 11/21] update cargo lock

---
 Cargo.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index c72e84f14..68645c4f0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1113,7 +1113,7 @@ dependencies = [
  "slog-term",
  "statistical",
  "tempfile",
- "thiserror 1.0.66",
+ "thiserror 1.0.69",
  "tokio",
  "tokio-rustls 0.24.1",
  "tokio-util",

From e4f3f020b2e6134ac106647d2ab3d38816b7c1c9 Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxide.computer>
Date: Thu, 6 Mar 2025 02:02:32 +0000
Subject: [PATCH 12/21] remove test REGION_ROOT

---
 tools/test_nightly.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tools/test_nightly.sh b/tools/test_nightly.sh
index d10324006..4dba11370 100755
--- a/tools/test_nightly.sh
+++ b/tools/test_nightly.sh
@@ -10,9 +10,6 @@ err=0
 output_file="/tmp/nightly_results"
 rm -f "$output_file"
 
-REGION_ROOT=/regions/ubuntu/nightly
-echo "using REGION_ROOT=$REGION_ROOT"
-
 ROOT=$(cd "$(dirname "$0")/.." && pwd)
 cd "$ROOT" || (echo failed to cd "$ROOT"; exit 1)
 export BINDIR=${BINDIR:-$ROOT/target/release}

From fa059a152275dcb629713f2726ae0d9628d0b18c Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxide.computer>
Date: Thu, 6 Mar 2025 02:08:59 +0000
Subject: [PATCH 13/21] ppppppolish

---
 tools/hammer_loop.sh | 2 +-
 tools/test_repair.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/hammer_loop.sh b/tools/hammer_loop.sh
index 8eb104ac2..bfc3111c2 100755
--- a/tools/hammer_loop.sh
+++ b/tools/hammer_loop.sh
@@ -45,7 +45,7 @@ fi
 
 loop_log="$TEST_ROOT/hammer_loop.log"
 test_log="$TEST_ROOT/hammer_loop_test.log"
-dsc_ds_log="$TEST_ROOT/test_live_repair_dsc.log"
+dsc_ds_log="$TEST_ROOT/hammer_loop_dsc.log"
 
 loops=20
 
diff --git a/tools/test_repair.sh b/tools/test_repair.sh
index faaa580e8..5c33d2627 100755
--- a/tools/test_repair.sh
+++ b/tools/test_repair.sh
@@ -118,7 +118,7 @@ ds1_pid=$!
 ${cds} run -d "${REGION_ROOT}/8830" -p 8830 &> "$ds_log_prefix"8830.txt &
 ds2_pid=$!
 
-# Some programatic way to wait for all the downstairs to start before we
+# TODO: Some programatic way to wait for all the downstairs to start before we
 # continue here.
 sleep 20
 

From a71330006969b4d17987bad8465c41dae08e2f5a Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxide.computer>
Date: Mon, 10 Mar 2025 16:12:31 +0000
Subject: [PATCH 14/21] Add git info to nightly summary

---
 tools/test_nightly.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/test_nightly.sh b/tools/test_nightly.sh
index 4dba11370..ef1c21162 100755
--- a/tools/test_nightly.sh
+++ b/tools/test_nightly.sh
@@ -15,6 +15,7 @@ cd "$ROOT" || (echo failed to cd "$ROOT"; exit 1)
 export BINDIR=${BINDIR:-$ROOT/target/release}
 
 echo "Nightly starts at $(date)" | tee "$output_file"
+echo "Running on $(git log -1 | head -20)" | tee -a "$output_file"
 echo "$(date) hammer start" >> "$output_file"
 banner hammer
 banner loop

From 154ab2ac4a21a4cd57fe28a6f5938db7fbaa96c6 Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxidecomputer.com>
Date: Mon, 10 Mar 2025 17:57:21 +0000
Subject: [PATCH 15/21] Better REGION_DIR behavior

---
 tools/hammer_loop.sh          | 22 ++++++++-----
 tools/test_live_repair.sh     | 24 ++++++++------
 tools/test_repair.sh          | 47 ++++++++++++++--------------
 tools/test_replace_special.sh | 23 ++++++++------
 tools/test_replay.sh          | 24 ++++++++------
 tools/test_restart_repair.sh  | 59 ++++++++++++++++-------------------
 6 files changed, 108 insertions(+), 91 deletions(-)

diff --git a/tools/hammer_loop.sh b/tools/hammer_loop.sh
index bfc3111c2..08b92e457 100755
--- a/tools/hammer_loop.sh
+++ b/tools/hammer_loop.sh
@@ -31,7 +31,6 @@ fi
 
 WORK_ROOT=${WORK_ROOT:-/tmp}
 TEST_ROOT="$WORK_ROOT/hammer_loop"
-REGION_ROOT=${REGION_ROOT:-/var/tmp/hammer_loop}
 if [[ ! -d "$TEST_ROOT" ]]; then
     mkdir -p "$TEST_ROOT"
     if [[ $? -ne 0 ]]; then
@@ -42,6 +41,17 @@ else
     # Delete previous test data
     rm -r "$TEST_ROOT"
 fi
+REGION_ROOT=${REGION_ROOT:-/var/tmp}
+MY_REGION_ROOT=${REGION_ROOT/hammer_loop}
+if [[ ! -d "$MY_REGION_ROOT" ]]; then
+    mkdir -p "$MY_REGION_ROOT"
+    if [[ $? -ne 0 ]]; then
+        echo "Failed to make region root $MY_REGION_ROOT"
+        exit 1
+    fi
+else
+    rm -rf "$MY_REGION_ROOT"
+fi
 
 loop_log="$TEST_ROOT/hammer_loop.log"
 test_log="$TEST_ROOT/hammer_loop_test.log"
@@ -67,14 +77,14 @@ done
 
 if ! "$dsc" create --cleanup --ds-bin "$cds" --extent-count 60 \
     --output-dir "$dsc_ds_log" \
-    --extent-size 50 --region-dir "$REGION_ROOT"
+    --extent-size 50 --region-dir "$MY_REGION_ROOT"
 then
     echo "Failed to create region"
     exit 1
 fi
 
 # Start up dsc, verify it really did start.
-"$dsc" start --ds-bin "$cds" --region-dir "$REGION_ROOT" \
+"$dsc" start --ds-bin "$cds" --region-dir "$MY_REGION_ROOT" \
     --output-dir "$dsc_ds_log" &
 dsc_pid=$!
 sleep 5
@@ -168,10 +178,6 @@ fi
 if [[ $err -eq 0 ]]; then
     # No errors, then cleanup all our logs and the region directories.
     rm -r "$TEST_ROOT"
-    rm -r "$REGION_ROOT"/8810
-    rm -r "$REGION_ROOT"/8820
-    rm -r "$REGION_ROOT"/8830
-    # If empty, remove the region directory
-    rmdir "$REGION_ROOT"
+    rm -rf "$MY_REGION_ROOT"
 fi
 exit "$err"
diff --git a/tools/test_live_repair.sh b/tools/test_live_repair.sh
index ec1644443..d308473ed 100755
--- a/tools/test_live_repair.sh
+++ b/tools/test_live_repair.sh
@@ -21,8 +21,17 @@ function ctrl_c() {
     exit 1
 }
 
-REGION_ROOT=${REGION_ROOT:-/var/tmp/test_live_repair}
-mkdir -p "$REGION_ROOT"
+REGION_ROOT=${REGION_ROOT:-/var/tmp}
+MY_REGION_ROOT=${REGION_ROOT/test_live_repair}
+if [[ ! -d "$MY_REGION_ROOT" ]]; then
+    mkdir -p "$MY_REGION_ROOT"
+    if [[ $? -ne 0 ]]; then
+        echo "Failed to make region root $MY_REGION_ROOT"
+        exit 1
+    fi
+else
+    rm -rf "$MY_REGION_ROOT"
+fi
 
 # Location of logs and working files
 WORK_ROOT=${WORK_ROOT:-/tmp}
@@ -99,7 +108,7 @@ fi
 # be used by the replace test.  We can use dsc to determine what the port will
 # be for the final region.
 if ! ${dsc} create --cleanup \
-  --region-dir "$REGION_ROOT" \
+  --region-dir "$MY_REGION_ROOT" \
   --region-count "$region_count" \
   --output-dir "$dsc_ds_log" \
   --ds-bin "$downstairs" \
@@ -110,7 +119,7 @@ then
     exit 1
 fi
 ${dsc} start --ds-bin "$downstairs" \
-  --region-dir "$REGION_ROOT" \
+  --region-dir "$MY_REGION_ROOT" \
   --output-dir "$dsc_ds_log" \
   --region-count "$region_count" >> "$test_log" 2>&1 &
 dsc_pid=$!
@@ -163,12 +172,7 @@ wait "$dsc_pid"
 echo "$(date) Test ends with $result" | tee -a "$test_log"
 
 if [[ $result -eq 0 ]]; then
-    rm -rf "$REGION_ROOT"/8810
-    rm -rf "$REGION_ROOT"/8820
-    rm -rf "$REGION_ROOT"/8830
-    rm -rf "$REGION_ROOT"/8840
-    # If empty, remove the region directory
-    rmdir "$REGION_ROOT"
+    rm -rf "$MY_REGION_ROOT"
     rm -rf "$TEST_ROOT"
 fi
 
diff --git a/tools/test_repair.sh b/tools/test_repair.sh
index 5c33d2627..396f65c32 100755
--- a/tools/test_repair.sh
+++ b/tools/test_repair.sh
@@ -41,11 +41,16 @@ done
 
 
 # For buildomat, the regions should be in /var/tmp
-REGION_ROOT=${REGION_ROOT:-/var/tmp/test_repair}
-if [[ -d ${REGION_ROOT} ]]; then
-    rm -rf "$REGION_ROOT"/8810
-    rm -rf "$REGION_ROOT"/8820
-    rm -rf "$REGION_ROOT"/8830
+REGION_ROOT=${REGION_ROOT:-/var/tmp}
+MY_REGION_ROOT=${REGION_ROOT/test_repair}
+if [[ ! -d "$MY_REGION_ROOT" ]]; then
+    mkdir -p "$MY_REGION_ROOT"
+    if [[ $? -ne 0 ]]; then
+        echo "Failed to make region root $MY_REGION_ROOT"
+        exit 1
+    fi
+else
+    rm -rf "$MY_REGION_ROOT"
 fi
 
 # Location of logs and working files
@@ -90,7 +95,7 @@ while getopts 'l:N' opt; do
 done
 
 if ! "$dsc" create --cleanup --ds-bin "$cds" --extent-count 30 \
-        --extent-size 20 --region-dir "$REGION_ROOT" \
+        --extent-size 20 --region-dir "$MY_REGION_ROOT" \
         --output-dir "$dsc_output_dir"; then
     echo "Failed to create region"
     exit 1
@@ -100,9 +105,9 @@ fi
 # are the same as what DSC uses by default.  If either side changes, then
 # the other will need to be update manually.
 target_args="-t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830"
-dump_args+=("-d ${REGION_ROOT}/8810")
-dump_args+=("-d ${REGION_ROOT}/8820")
-dump_args+=("-d ${REGION_ROOT}/8830")
+dump_args+=("-d ${MY_REGION_ROOT}/8810")
+dump_args+=("-d ${MY_REGION_ROOT}/8820")
+dump_args+=("-d ${MY_REGION_ROOT}/8830")
 
 if pgrep -fl -U "$(id -u)" "$cds"; then
     echo "Downstairs already running" >&2
@@ -111,11 +116,11 @@ if pgrep -fl -U "$(id -u)" "$cds"; then
 fi
 
 # Start all three downstairs
-${cds} run -d "${REGION_ROOT}/8810" -p 8810 &> "$ds_log_prefix"8810.txt &
+${cds} run -d "${MY_REGION_ROOT}/8810" -p 8810 &> "$ds_log_prefix"8810.txt &
 ds0_pid=$!
-${cds} run -d "${REGION_ROOT}/8820" -p 8820 &> "$ds_log_prefix"8820.txt &
+${cds} run -d "${MY_REGION_ROOT}/8820" -p 8820 &> "$ds_log_prefix"8820.txt &
 ds1_pid=$!
-${cds} run -d "${REGION_ROOT}/8830" -p 8830 &> "$ds_log_prefix"8830.txt &
+${cds} run -d "${MY_REGION_ROOT}/8830" -p 8830 &> "$ds_log_prefix"8830.txt &
 ds2_pid=$!
 
 # TODO: Some programatic way to wait for all the downstairs to start before we
@@ -154,17 +159,17 @@ while [[ $count -lt $loops ]]; do
     if [[ $choice -eq 0 ]]; then
         kill "$ds0_pid"
         wait "$ds0_pid" || true
-        ${cds} run -d "${REGION_ROOT}/8810" -p 8810 --lossy &> "$ds_log_prefix"8810.txt &
+        ${cds} run -d "${MY_REGION_ROOT}/8810" -p 8810 --lossy &> "$ds_log_prefix"8810.txt &
         ds0_pid=$!
     elif [[ $choice -eq 1 ]]; then
         kill "$ds1_pid"
         wait "$ds1_pid" || true
-        ${cds} run -d "${REGION_ROOT}/8820" -p 8820 --lossy &> "$ds_log_prefix"8820.txt &
+        ${cds} run -d "${MY_REGION_ROOT}/8820" -p 8820 --lossy &> "$ds_log_prefix"8820.txt &
         ds1_pid=$!
     else
         kill "$ds2_pid"
         wait "$ds2_pid" || true
-        ${cds} run -d "${REGION_ROOT}/8830" -p 8830 --lossy &> "$ds_log_prefix"8830.txt &
+        ${cds} run -d "${MY_REGION_ROOT}/8830" -p 8830 --lossy &> "$ds_log_prefix"8830.txt &
         ds2_pid=$!
     fi
 
@@ -200,13 +205,13 @@ while [[ $count -lt $loops ]]; do
     echo ""
     # Start downstairs without lossy
     if [[ $choice -eq 0 ]]; then
-        ${cds} run -d "${REGION_ROOT}/8810" -p 8810 &> "$ds_log_prefix"8810.txt &
+        ${cds} run -d "${MY_REGION_ROOT}/8810" -p 8810 &> "$ds_log_prefix"8810.txt &
         ds0_pid=$!
     elif [[ $choice -eq 1 ]]; then
-        ${cds} run -d "${REGION_ROOT}/8820" -p 8820 &> "$ds_log_prefix"8820.txt &
+        ${cds} run -d "${MY_REGION_ROOT}/8820" -p 8820 &> "$ds_log_prefix"8820.txt &
         ds1_pid=$!
     else
-        ${cds} run -d "${REGION_ROOT}/8830" -p 8830 &> "$ds_log_prefix"8830.txt &
+        ${cds} run -d "${MY_REGION_ROOT}/8830" -p 8830 &> "$ds_log_prefix"8830.txt &
         ds2_pid=$!
     fi
 
@@ -245,8 +250,4 @@ cleanup
 
 # Errors exit directly, so arrival here indicates success.
 rm -rf "$TEST_ROOT"
-rm -rf "$REGION_ROOT"/8810
-rm -rf "$REGION_ROOT"/8820
-rm -rf "$REGION_ROOT"/8830
-# If empty, remove the region directory
-rmdir "$REGION_ROOT"
+rm -rf "$MY_REGION_ROOT"
diff --git a/tools/test_replace_special.sh b/tools/test_replace_special.sh
index 9b41090f4..188d97c5f 100755
--- a/tools/test_replace_special.sh
+++ b/tools/test_replace_special.sh
@@ -17,8 +17,17 @@ function ctrl_c() {
     exit 1
 }
 
-REGION_ROOT=${REGION_ROOT:-/var/tmp/test_replace_special}
-mkdir -p "$REGION_ROOT"
+REGION_ROOT=${REGION_ROOT:-/var/tmp}
+MY_REGION_ROOT=${REGION_ROOT/test_replace_special}
+if [[ ! -d "$MY_REGION_ROOT" ]]; then
+    mkdir -p "$MY_REGION_ROOT"
+    if [[ $? -ne 0 ]]; then
+        echo "Failed to make region root $MY_REGION_ROOT"
+        exit 1
+    fi
+else
+    rm -rf "$MY_REGION_ROOT"
+fi
 
 # Location of logs and working files
 WORK_ROOT=${WORK_ROOT:-/tmp}
@@ -84,7 +93,7 @@ echo "Tail $test_log for test output"
 # to be used for replacement.  We can use dsc to determine what the port will
 # be for the final region
 if ! ${dsc} create --cleanup \
-  --region-dir "$REGION_ROOT" \
+  --region-dir "$MY_REGION_ROOT" \
   --region-count "$region_count" \
   --output-dir "$dsc_ds_log" \
   --ds-bin "$downstairs" \
@@ -94,7 +103,7 @@ if ! ${dsc} create --cleanup \
     exit 1
 fi
 ${dsc} start --ds-bin "$downstairs" \
-  --region-dir "$REGION_ROOT" \
+  --region-dir "$MY_REGION_ROOT" \
   --output-dir "$dsc_ds_log" \
   --region-count "$region_count" >> "$test_log" 2>&1 &
 dsc_pid=$!
@@ -146,11 +155,7 @@ echo "$(date) Test ends with $result" | tee -a "$test_log"
 if [[ $result -eq 0 ]]; then
     # Cleanup
     echo "$(date) Cleanup for $0" | tee -a "$test_log"
-    rm -rf "$REGION_ROOT"/8810
-    rm -rf "$REGION_ROOT"/8820
-    rm -rf "$REGION_ROOT"/8830
-    rm -rf "$REGION_ROOT"/8840
-    rmdir "$REGION_ROOT"
+    rm -rf "$MY_REGION_ROOT"
     rm -rf "$TEST_ROOT"
 fi
 exit $result
diff --git a/tools/test_replay.sh b/tools/test_replay.sh
index 82ad25f21..bd4423122 100755
--- a/tools/test_replay.sh
+++ b/tools/test_replay.sh
@@ -15,8 +15,18 @@ function ctrl_c() {
     exit 1
 }
 
-REGION_ROOT=${REGION_ROOT:-/var/tmp/test_replay}
-mkdir -p "$REGION_ROOT"
+REGION_ROOT=${REGION_ROOT:-/var/tmp}
+MY_REGION_ROOT=${REGION_ROOT/test_replay}
+if [[ ! -d "$MY_REGION_ROOT" ]]; then
+    mkdir -p "$MY_REGION_ROOT"
+    if [[ $? -ne 0 ]]; then
+        echo "Failed to make region root $MY_REGION_ROOT"
+        exit 1
+    fi
+else
+    rm -rf "$MY_REGION_ROOT"
+fi
+
 WORK_ROOT=${WORK_ROOT:-/tmp}
 TEST_ROOT="$WORK_ROOT/test_replay"
 if [[ ! -d "$TEST_ROOT" ]]; then
@@ -77,7 +87,7 @@ echo "Tail $test_log for test output"
 
 echo "Creating $region_count downstairs regions" | tee -a "$test_log"
 if ! ${dsc} create --cleanup --ds-bin "$downstairs" \
-	--region-dir "$REGION_ROOT" \
+	--region-dir "$MY_REGION_ROOT" \
         --output-dir "$dsc_ds_log" \
         --extent-count 50 --region-count "$region_count" >> "$test_log"; then
     echo "Failed to create downstairs regions"
@@ -86,7 +96,7 @@ fi
 
 echo "Starting $region_count downstairs" | tee -a "$test_log"
 ${dsc} start --ds-bin "$downstairs" --output-dir "$dsc_ds_log" \
-	--region-dir "$REGION_ROOT" \
+	--region-dir "$MY_REGION_ROOT" \
 	--region-count "$region_count" >> "$test_log" 2>&1 &
 dsc_pid=$!
 sleep 5
@@ -140,11 +150,7 @@ wait "$dsc_pid"
 sleep 4
 echo "$(date) Test ends with $result" | tee -a "$test_log" 2>&1
 if [[ $result -eq 0 ]]; then
-    rm -rf "$REGION_ROOT"/8810
-    rm -rf "$REGION_ROOT"/8820
-    rm -rf "$REGION_ROOT"/8830
-    # If empty, remove the region directory
-    rmdir "$REGION_ROOT"
+    rm -rf "$MY_REGION_ROOT"
     rm -r "$TEST_ROOT"
 fi
 exit "$result"
diff --git a/tools/test_restart_repair.sh b/tools/test_restart_repair.sh
index 66ae73c3e..3b89061c5 100755
--- a/tools/test_restart_repair.sh
+++ b/tools/test_restart_repair.sh
@@ -97,14 +97,16 @@ while getopts 'l:' opt; do
     esac
 done
 
-REGION_ROOT=${REGION_ROOT:-/var/tmp/test_restart_repair}
-if [[ -d "$REGION_ROOT" ]]; then
-    rm -r "$REGION_ROOT"/8810
-    rm -r "$REGION_ROOT"/8810.old
-    rm -r "$REGION_ROOT"/8820
-    rm -r "$REGION_ROOT"/8820.old
-    rm -r "$REGION_ROOT"/8830
-    rm -r "$REGION_ROOT"/8830.old
+REGION_ROOT=${REGION_ROOT:-/var/tmp}
+MY_REGION_ROOT=${REGION_ROOT/test_restart_repair}
+if [[ ! -d "$MY_REGION_ROOT" ]]; then
+    mkdir -p "$MY_REGION_ROOT"
+    if [[ $? -ne 0 ]]; then
+        echo "Failed to make region root $MY_REGION_ROOT"
+        exit 1
+    fi
+else
+    rm -rf "$MY_REGION_ROOT"
 fi
 
 WORK_ROOT=${WORK_ROOT:-/tmp}
@@ -136,14 +138,14 @@ echo "Create a new region to test" | tee -a "${loop_log}"
 ulimit -n 65536
 if ! "$dsc" create --cleanup --ds-bin "$cds" --extent-count 61 \
     --extent-size 5120 --output-dir "$dsc_ds_log" \
-    --region-dir "$REGION_ROOT"
+    --region-dir "$MY_REGION_ROOT"
 then
-    echo "Failed to create region at $REGION_ROOT"
+    echo "Failed to create region at $MY_REGION_ROOT"
     exit 1
 fi
 
 echo "Starting the downstairs" | tee -a "${loop_log}"
-"$dsc" start --ds-bin "$cds" --region-dir "$REGION_ROOT" \
+"$dsc" start --ds-bin "$cds" --region-dir "$MY_REGION_ROOT" \
     --output-dir "$dsc_ds_log" >> "$dsc_log" 2>&1 &
 dsc_pid=$!
 # Sleep 5 to give the downstairs time to get going.
@@ -186,10 +188,10 @@ stop_all_downstairs
 sleep 7
 
 # Create the "old" region files
-rm -rf "$REGION_ROOT"/8810.old "$REGION_ROOT"/8820.old "$REGION_ROOT"/8830.old
-cp -R  "$REGION_ROOT"/8810 "$REGION_ROOT"/8810.old || ctrl_c
-cp -R  "$REGION_ROOT"/8820 "$REGION_ROOT"/8820.old || ctrl_c
-cp -R  "$REGION_ROOT"/8830 "$REGION_ROOT"/8830.old || ctrl_c
+rm -rf "$MY_REGION_ROOT"/8810.old "$MY_REGION_ROOT"/8820.old "$MY_REGION_ROOT"/8830.old
+cp -R  "$MY_REGION_ROOT"/8810 "$MY_REGION_ROOT"/8810.old || ctrl_c
+cp -R  "$MY_REGION_ROOT"/8820 "$MY_REGION_ROOT"/8820.old || ctrl_c
+cp -R  "$MY_REGION_ROOT"/8830 "$MY_REGION_ROOT"/8830.old || ctrl_c
 
 # Bring the downstairs back online.
 echo "$(date) Bring downstairs back online" | tee -a "$loop_log"
@@ -230,19 +232,19 @@ while [[ $count -le $loops ]]; do
     echo "$(date) move regions" >> "$test_log"
     choice=$((RANDOM % 3))
     if [[ $choice -eq 0 ]]; then
-        rm -rf "$REGION_ROOT"/8810
-        cp -R  "$REGION_ROOT"/8810.old "$REGION_ROOT"/8810
+        rm -rf "$MY_REGION_ROOT"/8810
+        cp -R  "$MY_REGION_ROOT"/8810.old "$MY_REGION_ROOT"/8810
     elif [[ $choice -eq 1 ]]; then
-        rm -rf "$REGION_ROOT"/8820
-        cp -R  "$REGION_ROOT"/8820.old "$REGION_ROOT"/8820
+        rm -rf "$MY_REGION_ROOT"/8820
+        cp -R  "$MY_REGION_ROOT"/8820.old "$MY_REGION_ROOT"/8820
     else
-        rm -rf "$REGION_ROOT"/8830
-        cp -R  "$REGION_ROOT"/8830.old "$REGION_ROOT"/8830
+        rm -rf "$MY_REGION_ROOT"/8830
+        cp -R  "$MY_REGION_ROOT"/8830.old "$MY_REGION_ROOT"/8830
     fi
     echo "$(date) regions moved, current dump outputs:" >> "$test_log"
-    $cds dump --no-color -d "$REGION_ROOT"/8810 \
-        -d "$REGION_ROOT"/8820 \
-        -d "$REGION_ROOT"/8830 >> "$test_log" 2>&1
+    $cds dump --no-color -d "$MY_REGION_ROOT"/8810 \
+        -d "$MY_REGION_ROOT"/8820 \
+        -d "$MY_REGION_ROOT"/8830 >> "$test_log" 2>&1
 
     echo "$(date) resume downstairs" >> "$test_log"
     bring_all_downstairs_online
@@ -296,13 +298,6 @@ printf "[%03d] %d:%02d  ave:%d:%02d  total:%d:%02d errors:%d last_run_seconds:%d
 if [[ $err -eq 0 ]]; then
     # No errors, then cleanup all our logs and the region directories.
     rm -r "$TEST_ROOT"
-    rm -r "$REGION_ROOT"/8810
-    rm -r "$REGION_ROOT"/8810.old
-    rm -r "$REGION_ROOT"/8820
-    rm -r "$REGION_ROOT"/8820.old
-    rm -r "$REGION_ROOT"/8830
-    rm -r "$REGION_ROOT"/8830.old
-    # If the directory is empty, remove it.
-    rmdir "$REGION_ROOT" || true
+    rm -rf "$MY_REGION_ROOT"
 fi
 exit "$err"

From a025ab08329de7113c934a150aa70345f5cc126d Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxidecomputer.com>
Date: Wed, 12 Mar 2025 16:38:48 +0000
Subject: [PATCH 16/21] fix MY_REGION_ROOT everywhere

---
 tools/hammer_loop.sh          | 2 +-
 tools/test_live_repair.sh     | 2 +-
 tools/test_replace_special.sh | 2 +-
 tools/test_replay.sh          | 2 +-
 tools/test_restart_repair.sh  | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tools/hammer_loop.sh b/tools/hammer_loop.sh
index 08b92e457..1f44e2f46 100755
--- a/tools/hammer_loop.sh
+++ b/tools/hammer_loop.sh
@@ -42,7 +42,7 @@ else
     rm -r "$TEST_ROOT"
 fi
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
-MY_REGION_ROOT=${REGION_ROOT/hammer_loop}
+MY_REGION_ROOT=${REGION_ROOT}/hammer_loop
 if [[ ! -d "$MY_REGION_ROOT" ]]; then
     mkdir -p "$MY_REGION_ROOT"
     if [[ $? -ne 0 ]]; then
diff --git a/tools/test_live_repair.sh b/tools/test_live_repair.sh
index d308473ed..0f8131bd8 100755
--- a/tools/test_live_repair.sh
+++ b/tools/test_live_repair.sh
@@ -22,7 +22,7 @@ function ctrl_c() {
 }
 
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
-MY_REGION_ROOT=${REGION_ROOT/test_live_repair}
+MY_REGION_ROOT=${REGION_ROOT}/test_live_repair
 if [[ ! -d "$MY_REGION_ROOT" ]]; then
     mkdir -p "$MY_REGION_ROOT"
     if [[ $? -ne 0 ]]; then
diff --git a/tools/test_replace_special.sh b/tools/test_replace_special.sh
index 188d97c5f..ac132b630 100755
--- a/tools/test_replace_special.sh
+++ b/tools/test_replace_special.sh
@@ -18,7 +18,7 @@ function ctrl_c() {
 }
 
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
-MY_REGION_ROOT=${REGION_ROOT/test_replace_special}
+MY_REGION_ROOT=${REGION_ROOT}/test_replace_special
 if [[ ! -d "$MY_REGION_ROOT" ]]; then
     mkdir -p "$MY_REGION_ROOT"
     if [[ $? -ne 0 ]]; then
diff --git a/tools/test_replay.sh b/tools/test_replay.sh
index bd4423122..b9fe39379 100755
--- a/tools/test_replay.sh
+++ b/tools/test_replay.sh
@@ -16,7 +16,7 @@ function ctrl_c() {
 }
 
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
-MY_REGION_ROOT=${REGION_ROOT/test_replay}
+MY_REGION_ROOT=${REGION_ROOT}/test_replay
 if [[ ! -d "$MY_REGION_ROOT" ]]; then
     mkdir -p "$MY_REGION_ROOT"
     if [[ $? -ne 0 ]]; then
diff --git a/tools/test_restart_repair.sh b/tools/test_restart_repair.sh
index 3b89061c5..a598ccb75 100755
--- a/tools/test_restart_repair.sh
+++ b/tools/test_restart_repair.sh
@@ -98,7 +98,7 @@ while getopts 'l:' opt; do
 done
 
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
-MY_REGION_ROOT=${REGION_ROOT/test_restart_repair}
+MY_REGION_ROOT=${REGION_ROOT}/test_restart_repair
 if [[ ! -d "$MY_REGION_ROOT" ]]; then
     mkdir -p "$MY_REGION_ROOT"
     if [[ $? -ne 0 ]]; then

From 2e4506a6a8e7efd9c4907618916c6c39ad0b3da5 Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxidecomputer.com>
Date: Wed, 12 Mar 2025 18:07:45 +0000
Subject: [PATCH 17/21] mkdir echo, work together

---
 tools/hammer_loop.sh          | 14 +++++++-------
 tools/test_live_repair.sh     | 13 ++++++-------
 tools/test_replace_special.sh | 13 ++++++-------
 tools/test_replay.sh          | 13 ++++++-------
 tools/test_restart_repair.sh  | 13 ++++++-------
 5 files changed, 31 insertions(+), 35 deletions(-)

diff --git a/tools/hammer_loop.sh b/tools/hammer_loop.sh
index 1f44e2f46..e56ad1331 100755
--- a/tools/hammer_loop.sh
+++ b/tools/hammer_loop.sh
@@ -31,16 +31,16 @@ fi
 
 WORK_ROOT=${WORK_ROOT:-/tmp}
 TEST_ROOT="$WORK_ROOT/hammer_loop"
-if [[ ! -d "$TEST_ROOT" ]]; then
-    mkdir -p "$TEST_ROOT"
-    if [[ $? -ne 0 ]]; then
-        echo "Failed to make test root $TEST_ROOT"
-        exit 1
-    fi
-else
+if [[ -d "$TEST_ROOT" ]]; then
     # Delete previous test data
     rm -r "$TEST_ROOT"
 fi
+mkdir -p "$TEST_ROOT"
+if [[ $? -ne 0 ]]; then
+    echo "Failed to make test root $TEST_ROOT"
+    exit 1
+fi
+
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
 MY_REGION_ROOT=${REGION_ROOT}/hammer_loop
 if [[ ! -d "$MY_REGION_ROOT" ]]; then
diff --git a/tools/test_live_repair.sh b/tools/test_live_repair.sh
index 0f8131bd8..5cf055e29 100755
--- a/tools/test_live_repair.sh
+++ b/tools/test_live_repair.sh
@@ -36,16 +36,15 @@ fi
 # Location of logs and working files
 WORK_ROOT=${WORK_ROOT:-/tmp}
 TEST_ROOT="$WORK_ROOT/test_live_repair"
-if [[ ! -d "$TEST_ROOT" ]]; then
-    mkdir -p "$TEST_ROOT"
-    if [[ $? -ne 0 ]]; then
-        echo "Failed to make test root $TEST_ROOT"
-        exit 1
-    fi  
-else
+if [[ -d "$TEST_ROOT" ]]; then
     # Delete previous test data
     rm -r "$TEST_ROOT"
 fi
+mkdir -p "$TEST_ROOT"
+if [[ $? -ne 0 ]]; then
+    echo "Failed to make test root $TEST_ROOT"
+    exit 1
+fi
 
 loop_log="$TEST_ROOT"/test_live_repair_summary.log
 test_log="$TEST_ROOT"/test_live_repair.log
diff --git a/tools/test_replace_special.sh b/tools/test_replace_special.sh
index ac132b630..f89e30175 100755
--- a/tools/test_replace_special.sh
+++ b/tools/test_replace_special.sh
@@ -32,16 +32,15 @@ fi
 # Location of logs and working files
 WORK_ROOT=${WORK_ROOT:-/tmp}
 TEST_ROOT="${WORK_ROOT}/test_replace_special"
-if [[ ! -d "$TEST_ROOT" ]]; then
-    mkdir -p "$TEST_ROOT"
-    if [[ $? -ne 0 ]]; then
-        echo "Failed to make test root $TEST_ROOT"
-        exit 1
-    fi
-else
+if [[ -d "$TEST_ROOT" ]]; then
     # Delete previous test data
     rm -r "$TEST_ROOT"
 fi
+mkdir -p "$TEST_ROOT"
+if [[ $? -ne 0 ]]; then
+    echo "Failed to make test root $TEST_ROOT"
+    exit 1
+fi
 
 loop_log="$TEST_ROOT/test_replace_special_summary.log"
 test_log="$TEST_ROOT/test_replace_special.log"
diff --git a/tools/test_replay.sh b/tools/test_replay.sh
index b9fe39379..3f352cee9 100755
--- a/tools/test_replay.sh
+++ b/tools/test_replay.sh
@@ -29,16 +29,15 @@ fi
 
 WORK_ROOT=${WORK_ROOT:-/tmp}
 TEST_ROOT="$WORK_ROOT/test_replay"
-if [[ ! -d "$TEST_ROOT" ]]; then
-    mkdir -p "$TEST_ROOT"
-    if [[ $? -ne 0 ]]; then
-        echo "Failed to make test root $TEST_ROOT"
-        exit 1
-    fi
-else
+if [[ -d "$TEST_ROOT" ]]; then
     # Delete previous test data
     rm -r "$TEST_ROOT"
 fi
+mkdir -p "$TEST_ROOT"
+if [[ $? -ne 0 ]]; then
+    echo "Failed to make test root $TEST_ROOT"
+    exit 1
+fi
 
 test_log="$TEST_ROOT/test_replay.log"
 verify_log="$TEST_ROOT/test_replay_verify.log"
diff --git a/tools/test_restart_repair.sh b/tools/test_restart_repair.sh
index a598ccb75..53bfb526b 100755
--- a/tools/test_restart_repair.sh
+++ b/tools/test_restart_repair.sh
@@ -111,16 +111,15 @@ fi
 
 WORK_ROOT=${WORK_ROOT:-/tmp}
 TEST_ROOT="$WORK_ROOT/test_restart_repair"
-if [[ ! -d "$TEST_ROOT" ]]; then
-    mkdir -p "$TEST_ROOT"
-    if [[ $? -ne 0 ]]; then
-        echo "Failed to make test root $TEST_ROOT"
-        exit 1
-    fi
-else
+if [[ -d "$TEST_ROOT" ]]; then
     # Delete previous test data
     rm -r "$TEST_ROOT"
 fi
+mkdir -p "$TEST_ROOT"
+if [[ $? -ne 0 ]]; then
+    echo "Failed to make test root $TEST_ROOT"
+    exit 1
+fi
 
 export loop_log="$TEST_ROOT/test_restart_repair.log"
 export test_log="$TEST_ROOT/test_restart_repair_test.log"

From b67778bbe76172bc4dd7fd3f88a3d2b4dc9ef2f4 Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxidecomputer.com>
Date: Wed, 12 Mar 2025 18:10:50 +0000
Subject: [PATCH 18/21] and this one

---
 tools/test_repair.sh | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/tools/test_repair.sh b/tools/test_repair.sh
index 396f65c32..92e07c9bd 100755
--- a/tools/test_repair.sh
+++ b/tools/test_repair.sh
@@ -42,7 +42,7 @@ done
 
 # For buildomat, the regions should be in /var/tmp
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
-MY_REGION_ROOT=${REGION_ROOT/test_repair}
+MY_REGION_ROOT=${REGION_ROOT}/test_repair
 if [[ ! -d "$MY_REGION_ROOT" ]]; then
     mkdir -p "$MY_REGION_ROOT"
     if [[ $? -ne 0 ]]; then
@@ -56,16 +56,15 @@ fi
 # Location of logs and working files
 WORK_ROOT=${WORK_ROOT:-/tmp}
 TEST_ROOT="$WORK_ROOT/test_live_repair"
-if [[ ! -d "$TEST_ROOT" ]]; then
-    mkdir -p "$TEST_ROOT"
-    if [[ $? -ne 0 ]]; then
-        echo "Failed to make test root $TEST_ROOT"
-        exit 1
-    fi
-else
+if [[ -d "$TEST_ROOT" ]]; then
     # Delete previous test data
     rm -r "$TEST_ROOT"
 fi
+mkdir -p "$TEST_ROOT"
+if [[ $? -ne 0 ]]; then
+    echo "Failed to make test root $TEST_ROOT"
+    exit 1
+fi
 
 verify_file="$TEST_ROOT/test_repair_verify.data"
 test_log="$TEST_ROOT/test_repair_out.txt"

From 137d07acf5551e0f4754fee6f79ec839e069fa1c Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxidecomputer.com>
Date: Thu, 13 Mar 2025 23:04:55 +0000
Subject: [PATCH 19/21] Delete MY_REGION_DIR, then mkdir

---
 tools/hammer_loop.sh          | 13 ++++++-------
 tools/test_live_repair.sh     | 13 ++++++-------
 tools/test_repair.sh          | 13 ++++++-------
 tools/test_replace_special.sh | 13 ++++++-------
 tools/test_replay.sh          | 13 ++++++-------
 tools/test_restart_repair.sh  | 13 ++++++-------
 6 files changed, 36 insertions(+), 42 deletions(-)

diff --git a/tools/hammer_loop.sh b/tools/hammer_loop.sh
index e56ad1331..b3684f747 100755
--- a/tools/hammer_loop.sh
+++ b/tools/hammer_loop.sh
@@ -43,15 +43,14 @@ fi
 
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
 MY_REGION_ROOT=${REGION_ROOT}/hammer_loop
-if [[ ! -d "$MY_REGION_ROOT" ]]; then
-    mkdir -p "$MY_REGION_ROOT"
-    if [[ $? -ne 0 ]]; then
-        echo "Failed to make region root $MY_REGION_ROOT"
-        exit 1
-    fi
-else
+if [[ -d "$MY_REGION_ROOT" ]]; then
     rm -rf "$MY_REGION_ROOT"
 fi
+mkdir -p "$MY_REGION_ROOT"
+if [[ $? -ne 0 ]]; then
+    echo "Failed to make region root $MY_REGION_ROOT"
+    exit 1
+fi
 
 loop_log="$TEST_ROOT/hammer_loop.log"
 test_log="$TEST_ROOT/hammer_loop_test.log"
diff --git a/tools/test_live_repair.sh b/tools/test_live_repair.sh
index 5cf055e29..7307b7ab0 100755
--- a/tools/test_live_repair.sh
+++ b/tools/test_live_repair.sh
@@ -23,15 +23,14 @@ function ctrl_c() {
 
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
 MY_REGION_ROOT=${REGION_ROOT}/test_live_repair
-if [[ ! -d "$MY_REGION_ROOT" ]]; then
-    mkdir -p "$MY_REGION_ROOT"
-    if [[ $? -ne 0 ]]; then
-        echo "Failed to make region root $MY_REGION_ROOT"
-        exit 1
-    fi
-else
+if [[ -d "$MY_REGION_ROOT" ]]; then
     rm -rf "$MY_REGION_ROOT"
 fi
+mkdir -p "$MY_REGION_ROOT"
+if [[ $? -ne 0 ]]; then
+    echo "Failed to make region root $MY_REGION_ROOT"
+    exit 1
+fi
 
 # Location of logs and working files
 WORK_ROOT=${WORK_ROOT:-/tmp}
diff --git a/tools/test_repair.sh b/tools/test_repair.sh
index 92e07c9bd..95145160d 100755
--- a/tools/test_repair.sh
+++ b/tools/test_repair.sh
@@ -43,15 +43,14 @@ done
 # For buildomat, the regions should be in /var/tmp
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
 MY_REGION_ROOT=${REGION_ROOT}/test_repair
-if [[ ! -d "$MY_REGION_ROOT" ]]; then
-    mkdir -p "$MY_REGION_ROOT"
-    if [[ $? -ne 0 ]]; then
-        echo "Failed to make region root $MY_REGION_ROOT"
-        exit 1
-    fi
-else
+if [[ -d "$MY_REGION_ROOT" ]]; then
     rm -rf "$MY_REGION_ROOT"
 fi
+mkdir -p "$MY_REGION_ROOT"
+if [[ $? -ne 0 ]]; then
+    echo "Failed to make region root $MY_REGION_ROOT"
+    exit 1
+fi
 
 # Location of logs and working files
 WORK_ROOT=${WORK_ROOT:-/tmp}
diff --git a/tools/test_replace_special.sh b/tools/test_replace_special.sh
index f89e30175..d4eed8254 100755
--- a/tools/test_replace_special.sh
+++ b/tools/test_replace_special.sh
@@ -19,15 +19,14 @@ function ctrl_c() {
 
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
 MY_REGION_ROOT=${REGION_ROOT}/test_replace_special
-if [[ ! -d "$MY_REGION_ROOT" ]]; then
-    mkdir -p "$MY_REGION_ROOT"
-    if [[ $? -ne 0 ]]; then
-        echo "Failed to make region root $MY_REGION_ROOT"
-        exit 1
-    fi
-else
+if [[ -d "$MY_REGION_ROOT" ]]; then
     rm -rf "$MY_REGION_ROOT"
 fi
+mkdir -p "$MY_REGION_ROOT"
+if [[ $? -ne 0 ]]; then
+    echo "Failed to make region root $MY_REGION_ROOT"
+    exit 1
+fi
 
 # Location of logs and working files
 WORK_ROOT=${WORK_ROOT:-/tmp}
diff --git a/tools/test_replay.sh b/tools/test_replay.sh
index 3f352cee9..c736da385 100755
--- a/tools/test_replay.sh
+++ b/tools/test_replay.sh
@@ -17,15 +17,14 @@ function ctrl_c() {
 
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
 MY_REGION_ROOT=${REGION_ROOT}/test_replay
-if [[ ! -d "$MY_REGION_ROOT" ]]; then
-    mkdir -p "$MY_REGION_ROOT"
-    if [[ $? -ne 0 ]]; then
-        echo "Failed to make region root $MY_REGION_ROOT"
-        exit 1
-    fi
-else
+if [[ -d "$MY_REGION_ROOT" ]]; then
     rm -rf "$MY_REGION_ROOT"
 fi
+mkdir -p "$MY_REGION_ROOT"
+if [[ $? -ne 0 ]]; then
+    echo "Failed to make region root $MY_REGION_ROOT"
+    exit 1
+fi
 
 WORK_ROOT=${WORK_ROOT:-/tmp}
 TEST_ROOT="$WORK_ROOT/test_replay"
diff --git a/tools/test_restart_repair.sh b/tools/test_restart_repair.sh
index 53bfb526b..f95e72baf 100755
--- a/tools/test_restart_repair.sh
+++ b/tools/test_restart_repair.sh
@@ -99,15 +99,14 @@ done
 
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
 MY_REGION_ROOT=${REGION_ROOT}/test_restart_repair
-if [[ ! -d "$MY_REGION_ROOT" ]]; then
-    mkdir -p "$MY_REGION_ROOT"
-    if [[ $? -ne 0 ]]; then
-        echo "Failed to make region root $MY_REGION_ROOT"
-        exit 1
-    fi
-else
+if [[ -d "$MY_REGION_ROOT" ]]; then
     rm -rf "$MY_REGION_ROOT"
 fi
+mkdir -p "$MY_REGION_ROOT"
+if [[ $? -ne 0 ]]; then
+    echo "Failed to make region root $MY_REGION_ROOT"
+    exit 1
+fi
 
 WORK_ROOT=${WORK_ROOT:-/tmp}
 TEST_ROOT="$WORK_ROOT/test_restart_repair"

From 3372195a867b236a12b0dc552f78f3398132e880 Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxidecomputer.com>
Date: Mon, 17 Mar 2025 16:56:45 +0000
Subject: [PATCH 20/21] set REGION_ROOT and TEST_ROOT the same

---
 tools/hammer_loop.sh          | 2 +-
 tools/test_live_repair.sh     | 4 ++--
 tools/test_repair.sh          | 4 ++--
 tools/test_replace_special.sh | 2 +-
 tools/test_replay.sh          | 4 ++--
 tools/test_restart_repair.sh  | 4 ++--
 6 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tools/hammer_loop.sh b/tools/hammer_loop.sh
index b3684f747..40b517e66 100755
--- a/tools/hammer_loop.sh
+++ b/tools/hammer_loop.sh
@@ -42,7 +42,7 @@ if [[ $? -ne 0 ]]; then
 fi
 
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
-MY_REGION_ROOT=${REGION_ROOT}/hammer_loop
+MY_REGION_ROOT="${REGION_ROOT}/hammer_loop"
 if [[ -d "$MY_REGION_ROOT" ]]; then
     rm -rf "$MY_REGION_ROOT"
 fi
diff --git a/tools/test_live_repair.sh b/tools/test_live_repair.sh
index 7307b7ab0..bbfe0ca0c 100755
--- a/tools/test_live_repair.sh
+++ b/tools/test_live_repair.sh
@@ -22,7 +22,7 @@ function ctrl_c() {
 }
 
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
-MY_REGION_ROOT=${REGION_ROOT}/test_live_repair
+MY_REGION_ROOT="${REGION_ROOT}/test_live_repair"
 if [[ -d "$MY_REGION_ROOT" ]]; then
     rm -rf "$MY_REGION_ROOT"
 fi
@@ -34,7 +34,7 @@ fi
 
 # Location of logs and working files
 WORK_ROOT=${WORK_ROOT:-/tmp}
-TEST_ROOT="$WORK_ROOT/test_live_repair"
+TEST_ROOT="${WORK_ROOT}/test_live_repair"
 if [[ -d "$TEST_ROOT" ]]; then
     # Delete previous test data
     rm -r "$TEST_ROOT"
diff --git a/tools/test_repair.sh b/tools/test_repair.sh
index 95145160d..aec74b15c 100755
--- a/tools/test_repair.sh
+++ b/tools/test_repair.sh
@@ -42,7 +42,7 @@ done
 
 # For buildomat, the regions should be in /var/tmp
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
-MY_REGION_ROOT=${REGION_ROOT}/test_repair
+MY_REGION_ROOT="${REGION_ROOT}/test_repair"
 if [[ -d "$MY_REGION_ROOT" ]]; then
     rm -rf "$MY_REGION_ROOT"
 fi
@@ -54,7 +54,7 @@ fi
 
 # Location of logs and working files
 WORK_ROOT=${WORK_ROOT:-/tmp}
-TEST_ROOT="$WORK_ROOT/test_live_repair"
+TEST_ROOT="${WORK_ROOT}/test_live_repair"
 if [[ -d "$TEST_ROOT" ]]; then
     # Delete previous test data
     rm -r "$TEST_ROOT"
diff --git a/tools/test_replace_special.sh b/tools/test_replace_special.sh
index d4eed8254..382eb875e 100755
--- a/tools/test_replace_special.sh
+++ b/tools/test_replace_special.sh
@@ -18,7 +18,7 @@ function ctrl_c() {
 }
 
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
-MY_REGION_ROOT=${REGION_ROOT}/test_replace_special
+MY_REGION_ROOT="${REGION_ROOT}/test_replace_special"
 if [[ -d "$MY_REGION_ROOT" ]]; then
     rm -rf "$MY_REGION_ROOT"
 fi
diff --git a/tools/test_replay.sh b/tools/test_replay.sh
index c736da385..516e19495 100755
--- a/tools/test_replay.sh
+++ b/tools/test_replay.sh
@@ -16,7 +16,7 @@ function ctrl_c() {
 }
 
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
-MY_REGION_ROOT=${REGION_ROOT}/test_replay
+MY_REGION_ROOT="${REGION_ROOT}/test_replay"
 if [[ -d "$MY_REGION_ROOT" ]]; then
     rm -rf "$MY_REGION_ROOT"
 fi
@@ -27,7 +27,7 @@ if [[ $? -ne 0 ]]; then
 fi
 
 WORK_ROOT=${WORK_ROOT:-/tmp}
-TEST_ROOT="$WORK_ROOT/test_replay"
+TEST_ROOT="${WORK_ROOT}/test_replay"
 if [[ -d "$TEST_ROOT" ]]; then
     # Delete previous test data
     rm -r "$TEST_ROOT"
diff --git a/tools/test_restart_repair.sh b/tools/test_restart_repair.sh
index f95e72baf..679f8b503 100755
--- a/tools/test_restart_repair.sh
+++ b/tools/test_restart_repair.sh
@@ -98,7 +98,7 @@ while getopts 'l:' opt; do
 done
 
 REGION_ROOT=${REGION_ROOT:-/var/tmp}
-MY_REGION_ROOT=${REGION_ROOT}/test_restart_repair
+MY_REGION_ROOT="${REGION_ROOT}/test_restart_repair"
 if [[ -d "$MY_REGION_ROOT" ]]; then
     rm -rf "$MY_REGION_ROOT"
 fi
@@ -109,7 +109,7 @@ if [[ $? -ne 0 ]]; then
 fi
 
 WORK_ROOT=${WORK_ROOT:-/tmp}
-TEST_ROOT="$WORK_ROOT/test_restart_repair"
+TEST_ROOT="${WORK_ROOT}/test_restart_repair"
 if [[ -d "$TEST_ROOT" ]]; then
     # Delete previous test data
     rm -r "$TEST_ROOT"

From b97d4d6dc6571d48a617bb70063a2bb49f1de3bc Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxidecomputer.com>
Date: Mon, 17 Mar 2025 21:41:20 +0000
Subject: [PATCH 21/21] Maybe final polishing

---
 tools/hammer_loop.sh          | 14 +++++++-------
 tools/test_live_repair.sh     | 14 +++++++-------
 tools/test_repair.sh          | 14 +++++++-------
 tools/test_replace_special.sh | 14 +++++++-------
 tools/test_replay.sh          | 12 ++++++------
 tools/test_restart_repair.sh  | 16 ++++++++--------
 6 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/tools/hammer_loop.sh b/tools/hammer_loop.sh
index 40b517e66..016bfbf4c 100755
--- a/tools/hammer_loop.sh
+++ b/tools/hammer_loop.sh
@@ -13,9 +13,9 @@ ROOT=$(cd "$(dirname "$0")/.." && pwd)
 cd "$ROOT" || (echo failed to cd "$ROOT"; exit 1)
 
 export BINDIR=${BINDIR:-$ROOT/target/debug}
-hammer="$BINDIR/crucible-hammer"
-cds="$BINDIR/crucible-downstairs"
-dsc="$BINDIR/dsc"
+hammer="${BINDIR}/crucible-hammer"
+cds="${BINDIR}/crucible-downstairs"
+dsc="${BINDIR}/dsc"
 for bin in $hammer $cds $dsc; do
     if [[ ! -f "$bin" ]]; then
         echo "Can't find crucible binary at $bin" >&2
@@ -30,7 +30,7 @@ if pgrep -fl -U "$(id -u)" "$cds"; then
 fi
 
 WORK_ROOT=${WORK_ROOT:-/tmp}
-TEST_ROOT="$WORK_ROOT/hammer_loop"
+TEST_ROOT="${WORK_ROOT}/hammer_loop"
 if [[ -d "$TEST_ROOT" ]]; then
     # Delete previous test data
     rm -r "$TEST_ROOT"
@@ -52,9 +52,9 @@ if [[ $? -ne 0 ]]; then
     exit 1
 fi
 
-loop_log="$TEST_ROOT/hammer_loop.log"
-test_log="$TEST_ROOT/hammer_loop_test.log"
-dsc_ds_log="$TEST_ROOT/hammer_loop_dsc.log"
+loop_log="${TEST_ROOT}/hammer_loop.log"
+test_log="${TEST_ROOT}/hammer_loop_test.log"
+dsc_ds_log="${TEST_ROOT}/hammer_loop_dsc.log"
 
 loops=20
 
diff --git a/tools/test_live_repair.sh b/tools/test_live_repair.sh
index bbfe0ca0c..c7cd2d76a 100755
--- a/tools/test_live_repair.sh
+++ b/tools/test_live_repair.sh
@@ -45,17 +45,17 @@ if [[ $? -ne 0 ]]; then
     exit 1
 fi
 
-loop_log="$TEST_ROOT"/test_live_repair_summary.log
-test_log="$TEST_ROOT"/test_live_repair.log
-verify_log="$TEST_ROOT/test_live_repair_verify.log"
-dsc_ds_log="$TEST_ROOT/test_live_repair_dsc.log"
+loop_log="${TEST_ROOT}/test_live_repair_summary.log"
+test_log="${TEST_ROOT}/test_live_repair.log"
+verify_log="${TEST_ROOT}/test_live_repair_verify.log"
+dsc_ds_log="${TEST_ROOT}/test_live_repair_dsc.log"
 
 ROOT=$(cd "$(dirname "$0")/.." && pwd)
 cd "$ROOT" || (echo failed to cd "$ROOT"; exit 1)
 export BINDIR=${BINDIR:-$ROOT/target/debug}
-crucible_test="$BINDIR/crutest"
-dsc="$BINDIR/dsc"
-downstairs="$BINDIR/crucible-downstairs"
+crucible_test="${BINDIR}/crutest"
+dsc="${BINDIR}/dsc"
+downstairs="${BINDIR}/crucible-downstairs"
 if [[ ! -f "$crucible_test" ]] || [[ ! -f "$dsc" ]] || [[ ! -f "$downstairs" ]]; then
     echo "Can't find required binaries"
     echo "Missing $crucible_test or $dsc or $downstairs"
diff --git a/tools/test_repair.sh b/tools/test_repair.sh
index aec74b15c..5f50b5451 100755
--- a/tools/test_repair.sh
+++ b/tools/test_repair.sh
@@ -28,9 +28,9 @@ ROOT=$(cd "$(dirname "$0")/.." && pwd)
 cd "$ROOT" || (echo failed to cd "$ROOT"; exit 1)
 export BINDIR=${BINDIR:-$ROOT/target/debug}
 
-cds="$BINDIR/crucible-downstairs"
-ct="$BINDIR/crutest"
-dsc="$BINDIR/dsc"
+cds="${BINDIR}/crucible-downstairs"
+ct="${BINDIR}/crutest"
+dsc="${BINDIR}/dsc"
 
 for bin in $cds $ct $dsc; do
     if [[ ! -f "$bin" ]]; then
@@ -65,10 +65,10 @@ if [[ $? -ne 0 ]]; then
     exit 1
 fi
 
-verify_file="$TEST_ROOT/test_repair_verify.data"
-test_log="$TEST_ROOT/test_repair_out.txt"
-ds_log_prefix="$TEST_ROOT/test_repair_ds"
-dsc_output_dir="$TEST_ROOT/test_repair_dsc"
+verify_file="${TEST_ROOT}/test_repair_verify.data"
+test_log="${TEST_ROOT}/test_repair_out.txt"
+ds_log_prefix="${TEST_ROOT}/test_repair_ds"
+dsc_output_dir="${TEST_ROOT}/test_repair_dsc"
 loops=100
 
 usage () {
diff --git a/tools/test_replace_special.sh b/tools/test_replace_special.sh
index 382eb875e..8d94c55a7 100755
--- a/tools/test_replace_special.sh
+++ b/tools/test_replace_special.sh
@@ -41,17 +41,17 @@ if [[ $? -ne 0 ]]; then
     exit 1
 fi
 
-loop_log="$TEST_ROOT/test_replace_special_summary.log"
-test_log="$TEST_ROOT/test_replace_special.log"
-verify_log="$TEST_ROOT/test_replace_special_verify.log"
-dsc_ds_log="$TEST_ROOT/test_replace_special_dsc.log"
+loop_log="${TEST_ROOT}/test_replace_special_summary.log"
+test_log="${TEST_ROOT}/test_replace_special.log"
+verify_log="${TEST_ROOT}/test_replace_special_verify.log"
+dsc_ds_log="${TEST_ROOT}/test_replace_special_dsc.log"
 
 ROOT=$(cd "$(dirname "$0")/.." && pwd)
 cd "$ROOT" || (echo failed to cd "$ROOT"; exit 1)
 export BINDIR=${BINDIR:-$ROOT/target/debug}
-crucible_test="$BINDIR/crutest"
-dsc="$BINDIR/dsc"
-downstairs="$BINDIR/crucible-downstairs"
+crucible_test="${BINDIR}/crutest"
+dsc="${BINDIR}/dsc"
+downstairs="${BINDIR}/crucible-downstairs"
 if [[ ! -f "$crucible_test" ]] || [[ ! -f "$dsc" ]] || [[ ! -f "$downstairs" ]]; then
     echo "Can't find required binaries"
     echo "Missing $crucible_test or $dsc or $downstairs"
diff --git a/tools/test_replay.sh b/tools/test_replay.sh
index 516e19495..907ebf5cf 100755
--- a/tools/test_replay.sh
+++ b/tools/test_replay.sh
@@ -38,16 +38,16 @@ if [[ $? -ne 0 ]]; then
     exit 1
 fi
 
-test_log="$TEST_ROOT/test_replay.log"
-verify_log="$TEST_ROOT/test_replay_verify.log"
-dsc_ds_log="$TEST_ROOT/test_replay_dsc.log"
+test_log="${TEST_ROOT}/test_replay.log"
+verify_log="${TEST_ROOT}/test_replay_verify.log"
+dsc_ds_log="${TEST_ROOT}/test_replay_dsc.log"
 
 ROOT=$(cd "$(dirname "$0")/.." && pwd)
 cd "$ROOT" || (echo failed to cd "$ROOT"; exit 1)
 export BINDIR=${BINDIR:-$ROOT/target/debug}
-crucible_test="$BINDIR/crutest"
-dsc="$BINDIR/dsc"
-downstairs="$BINDIR/crucible-downstairs"
+crucible_test="${BINDIR}/crutest"
+dsc="${BINDIR}/dsc"
+downstairs="${BINDIR}/crucible-downstairs"
 if [[ ! -f "$crucible_test" ]] || [[ ! -f "$dsc" ]] || [[ ! -f "$downstairs" ]]; then
     echo "Can't find required binaries"
     echo "Missing $crucible_test or $dsc or $downstairs"
diff --git a/tools/test_restart_repair.sh b/tools/test_restart_repair.sh
index 679f8b503..dbec2770f 100755
--- a/tools/test_restart_repair.sh
+++ b/tools/test_restart_repair.sh
@@ -63,9 +63,9 @@ ROOT=$(cd "$(dirname "$0")/.." && pwd)
 cd "$ROOT" || (echo failed to cd "$ROOT"; exit 1)
 export BINDIR=${BINDIR:-$ROOT/target/debug}
 
-cds="$BINDIR/crucible-downstairs"
-ct="$BINDIR/crutest"
-dsc="$BINDIR/dsc"
+cds="${BINDIR}/crucible-downstairs"
+ct="${BINDIR}/crutest"
+dsc="${BINDIR}/dsc"
 for bin in $cds $ct $dsc; do
     if [[ ! -f "$bin" ]]; then
         echo "Can't find crucible binary at $bin" >&2
@@ -120,11 +120,11 @@ if [[ $? -ne 0 ]]; then
     exit 1
 fi
 
-export loop_log="$TEST_ROOT/test_restart_repair.log"
-export test_log="$TEST_ROOT/test_restart_repair_test.log"
-export verify_log="$TEST_ROOT/test_restart_repair_verify.log"
-export dsc_log="$TEST_ROOT/test_restart_repair_dsc.log"
-export dsc_ds_log="$TEST_ROOT/dsc"
+export loop_log="${TEST_ROOT}/test_restart_repair.log"
+export test_log="${TEST_ROOT}/test_restart_repair_test.log"
+export verify_log="${TEST_ROOT}/test_restart_repair_verify.log"
+export dsc_log="${TEST_ROOT}/test_restart_repair_dsc.log"
+export dsc_ds_log="${TEST_ROOT}/dsc"
 
 touch "$loop_log"
 echo "starting $(date)" | tee "$loop_log"