test-crudd can collect more info, test_up can be gentle (#997)

Update test-crudd to have a timeout and gather more info when
things go wrong.

Update test_up.sh to wait a little between tests.

Author: leftwo

.github/buildomat/jobs/test-crudd-benchmark.sh

@@ -5,6 +5,8 @@
 #: target = "helios-2.0"
 #: output_rules = [
 #:  "/tmp/crudd-speed-battery-results.json",
+#:  "%/tmp/debug/*.txt",
+#:  "/tmp/dsc/*.txt",
 #:  "/tmp/core.*",
 #: ]
 #: skip_clone = true
@@ -42,6 +44,22 @@ done
 
 export BINDIR=/var/tmp/bins
 
+banner setup
+pfexec plimit -n 9123456 $$
+
+echo "Setup self timeout"
+jobpid=$$; (sleep $(( 2 * 60 * 60 )); ps -ef; zfs list;kill $jobpid) &
+
+echo "Setup debug logging"
+mkdir /tmp/debug
+psrinfo -v > /tmp/debug/psrinfo.txt
+df -h > /tmp/debug/df.txt
+prstat -d d -mLc 1 > /tmp/debug/prstat.txt 2>&1 &
+iostat -T d -xn 1 > /tmp/debug/iostat.txt 2>&1 &
+mpstat -T d 1 > /tmp/debug/mpstat.txt 2>&1 &
+vmstat -T d -p 1 < /dev/null > /tmp/debug/paging.txt 2>&1 &
+pfexec dtrace -Z -s $input/scripts/perf-downstairs-tick.d > /tmp/debug/dtrace.txt 2>&1 &
+
 banner "crudd bench"
 pfexec plimit -n 9123456 $$
 

tools/test_up.sh

@@ -155,8 +155,8 @@ for tt in ${test_list}; do
     echo "" >> "${log_prefix}_out.txt"
     echo "Running test: $tt" | tee -a "${log_prefix}_out.txt"
     echo "Running test: $tt" >> "${log_prefix}_out.txt"
-    echo "$ct" "$tt" -g "$gen" --verify-at-end -q "${args[@]}" >> "${log_prefix}_out.txt"
-    if ! "$ct" "$tt" -g "$gen" --verify-at-end -q "${args[@]}" >> "${log_prefix}_out.txt" 2>&1; then
+    echo "$ct" "$tt" -g "$gen" --verify-at-end --stable "${args[@]}" >> "${log_prefix}_out.txt"
+    if ! "$ct" "$tt" -g "$gen" --verify-at-end --stable "${args[@]}" >> "${log_prefix}_out.txt" 2>&1; then
         (( res += 1 ))
         echo ""
         echo "Failed crutest $tt test"
@@ -166,7 +166,10 @@ for tt in ${test_list}; do
     else
         echo "Completed test: $tt"
     fi
-    (( gen += 1 ))
+    (( gen += 2 ))
+    # Yes, this is bad.  Sometimes we just need a little time for the test
+    # to finish everything before the next one starts.
+    sleep 5
 done
 (( gen += 10 ))