From 36bfdb35740d212ccff3f0df38b7e31989b940da Mon Sep 17 00:00:00 2001 From: Michael Schwarz Date: Tue, 4 Mar 2025 15:17:36 +0100 Subject: [PATCH 1/3] Fix flaky cram tests where `vid`s leak into witnesses --- .../56-witness/66-ghost-alloc-lock.t | 136 +----------------- .../56-witness/66-ghost-alloc-stripped.yml | 132 +++++++++++++++++ .../56-witness/compare-ghost-alloc.sh | 45 ++++++ tests/regression/56-witness/dune | 2 +- 4 files changed, 181 insertions(+), 134 deletions(-) create mode 100644 tests/regression/56-witness/66-ghost-alloc-stripped.yml create mode 100755 tests/regression/56-witness/compare-ghost-alloc.sh diff --git a/tests/regression/56-witness/66-ghost-alloc-lock.t b/tests/regression/56-witness/66-ghost-alloc-lock.t index 09c7c4250b..afbb276726 100644 --- a/tests/regression/56-witness/66-ghost-alloc-lock.t +++ b/tests/regression/56-witness/66-ghost-alloc-lock.t @@ -16,136 +16,6 @@ unsafe: 0 total memory locations: 4 - $ yamlWitnessStrip < witness.yml - - entry_type: ghost_instrumentation - content: - ghost_variables: - - name: alloc_m559918035_locked - scope: global - type: int - initial: - value: "0" - format: c_expression - - name: alloc_m861095507_locked - scope: global - type: int - initial: - value: "0" - format: c_expression - - name: multithreaded - scope: global - type: int - initial: - value: "0" - format: c_expression - ghost_updates: - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 10 - column: 3 - function: t_fun - updates: - - variable: alloc_m559918035_locked - value: "1" - format: c_expression - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 13 - column: 3 - function: t_fun - updates: - - variable: alloc_m559918035_locked - value: "0" - format: c_expression - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 14 - column: 3 - function: t_fun - updates: - - variable: alloc_m861095507_locked - value: "1" - format: c_expression - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 17 - column: 3 - function: t_fun - updates: - - variable: alloc_m861095507_locked - value: "0" - format: c_expression - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 28 - column: 3 - function: main - updates: - - variable: multithreaded - value: "1" - format: c_expression - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 30 - column: 3 - function: main - updates: - - variable: alloc_m559918035_locked - value: "1" - format: c_expression - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 32 - column: 3 - function: main - updates: - - variable: alloc_m559918035_locked - value: "0" - format: c_expression - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 33 - column: 3 - function: main - updates: - - variable: alloc_m861095507_locked - value: "1" - format: c_expression - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 35 - column: 3 - function: main - updates: - - variable: alloc_m861095507_locked - value: "0" - format: c_expression - - entry_type: flow_insensitive_invariant - flow_insensitive_invariant: - string: '! multithreaded || (alloc_m861095507_locked || g2 == 0)' - type: assertion - format: C - - entry_type: flow_insensitive_invariant - flow_insensitive_invariant: - string: '! multithreaded || (alloc_m559918035_locked || g1 == 0)' - type: assertion - format: C - - entry_type: flow_insensitive_invariant - flow_insensitive_invariant: - string: '! multithreaded || (0 <= g2 && g2 <= 1)' - type: assertion - format: C - - entry_type: flow_insensitive_invariant - flow_insensitive_invariant: - string: '! multithreaded || (0 <= g1 && g1 <= 1)' - type: assertion - format: C + $ (yamlWitnessStrip < witness.yml) > new-stripped.yml + $ ./compare-ghost-alloc.sh 66-ghost-alloc-stripped.yml new-stripped.yml + The files are the same after renaming the variables. diff --git a/tests/regression/56-witness/66-ghost-alloc-stripped.yml b/tests/regression/56-witness/66-ghost-alloc-stripped.yml new file mode 100644 index 0000000000..da7f99354b --- /dev/null +++ b/tests/regression/56-witness/66-ghost-alloc-stripped.yml @@ -0,0 +1,132 @@ +- entry_type: ghost_instrumentation + content: + ghost_variables: + - name: alloc_m559918035_locked + scope: global + type: int + initial: + value: "0" + format: c_expression + - name: alloc_m861095507_locked + scope: global + type: int + initial: + value: "0" + format: c_expression + - name: multithreaded + scope: global + type: int + initial: + value: "0" + format: c_expression + ghost_updates: + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 10 + column: 3 + function: t_fun + updates: + - variable: alloc_m559918035_locked + value: "1" + format: c_expression + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 13 + column: 3 + function: t_fun + updates: + - variable: alloc_m559918035_locked + value: "0" + format: c_expression + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 14 + column: 3 + function: t_fun + updates: + - variable: alloc_m861095507_locked + value: "1" + format: c_expression + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 17 + column: 3 + function: t_fun + updates: + - variable: alloc_m861095507_locked + value: "0" + format: c_expression + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 28 + column: 3 + function: main + updates: + - variable: multithreaded + value: "1" + format: c_expression + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 30 + column: 3 + function: main + updates: + - variable: alloc_m559918035_locked + value: "1" + format: c_expression + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 32 + column: 3 + function: main + updates: + - variable: alloc_m559918035_locked + value: "0" + format: c_expression + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 33 + column: 3 + function: main + updates: + - variable: alloc_m861095507_locked + value: "1" + format: c_expression + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 35 + column: 3 + function: main + updates: + - variable: alloc_m861095507_locked + value: "0" + format: c_expression +- entry_type: flow_insensitive_invariant + flow_insensitive_invariant: + string: '! multithreaded || (alloc_m861095507_locked || g2 == 0)' + type: assertion + format: C +- entry_type: flow_insensitive_invariant + flow_insensitive_invariant: + string: '! multithreaded || (alloc_m559918035_locked || g1 == 0)' + type: assertion + format: C +- entry_type: flow_insensitive_invariant + flow_insensitive_invariant: + string: '! multithreaded || (0 <= g2 && g2 <= 1)' + type: assertion + format: C +- entry_type: flow_insensitive_invariant + flow_insensitive_invariant: + string: '! multithreaded || (0 <= g1 && g1 <= 1)' + type: assertion + format: C diff --git a/tests/regression/56-witness/compare-ghost-alloc.sh b/tests/regression/56-witness/compare-ghost-alloc.sh new file mode 100755 index 0000000000..4574dfe9f3 --- /dev/null +++ b/tests/regression/56-witness/compare-ghost-alloc.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Check if the correct number of arguments are provided +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +file1="$1" +file2="$2" + +# Function to extract the first and second occurrences of the pattern +extract_variables() { + grep -o -m 2 'alloc_m[0-9]\+_locked' "$1" +} + +# Extract variables from both files +var1_file1=$(extract_variables "$file1" | sed -n '1p') +var2_file1=$(extract_variables "$file1" | sed -n '2p') +var1_file2=$(extract_variables "$file2" | sed -n '1p') +var2_file2=$(extract_variables "$file2" | sed -n '2p') + +# Check if the variables were found +if [ -z "$var1_file1" ] || [ -z "$var2_file1" ] || [ -z "$var1_file2" ] || [ -z "$var2_file2" ]; then + echo "Error: Could not find the required pattern in one or both files." + exit 1 +fi + +# Create temporary files +temp1=$(mktemp) +temp2=$(mktemp) + +# Rename variables in both files +sed -e "s/\b$var1_file1\b/TEMP_VAR1/g" -e "s/\b$var2_file1\b/TEMP_VAR2/g" "$file1" > "$temp1" +sed -e "s/\b$var1_file2\b/TEMP_VAR1/g" -e "s/\b$var2_file2\b/TEMP_VAR2/g" "$file2" > "$temp2" + +# Compare the modified files +if diff -q "$temp1" "$temp2" > /dev/null; then + echo "The files are the same after renaming the variables." +else + echo "The files are different after renaming the variables." +fi + +# Clean up temporary files +rm "$temp1" "$temp2" diff --git a/tests/regression/56-witness/dune b/tests/regression/56-witness/dune index f6694c60ec..4ce326af28 100644 --- a/tests/regression/56-witness/dune +++ b/tests/regression/56-witness/dune @@ -25,7 +25,7 @@ (run %{update_suite} apron-unassume-set-tokens -q))))) (cram - (deps (glob_files *.c) (glob_files ??-*.yml))) + (deps (glob_files *.c) (glob_files ??-*.yml) (glob_files compare-ghost-alloc.sh))) (cram (applies_to 54-witness-lifter-abortUnless) From f2a8d30e1e159a41fd072da515dd7d42389c0524 Mon Sep 17 00:00:00 2001 From: Michael Schwarz Date: Wed, 5 Mar 2025 13:34:28 +0100 Subject: [PATCH 2/3] Simplify testing and make dune do the diff --- .../56-witness/66-ghost-alloc-lock.t | 135 +++++++++++++++++- .../56-witness/66-ghost-alloc-stripped.yml | 132 ----------------- .../56-witness/compare-ghost-alloc.sh | 45 ------ tests/regression/56-witness/dune | 2 +- .../56-witness/strip-ghost-alloc.sh | 27 ++++ 5 files changed, 161 insertions(+), 180 deletions(-) delete mode 100644 tests/regression/56-witness/66-ghost-alloc-stripped.yml delete mode 100755 tests/regression/56-witness/compare-ghost-alloc.sh create mode 100755 tests/regression/56-witness/strip-ghost-alloc.sh diff --git a/tests/regression/56-witness/66-ghost-alloc-lock.t b/tests/regression/56-witness/66-ghost-alloc-lock.t index afbb276726..b8f5678bc9 100644 --- a/tests/regression/56-witness/66-ghost-alloc-lock.t +++ b/tests/regression/56-witness/66-ghost-alloc-lock.t @@ -17,5 +17,136 @@ total memory locations: 4 $ (yamlWitnessStrip < witness.yml) > new-stripped.yml - $ ./compare-ghost-alloc.sh 66-ghost-alloc-stripped.yml new-stripped.yml - The files are the same after renaming the variables. + $ ./strip-ghost-alloc.sh new-stripped.yml + - entry_type: ghost_instrumentation + content: + ghost_variables: + - name: ALLOC_VAR1_LOCKED + scope: global + type: int + initial: + value: "0" + format: c_expression + - name: ALLOC_VAR2_LOCKED + scope: global + type: int + initial: + value: "0" + format: c_expression + - name: multithreaded + scope: global + type: int + initial: + value: "0" + format: c_expression + ghost_updates: + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 10 + column: 3 + function: t_fun + updates: + - variable: ALLOC_VAR1_LOCKED + value: "1" + format: c_expression + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 13 + column: 3 + function: t_fun + updates: + - variable: ALLOC_VAR1_LOCKED + value: "0" + format: c_expression + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 14 + column: 3 + function: t_fun + updates: + - variable: ALLOC_VAR2_LOCKED + value: "1" + format: c_expression + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 17 + column: 3 + function: t_fun + updates: + - variable: ALLOC_VAR2_LOCKED + value: "0" + format: c_expression + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 28 + column: 3 + function: main + updates: + - variable: multithreaded + value: "1" + format: c_expression + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 30 + column: 3 + function: main + updates: + - variable: ALLOC_VAR1_LOCKED + value: "1" + format: c_expression + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 32 + column: 3 + function: main + updates: + - variable: ALLOC_VAR1_LOCKED + value: "0" + format: c_expression + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 33 + column: 3 + function: main + updates: + - variable: ALLOC_VAR2_LOCKED + value: "1" + format: c_expression + - location: + file_name: 66-ghost-alloc-lock.c + file_hash: $FILE_HASH + line: 35 + column: 3 + function: main + updates: + - variable: ALLOC_VAR2_LOCKED + value: "0" + format: c_expression + - entry_type: flow_insensitive_invariant + flow_insensitive_invariant: + string: '! multithreaded || (ALLOC_VAR2_LOCKED || g2 == 0)' + type: assertion + format: C + - entry_type: flow_insensitive_invariant + flow_insensitive_invariant: + string: '! multithreaded || (ALLOC_VAR1_LOCKED || g1 == 0)' + type: assertion + format: C + - entry_type: flow_insensitive_invariant + flow_insensitive_invariant: + string: '! multithreaded || (0 <= g2 && g2 <= 1)' + type: assertion + format: C + - entry_type: flow_insensitive_invariant + flow_insensitive_invariant: + string: '! multithreaded || (0 <= g1 && g1 <= 1)' + type: assertion + format: C diff --git a/tests/regression/56-witness/66-ghost-alloc-stripped.yml b/tests/regression/56-witness/66-ghost-alloc-stripped.yml deleted file mode 100644 index da7f99354b..0000000000 --- a/tests/regression/56-witness/66-ghost-alloc-stripped.yml +++ /dev/null @@ -1,132 +0,0 @@ -- entry_type: ghost_instrumentation - content: - ghost_variables: - - name: alloc_m559918035_locked - scope: global - type: int - initial: - value: "0" - format: c_expression - - name: alloc_m861095507_locked - scope: global - type: int - initial: - value: "0" - format: c_expression - - name: multithreaded - scope: global - type: int - initial: - value: "0" - format: c_expression - ghost_updates: - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 10 - column: 3 - function: t_fun - updates: - - variable: alloc_m559918035_locked - value: "1" - format: c_expression - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 13 - column: 3 - function: t_fun - updates: - - variable: alloc_m559918035_locked - value: "0" - format: c_expression - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 14 - column: 3 - function: t_fun - updates: - - variable: alloc_m861095507_locked - value: "1" - format: c_expression - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 17 - column: 3 - function: t_fun - updates: - - variable: alloc_m861095507_locked - value: "0" - format: c_expression - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 28 - column: 3 - function: main - updates: - - variable: multithreaded - value: "1" - format: c_expression - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 30 - column: 3 - function: main - updates: - - variable: alloc_m559918035_locked - value: "1" - format: c_expression - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 32 - column: 3 - function: main - updates: - - variable: alloc_m559918035_locked - value: "0" - format: c_expression - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 33 - column: 3 - function: main - updates: - - variable: alloc_m861095507_locked - value: "1" - format: c_expression - - location: - file_name: 66-ghost-alloc-lock.c - file_hash: $FILE_HASH - line: 35 - column: 3 - function: main - updates: - - variable: alloc_m861095507_locked - value: "0" - format: c_expression -- entry_type: flow_insensitive_invariant - flow_insensitive_invariant: - string: '! multithreaded || (alloc_m861095507_locked || g2 == 0)' - type: assertion - format: C -- entry_type: flow_insensitive_invariant - flow_insensitive_invariant: - string: '! multithreaded || (alloc_m559918035_locked || g1 == 0)' - type: assertion - format: C -- entry_type: flow_insensitive_invariant - flow_insensitive_invariant: - string: '! multithreaded || (0 <= g2 && g2 <= 1)' - type: assertion - format: C -- entry_type: flow_insensitive_invariant - flow_insensitive_invariant: - string: '! multithreaded || (0 <= g1 && g1 <= 1)' - type: assertion - format: C diff --git a/tests/regression/56-witness/compare-ghost-alloc.sh b/tests/regression/56-witness/compare-ghost-alloc.sh deleted file mode 100755 index 4574dfe9f3..0000000000 --- a/tests/regression/56-witness/compare-ghost-alloc.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -# Check if the correct number of arguments are provided -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit 1 -fi - -file1="$1" -file2="$2" - -# Function to extract the first and second occurrences of the pattern -extract_variables() { - grep -o -m 2 'alloc_m[0-9]\+_locked' "$1" -} - -# Extract variables from both files -var1_file1=$(extract_variables "$file1" | sed -n '1p') -var2_file1=$(extract_variables "$file1" | sed -n '2p') -var1_file2=$(extract_variables "$file2" | sed -n '1p') -var2_file2=$(extract_variables "$file2" | sed -n '2p') - -# Check if the variables were found -if [ -z "$var1_file1" ] || [ -z "$var2_file1" ] || [ -z "$var1_file2" ] || [ -z "$var2_file2" ]; then - echo "Error: Could not find the required pattern in one or both files." - exit 1 -fi - -# Create temporary files -temp1=$(mktemp) -temp2=$(mktemp) - -# Rename variables in both files -sed -e "s/\b$var1_file1\b/TEMP_VAR1/g" -e "s/\b$var2_file1\b/TEMP_VAR2/g" "$file1" > "$temp1" -sed -e "s/\b$var1_file2\b/TEMP_VAR1/g" -e "s/\b$var2_file2\b/TEMP_VAR2/g" "$file2" > "$temp2" - -# Compare the modified files -if diff -q "$temp1" "$temp2" > /dev/null; then - echo "The files are the same after renaming the variables." -else - echo "The files are different after renaming the variables." -fi - -# Clean up temporary files -rm "$temp1" "$temp2" diff --git a/tests/regression/56-witness/dune b/tests/regression/56-witness/dune index 4ce326af28..3ef3cb78f5 100644 --- a/tests/regression/56-witness/dune +++ b/tests/regression/56-witness/dune @@ -25,7 +25,7 @@ (run %{update_suite} apron-unassume-set-tokens -q))))) (cram - (deps (glob_files *.c) (glob_files ??-*.yml) (glob_files compare-ghost-alloc.sh))) + (deps (glob_files *.c) (glob_files ??-*.yml) (glob_files strip-ghost-alloc.sh))) (cram (applies_to 54-witness-lifter-abortUnless) diff --git a/tests/regression/56-witness/strip-ghost-alloc.sh b/tests/regression/56-witness/strip-ghost-alloc.sh new file mode 100755 index 0000000000..76b0460492 --- /dev/null +++ b/tests/regression/56-witness/strip-ghost-alloc.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Check if the correct number of arguments are provided +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +file="$1" + +# Function to extract the first and second occurrences of the pattern +extract_variables() { + grep -o -m 2 'alloc_m[0-9]\+_locked' "$1" +} + +# Extract variables from the file +var1=$(extract_variables "$file" | sed -n '1p') +var2=$(extract_variables "$file" | sed -n '2p') + +# Check if both variables were found +if [ -z "$var1" ] || [ -z "$var2" ]; then + echo "Error: Could not find two occurrences of the pattern 'alloc_m[0-9]+_locked' in the file." + exit 1 +fi + +# Rename variables and print the modified file to stdout +sed -e "s/\b$var1\b/ALLOC_VAR1_LOCKED/g" -e "s/\b$var2\b/ALLOC_VAR2_LOCKED/g" "$file" From fcdcf13159b70db41417f471bdcc75aedef9fb68 Mon Sep 17 00:00:00 2001 From: Michael Schwarz Date: Wed, 5 Mar 2025 14:27:40 +0100 Subject: [PATCH 3/3] Get rid of GNU-ism `\b` --- tests/regression/56-witness/strip-ghost-alloc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/regression/56-witness/strip-ghost-alloc.sh b/tests/regression/56-witness/strip-ghost-alloc.sh index 76b0460492..c743e8fc8a 100755 --- a/tests/regression/56-witness/strip-ghost-alloc.sh +++ b/tests/regression/56-witness/strip-ghost-alloc.sh @@ -24,4 +24,4 @@ if [ -z "$var1" ] || [ -z "$var2" ]; then fi # Rename variables and print the modified file to stdout -sed -e "s/\b$var1\b/ALLOC_VAR1_LOCKED/g" -e "s/\b$var2\b/ALLOC_VAR2_LOCKED/g" "$file" +sed -e "s/$var1/ALLOC_VAR1_LOCKED/g" -e "s/$var2/ALLOC_VAR2_LOCKED/g" "$file"