tests: detect slowness through deterministic statistics

Cargo.toml

@@ -27,11 +27,6 @@ auto-country = ["dep:country-boundaries"]
 auto-timezone = ["dep:chrono-tz", "dep:tzf-rs"]
 log = ["opening-hours-syntax/log", "dep:log"]
 
-# Disable timeout behavior for performance tests. This is useful when tests
-# need to be in slow environments or with high overhead, for example when
-# measuring coverage.
-disable-test-timeouts = []
-
 [dependencies]
 chrono = "0.4"
 compact-calendar = { path = "compact-calendar", version = "1.1.0" }

fuzz/fuzz_targets/fuzz_oh.rs

@@ -1,6 +1,6 @@
 #![no_main]
+use fuzz::{run_fuzz_oh, Data};
 use libfuzzer_sys::{fuzz_target, Corpus};
-use opening_hours::fuzzing::{run_fuzz_oh, Data};
 
 fuzz_target!(|data: Data| -> Corpus {
     if run_fuzz_oh(data) {

opening-hours/src/opening_hours.rs

@@ -124,6 +124,9 @@ impl<L: Localize> OpeningHours<L> {
 
     /// Get the schedule at a given day.
     pub fn schedule_at(&self, date: NaiveDate) -> Schedule {
+        #[cfg(test)]
+        crate::tests::stats::notify::generated_schedule();
+
         if !(DATE_START.date()..DATE_END.date()).contains(&date) {
             return Schedule::default();
         }

opening-hours/src/tests/mod.rs

@@ -1,3 +1,5 @@
+pub(crate) mod stats;
+
 mod country;
 mod holiday_selector;
 mod issues;
@@ -13,78 +15,13 @@ mod week_selector;
 mod weekday_selector;
 mod year_selector;
 
-use criterion::black_box;
-use std::sync::{Arc, Condvar, Mutex};
-use std::thread;
-use std::time::{Duration, Instant};
-
 fn sample() -> impl Iterator<Item = &'static str> {
     include_str!("data/sample.txt")
         .lines()
         .map(str::trim)
         .filter(|line| !line.is_empty() && !line.starts_with('#'))
 }
 
-/// Wraps input function but panics if it runs longer than specified timeout.
-pub fn exec_with_timeout<R: Send + 'static>(
-    timeout: Duration,
-    f: impl FnOnce() -> R + Send + 'static,
-) -> R {
-    if cfg!(feature = "disable-test-timeouts") {
-        return f();
-    }
-
-    let result = Arc::new(Mutex::new((None, false)));
-    let finished = Arc::new(Condvar::new());
-
-    struct DetectPanic<R>(Arc<Mutex<(Option<R>, bool)>>);
-
-    impl<R> Drop for DetectPanic<R> {
-        fn drop(&mut self) {
-            if thread::panicking() {
-                self.0.lock().expect("failed to write panic").1 = true;
-            }
-        }
-    }
-
-    let _runner = {
-        let f = black_box(f);
-        let result = result.clone();
-        let finished = finished.clone();
-
-        thread::spawn(move || {
-            let _panic_guard = DetectPanic(result.clone());
-
-            let elapsed = {
-                let start = Instant::now();
-                let res = f();
-                (start.elapsed(), res)
-            };
-
-            result.lock().expect("failed to write result").0 = Some(elapsed);
-            finished.notify_all();
-        })
-    };
-
-    let (mut result, _) = finished
-        .wait_timeout(result.lock().expect("failed to fetch result"), timeout)
-        .expect("poisoned lock");
-
-    if result.1 {
-        panic!("exec stopped due to panic");
-    }
-
-    let Some((elapsed, res)) = result.0.take() else {
-        panic!("exec stopped due to {timeout:?} timeout");
-    };
-
-    if elapsed > timeout {
-        panic!("exec ran for {elapsed:?}");
-    }
-
-    res
-}
-
 #[macro_export]
 macro_rules! date {
     ( $date: expr ) => {{

opening-hours/src/tests/regression.rs

@@ -1,10 +1,8 @@
-use std::time::Duration;
-
 use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
 use opening_hours_syntax::error::Error;
 use opening_hours_syntax::rules::RuleKind::*;
 
-use crate::tests::exec_with_timeout;
+use crate::tests::stats::TestStats;
 use crate::{datetime, schedule_at, OpeningHours};
 
 #[test]
@@ -127,26 +125,24 @@ fn s009_pj_no_open_before_separator() {
 }
 
 #[test]
-fn s010_pj_slow_after_24_7() -> Result<(), Error> {
-    exec_with_timeout(Duration::from_millis(100), || {
-        assert!("24/7 open ; 2021Jan-Feb off"
-            .parse::<OpeningHours>()?
+fn s010_pj_slow_after_24_7() {
+    let stats = TestStats::watch(|| {
+        assert!(OpeningHours::parse("24/7 open ; 2021Jan-Feb off")
+            .unwrap()
             .next_change(datetime!("2021-07-09 19:30"))
             .is_none());
+    });
 
-        Ok::<(), Error>(())
-    })?;
+    assert!(stats.count_generated_schedules < 10);
 
-    exec_with_timeout(Duration::from_millis(100), || {
-        assert!("24/7 open ; 2021 Jan 01-Feb 10 off"
-            .parse::<OpeningHours>()?
+    let stats = TestStats::watch(|| {
+        assert!(OpeningHours::parse("24/7 open ; 2021 Jan 01-Feb 10 off")
+            .unwrap()
             .next_change(datetime!("2021-07-09 19:30"))
             .is_none());
+    });
 
-        Ok::<(), Error>(())
-    })?;
-
-    Ok(())
+    assert!(stats.count_generated_schedules < 10);
 }
 
 #[test]
@@ -159,38 +155,37 @@ fn s011_fuzz_extreme_year() -> Result<(), Error> {
     );
 
     assert!(oh.is_closed(dt));
-    assert!(oh.next_change(dt).is_none());
+    assert_eq!(oh.next_change(dt).unwrap(), datetime!("2000-01-01 00:00"));
     Ok(())
 }
 
 #[test]
-fn s012_fuzz_slow_sh() -> Result<(), Error> {
-    exec_with_timeout(Duration::from_millis(100), || {
-        assert!("SH"
-            .parse::<OpeningHours>()?
+fn s012_fuzz_slow_sh() {
+    let stats = TestStats::watch(|| {
+        assert!(OpeningHours::parse("SH")
+            .unwrap()
             .next_change(datetime!("2020-01-01 00:00"))
             .is_none());
+    });
 
-        Ok(())
-    })
+    assert!(stats.count_generated_schedules < 10);
 }
 
 #[test]
-fn s013_fuzz_slow_weeknum() -> Result<(), Error> {
-    exec_with_timeout(Duration::from_millis(200), || {
-        assert!("Novweek09"
-            .parse::<OpeningHours>()?
+fn s013_fuzz_slow_weeknum() {
+    let stats = TestStats::watch(|| {
+        assert!(OpeningHours::parse("Novweek09")
+            .unwrap()
             .next_change(datetime!("2020-01-01 00:00"))
             .is_none());
+    });
 
-        Ok(())
-    })
+    assert!(stats.count_generated_schedules < 50_000);
 }
 
 #[test]
 fn s014_fuzz_feb30_before_leap_year() -> Result<(), Error> {
-    "Feb30"
-        .parse::<OpeningHours>()?
+    OpeningHours::parse("Feb30")?
         .next_change(datetime!("4419-03-01 00:00"))
         .unwrap();
 
@@ -224,26 +219,26 @@ fn s016_fuzz_week01_sh() -> Result<(), Error> {
 }
 
 #[test]
-fn s017_fuzz_open_range_timeout() -> Result<(), Error> {
-    exec_with_timeout(Duration::from_millis(100), || {
+fn s017_fuzz_open_range_timeout() {
+    let stats = TestStats::watch(|| {
         assert_eq!(
-            "May2+"
-                .parse::<OpeningHours>()?
+            OpeningHours::parse("May2+")
+                .unwrap()
                 .next_change(datetime!("2020-01-01 12:00"))
                 .unwrap(),
             datetime!("2020-05-02 00:00")
         );
 
         assert_eq!(
-            "May2+"
-                .parse::<OpeningHours>()?
+            OpeningHours::parse("May2+")
+                .unwrap()
                 .next_change(datetime!("2020-05-15 12:00"))
                 .unwrap(),
             datetime!("2021-01-01 00:00")
         );
+    });
 
-        Ok(())
-    })
+    assert!(stats.count_generated_schedules < 10);
 }
 
 #[cfg(feature = "auto-country")]

opening-hours/src/tests/rules.rs

@@ -1,6 +1,4 @@
-use std::time::Duration;
-
-use crate::tests::exec_with_timeout;
+use crate::tests::stats::TestStats;
 use opening_hours_syntax::error::Error;
 use opening_hours_syntax::rules::RuleKind::*;
 
@@ -113,12 +111,13 @@ fn comments() -> Result<(), Error> {
 }
 
 #[test]
-fn explicit_closed_slow() -> Result<(), Error> {
-    exec_with_timeout(Duration::from_millis(100), || {
-        assert!(OpeningHours::parse("Feb Fr off")?
+fn explicit_closed_slow() {
+    let stats = TestStats::watch(|| {
+        assert!(OpeningHours::parse("Feb Fr off")
+            .unwrap()
             .next_change(datetime!("2021-07-09 19:30"))
             .is_none());
+    });
 
-        Ok::<(), Error>(())
-    })
+    assert!(stats.count_generated_schedules < 10);
 }

opening-hours/src/tests/stats.rs

@@ -0,0 +1,27 @@
+use std::cell::RefCell;
+
+thread_local! {
+    static THREAD_TEST_STATS: RefCell<TestStats> = RefCell::default();
+    static THREAD_LOCK: RefCell<()> = RefCell::default();
+}
+
+#[derive(Default)]
+pub(crate) struct TestStats {
+    pub(crate) count_generated_schedules: u64,
+}
+
+impl TestStats {
+    pub(crate) fn watch<F: FnOnce()>(func: F) -> Self {
+        THREAD_LOCK.with_borrow_mut(|_| {
+            THREAD_TEST_STATS.take();
+            func();
+            THREAD_TEST_STATS.take()
+        })
+    }
+}
+
+pub(crate) mod notify {
+    pub(crate) fn generated_schedule() {
+        super::THREAD_TEST_STATS.with_borrow_mut(|s| s.count_generated_schedules += 1)
+    }
+}