From a5b74f96080dc083e898f6a8e29e732e01c7b238 Mon Sep 17 00:00:00 2001
From: Benjamin Gaidioz <ben@raw-labs.com>
Date: Wed, 17 Jan 2024 08:48:13 +0100
Subject: [PATCH 01/11] Fixed RD-10460: LocalInferrerService is not stopped

---
 .../scala/raw/client/api/CompilerServiceProvider.scala    | 7 ++++++-
 .../client/rql2/truffle/Rql2TruffleCompilerService.scala  | 8 +++++++-
 .../rql2/api/Rql2CompilerServiceTestContext.scala         | 1 +
 .../scala/raw/inferrer/local/LocalInferrerService.scala   | 2 ++
 utils/src/main/scala/raw/utils/RawService.scala           | 6 ++++--
 5 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/client/src/main/scala/raw/client/api/CompilerServiceProvider.scala b/client/src/main/scala/raw/client/api/CompilerServiceProvider.scala
index 9284dde54..1e8ba2224 100644
--- a/client/src/main/scala/raw/client/api/CompilerServiceProvider.scala
+++ b/client/src/main/scala/raw/client/api/CompilerServiceProvider.scala
@@ -60,7 +60,12 @@ object CompilerServiceProvider {
   private[raw] def set(language: Set[String], instance: CompilerService): Unit = {
     instanceLock.synchronized {
       if (instance == null) {
-        instanceMap.remove((language, None))
+        // stop and remove entries that match the `language`, regardless the class loader.
+        instanceMap.filterKeys(_._1 == language).foreach {
+          case (key, compiler) =>
+            compiler.stop()
+            instanceMap.remove(key)
+        }
       } else {
         instanceMap.put((language, None), instance)
       }
diff --git a/snapi-client/src/main/scala/raw/client/rql2/truffle/Rql2TruffleCompilerService.scala b/snapi-client/src/main/scala/raw/client/rql2/truffle/Rql2TruffleCompilerService.scala
index 2fd7d420a..7451d7ce4 100644
--- a/snapi-client/src/main/scala/raw/client/rql2/truffle/Rql2TruffleCompilerService.scala
+++ b/snapi-client/src/main/scala/raw/client/rql2/truffle/Rql2TruffleCompilerService.scala
@@ -66,7 +66,13 @@ class Rql2TruffleCompilerService(maybeClassLoader: Option[ClassLoader] = None)(
     val inferrer = InferrerServiceProvider(maybeClassLoader)
 
     // Initialize compiler context
-    new CompilerContext(language, user, inferrer, sourceContext, maybeClassLoader)
+    try {
+      new CompilerContext(language, user, inferrer, sourceContext, maybeClassLoader)
+    } catch {
+      case NonFatal(ex) =>
+        inferrer.stop()
+        throw ex
+    }
   }
 
   private def getProgramContext(user: AuthenticatedUser, environment: ProgramEnvironment): ProgramContext = {
diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/api/Rql2CompilerServiceTestContext.scala b/snapi-client/src/test/scala/raw/compiler/rql2/api/Rql2CompilerServiceTestContext.scala
index 75c4d491d..531256aa0 100644
--- a/snapi-client/src/test/scala/raw/compiler/rql2/api/Rql2CompilerServiceTestContext.scala
+++ b/snapi-client/src/test/scala/raw/compiler/rql2/api/Rql2CompilerServiceTestContext.scala
@@ -36,6 +36,7 @@ trait Rql2CompilerServiceTestContext extends BeforeAndAfterAll {
   }
 
   override def afterAll(): Unit = {
+    instance.stop()
     setCompilerService(null)
     super.afterAll()
   }
diff --git a/snapi-frontend/src/main/scala/raw/inferrer/local/LocalInferrerService.scala b/snapi-frontend/src/main/scala/raw/inferrer/local/LocalInferrerService.scala
index 578d6bef8..cf0e67edf 100644
--- a/snapi-frontend/src/main/scala/raw/inferrer/local/LocalInferrerService.scala
+++ b/snapi-frontend/src/main/scala/raw/inferrer/local/LocalInferrerService.scala
@@ -13,6 +13,7 @@
 package raw.inferrer.local
 
 import com.typesafe.scalalogging.StrictLogging
+import org.apache.commons.lang3.exception.ExceptionUtils
 import org.bitbucket.inkytonik.kiama.output.PrettyPrinter
 import raw.utils.RawException
 import raw.inferrer.api._
@@ -37,6 +38,7 @@ class LocalInferrerService(implicit sourceContext: SourceContext)
     with PrettyPrinter
     with StrictLogging {
 
+  logger.info(s"Creating inferrer service: ${this}\n${ExceptionUtils.getStackTrace(new Throwable)}")
   private val settings = sourceContext.settings
 
   private val textInferrer = new TextInferrer
diff --git a/utils/src/main/scala/raw/utils/RawService.scala b/utils/src/main/scala/raw/utils/RawService.scala
index 54b4c9ed6..a1f0c4b54 100644
--- a/utils/src/main/scala/raw/utils/RawService.scala
+++ b/utils/src/main/scala/raw/utils/RawService.scala
@@ -44,7 +44,7 @@ trait RawService extends StrictLogging {
   import RawService._
   protected val stopped = new AtomicBoolean(false)
 
-  logger.debug(s"Adding service: $this")
+  logger.debug(s"Adding service: $this. Caller:\n${Thread.currentThread().getStackTrace.take(10).mkString("\t\n")}")
   services.add(this)
 
   /**
@@ -63,7 +63,9 @@ trait RawService extends StrictLogging {
       } finally {
         val removed = services.remove(this)
         if (removed) {
-          logger.debug(s"Stopping service: $this")
+          logger.debug(
+            s"Stopping service: $this. Caller:\n${Thread.currentThread().getStackTrace.take(10).mkString("\t\n")}"
+          )
         } else {
           logger.warn(s"Service was not found on active service list: $this")
         }

From 384770fc84ff0b9c8632e7767ec6edefe09c82e9 Mon Sep 17 00:00:00 2001
From: Benjamin Gaidioz <ben@raw-labs.com>
Date: Thu, 8 Feb 2024 09:25:44 +0100
Subject: [PATCH 02/11] drop Language caches in
 Rql2TruffleCompilerServiceTestContext

---
 .../truffle/Rql2TruffleCompilerServiceTestContext.scala     | 6 ++++++
 .../compiler/rql2/truffle/TruffleCompilerTestContext.scala  | 5 -----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/Rql2TruffleCompilerServiceTestContext.scala b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/Rql2TruffleCompilerServiceTestContext.scala
index 810dad5a3..c64684ddf 100644
--- a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/Rql2TruffleCompilerServiceTestContext.scala
+++ b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/Rql2TruffleCompilerServiceTestContext.scala
@@ -14,6 +14,7 @@ package raw.compiler.rql2.truffle
 
 import raw.client.rql2.truffle.Rql2TruffleCompilerService
 import raw.compiler.rql2.api.Rql2CompilerServiceTestContext
+import raw.runtime.truffle.RawLanguage
 import raw.utils.{RawTestSuite, RawUtils, SettingsTestContext}
 
 trait Rql2TruffleCompilerServiceTestContext extends Rql2CompilerServiceTestContext {
@@ -23,6 +24,9 @@ trait Rql2TruffleCompilerServiceTestContext extends Rql2CompilerServiceTestConte
 
   override def beforeAll(): Unit = {
     super.beforeAll()
+    // Forcibly drop all language caches before each test suite.
+    RawLanguage.dropCaches()
+
     property("raw.compiler.impl", "rql2-truffle")
 
     rql2TruffleCompilerService = new Rql2TruffleCompilerService
@@ -34,6 +38,8 @@ trait Rql2TruffleCompilerServiceTestContext extends Rql2CompilerServiceTestConte
       RawUtils.withSuppressNonFatalException(rql2TruffleCompilerService.stop())
       rql2TruffleCompilerService = null
     }
+    // Forcibly drop all language caches after each test suite.
+    RawLanguage.dropCaches()
     super.afterAll()
   }
 
diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/TruffleCompilerTestContext.scala b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/TruffleCompilerTestContext.scala
index 0f45d0ed4..d168eadac 100644
--- a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/TruffleCompilerTestContext.scala
+++ b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/TruffleCompilerTestContext.scala
@@ -14,7 +14,6 @@ package raw.compiler.rql2.truffle
 
 import raw.compiler.rql2.api.Rql2OutputTestContext
 import raw.compiler.rql2.tests.CompilerTestContext
-import raw.runtime.truffle.RawLanguage
 
 class TruffleCompilerTestContext
     extends CompilerTestContext
@@ -23,13 +22,9 @@ class TruffleCompilerTestContext
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    // Forcibly drop all language caches before each test suite.
-    RawLanguage.dropCaches()
   }
 
   override def afterAll(): Unit = {
-    // Forcibly drop all language caches after each test suite.
-    RawLanguage.dropCaches()
     super.afterAll()
   }
 

From e8756e14c8dddd6fae114f02826547a8aeae0238 Mon Sep 17 00:00:00 2001
From: Benjamin Gaidioz <ben@raw-labs.com>
Date: Thu, 8 Feb 2024 09:29:16 +0100
Subject: [PATCH 03/11] added set to CredentialsServiceProvider

---
 .../creds/api/CredentialsServiceProvider.scala    | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/snapi-frontend/src/main/scala/raw/creds/api/CredentialsServiceProvider.scala b/snapi-frontend/src/main/scala/raw/creds/api/CredentialsServiceProvider.scala
index 0c18b41e1..dd60bbaed 100644
--- a/snapi-frontend/src/main/scala/raw/creds/api/CredentialsServiceProvider.scala
+++ b/snapi-frontend/src/main/scala/raw/creds/api/CredentialsServiceProvider.scala
@@ -20,6 +20,8 @@ import scala.collection.JavaConverters._
 object CredentialsServiceProvider {
 
   private val CREDS_IMPL = "raw.creds.impl"
+  private val instanceLock = new Object
+  private var instance: Option[CredentialsService] = None
 
   def apply(maybeClassLoader: Option[ClassLoader] = None)(implicit settings: RawSettings): CredentialsService = {
     maybeClassLoader match {
@@ -29,7 +31,12 @@ object CredentialsServiceProvider {
   }
 
   def apply()(implicit settings: RawSettings): CredentialsService = {
-    build()
+    instanceLock.synchronized {
+      instance match {
+        case Some(service) => service
+        case None => build()
+      }
+    }
   }
 
   def apply(classLoader: ClassLoader)(implicit settings: RawSettings): CredentialsService = {
@@ -56,4 +63,10 @@ object CredentialsServiceProvider {
     }
   }
 
+  private[raw] def set(credsService: CredentialsService): Unit = {
+    instanceLock.synchronized {
+      instance = Some(credsService)
+    }
+  }
+
 }

From f39feed23accdde840172521de74eb8b88352757 Mon Sep 17 00:00:00 2001
From: Benjamin Gaidioz <ben@raw-labs.com>
Date: Thu, 8 Feb 2024 11:26:48 +0100
Subject: [PATCH 04/11] attempt to add set to CredentialsServiceProvider

---
 .../raw/creds/api/CredentialsServiceProvider.scala  | 13 +++++++++++--
 .../raw/creds/api/CredentialsTestContext.scala      |  1 +
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/snapi-frontend/src/main/scala/raw/creds/api/CredentialsServiceProvider.scala b/snapi-frontend/src/main/scala/raw/creds/api/CredentialsServiceProvider.scala
index dd60bbaed..b25080d70 100644
--- a/snapi-frontend/src/main/scala/raw/creds/api/CredentialsServiceProvider.scala
+++ b/snapi-frontend/src/main/scala/raw/creds/api/CredentialsServiceProvider.scala
@@ -40,7 +40,12 @@ object CredentialsServiceProvider {
   }
 
   def apply(classLoader: ClassLoader)(implicit settings: RawSettings): CredentialsService = {
-    build(Some(classLoader))
+    instanceLock.synchronized {
+      instance match {
+        case Some(service) => service
+        case None => build(Some(classLoader))
+      }
+    }
   }
 
   private def build(
@@ -65,7 +70,11 @@ object CredentialsServiceProvider {
 
   private[raw] def set(credsService: CredentialsService): Unit = {
     instanceLock.synchronized {
-      instance = Some(credsService)
+      if (credsService == null) {
+        instance = None
+      } else {
+        instance = Some(credsService)
+      }
     }
   }
 
diff --git a/snapi-frontend/src/test/scala/raw/creds/api/CredentialsTestContext.scala b/snapi-frontend/src/test/scala/raw/creds/api/CredentialsTestContext.scala
index 190824950..f7bff2e00 100644
--- a/snapi-frontend/src/test/scala/raw/creds/api/CredentialsTestContext.scala
+++ b/snapi-frontend/src/test/scala/raw/creds/api/CredentialsTestContext.scala
@@ -53,6 +53,7 @@ trait CredentialsTestContext extends BeforeAndAfterAll {
 
   def setCredentials(credentials: CredentialsService): Unit = {
     instance = credentials
+    CredentialsServiceProvider.set(credentials)
   }
 
   override def beforeAll(): Unit = {

From 8ce533a47c14631f03d3097e5fbaf3a8afcb890a Mon Sep 17 00:00:00 2001
From: Benjamin Gaidioz <ben@raw-labs.com>
Date: Wed, 14 Feb 2024 15:43:39 +0100
Subject: [PATCH 05/11] wip

---
 snapi-client/src/main/resources/reference.conf                | 3 ++-
 .../src/test/scala/raw/creds/api/CredentialsTestContext.scala | 4 ----
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/snapi-client/src/main/resources/reference.conf b/snapi-client/src/main/resources/reference.conf
index c8b6d3f31..bf299430e 100644
--- a/snapi-client/src/main/resources/reference.conf
+++ b/snapi-client/src/main/resources/reference.conf
@@ -1,2 +1,3 @@
 // Default to local credentials service.
-raw.creds.impl = "local"
\ No newline at end of file
+raw.creds.impl = "local"
+raw.creds.impl = ${?CREDS_IMPL}
\ No newline at end of file
diff --git a/snapi-frontend/src/test/scala/raw/creds/api/CredentialsTestContext.scala b/snapi-frontend/src/test/scala/raw/creds/api/CredentialsTestContext.scala
index f7bff2e00..06dd0c1dc 100644
--- a/snapi-frontend/src/test/scala/raw/creds/api/CredentialsTestContext.scala
+++ b/snapi-frontend/src/test/scala/raw/creds/api/CredentialsTestContext.scala
@@ -65,10 +65,6 @@ trait CredentialsTestContext extends BeforeAndAfterAll {
     secrets.foreach { case (user, secret) => credentials.registerSecret(user, secret) }
     // Set the system properties for the credentials so that this is overriding what RawLanguage
     // runs with. This way, the credential server booted by the test framework is used.
-    for (property <- Seq("raw.creds.impl", "raw.creds.client.server-address")) {
-      System.clearProperty(property)
-      settings.getStringOpt(property).foreach(v => System.setProperty(property, v))
-    }
   }
 
   override def afterAll(): Unit = {

From 26e08f484f43efc0b5b7330256a15747924c953c Mon Sep 17 00:00:00 2001
From: Benjamin Gaidioz <ben@raw-labs.com>
Date: Thu, 15 Feb 2024 17:39:41 +0100
Subject: [PATCH 06/11] wip

---
 .../api/CredentialsServiceProvider.scala      | 26 ++-----------------
 .../inferrer/local/LocalInferrerService.scala |  2 --
 .../creds/api/CredentialsTestContext.scala    |  3 ---
 3 files changed, 2 insertions(+), 29 deletions(-)

diff --git a/snapi-frontend/src/main/scala/raw/creds/api/CredentialsServiceProvider.scala b/snapi-frontend/src/main/scala/raw/creds/api/CredentialsServiceProvider.scala
index b25080d70..0c18b41e1 100644
--- a/snapi-frontend/src/main/scala/raw/creds/api/CredentialsServiceProvider.scala
+++ b/snapi-frontend/src/main/scala/raw/creds/api/CredentialsServiceProvider.scala
@@ -20,8 +20,6 @@ import scala.collection.JavaConverters._
 object CredentialsServiceProvider {
 
   private val CREDS_IMPL = "raw.creds.impl"
-  private val instanceLock = new Object
-  private var instance: Option[CredentialsService] = None
 
   def apply(maybeClassLoader: Option[ClassLoader] = None)(implicit settings: RawSettings): CredentialsService = {
     maybeClassLoader match {
@@ -31,21 +29,11 @@ object CredentialsServiceProvider {
   }
 
   def apply()(implicit settings: RawSettings): CredentialsService = {
-    instanceLock.synchronized {
-      instance match {
-        case Some(service) => service
-        case None => build()
-      }
-    }
+    build()
   }
 
   def apply(classLoader: ClassLoader)(implicit settings: RawSettings): CredentialsService = {
-    instanceLock.synchronized {
-      instance match {
-        case Some(service) => service
-        case None => build(Some(classLoader))
-      }
-    }
+    build(Some(classLoader))
   }
 
   private def build(
@@ -68,14 +56,4 @@ object CredentialsServiceProvider {
     }
   }
 
-  private[raw] def set(credsService: CredentialsService): Unit = {
-    instanceLock.synchronized {
-      if (credsService == null) {
-        instance = None
-      } else {
-        instance = Some(credsService)
-      }
-    }
-  }
-
 }
diff --git a/snapi-frontend/src/main/scala/raw/inferrer/local/LocalInferrerService.scala b/snapi-frontend/src/main/scala/raw/inferrer/local/LocalInferrerService.scala
index cf0e67edf..578d6bef8 100644
--- a/snapi-frontend/src/main/scala/raw/inferrer/local/LocalInferrerService.scala
+++ b/snapi-frontend/src/main/scala/raw/inferrer/local/LocalInferrerService.scala
@@ -13,7 +13,6 @@
 package raw.inferrer.local
 
 import com.typesafe.scalalogging.StrictLogging
-import org.apache.commons.lang3.exception.ExceptionUtils
 import org.bitbucket.inkytonik.kiama.output.PrettyPrinter
 import raw.utils.RawException
 import raw.inferrer.api._
@@ -38,7 +37,6 @@ class LocalInferrerService(implicit sourceContext: SourceContext)
     with PrettyPrinter
     with StrictLogging {
 
-  logger.info(s"Creating inferrer service: ${this}\n${ExceptionUtils.getStackTrace(new Throwable)}")
   private val settings = sourceContext.settings
 
   private val textInferrer = new TextInferrer
diff --git a/snapi-frontend/src/test/scala/raw/creds/api/CredentialsTestContext.scala b/snapi-frontend/src/test/scala/raw/creds/api/CredentialsTestContext.scala
index 06dd0c1dc..8422cc498 100644
--- a/snapi-frontend/src/test/scala/raw/creds/api/CredentialsTestContext.scala
+++ b/snapi-frontend/src/test/scala/raw/creds/api/CredentialsTestContext.scala
@@ -53,7 +53,6 @@ trait CredentialsTestContext extends BeforeAndAfterAll {
 
   def setCredentials(credentials: CredentialsService): Unit = {
     instance = credentials
-    CredentialsServiceProvider.set(credentials)
   }
 
   override def beforeAll(): Unit = {
@@ -63,8 +62,6 @@ trait CredentialsTestContext extends BeforeAndAfterAll {
     newHttpCreds.foreach { case (user, (name, cred)) => credentials.registerNewHttpCredential(user, name, cred) }
     dropboxTokens.foreach { case (user, token) => credentials.registerDropboxToken(user, token) }
     secrets.foreach { case (user, secret) => credentials.registerSecret(user, secret) }
-    // Set the system properties for the credentials so that this is overriding what RawLanguage
-    // runs with. This way, the credential server booted by the test framework is used.
   }
 
   override def afterAll(): Unit = {

From 971b829d693d81c768b3a4e5506281867e45bc5e Mon Sep 17 00:00:00 2001
From: Benjamin Gaidioz <ben@raw-labs.com>
Date: Mon, 19 Feb 2024 09:05:50 +0100
Subject: [PATCH 07/11] wip

---
 utils/src/main/scala/raw/utils/RawService.scala   | 6 ++----
 utils/src/test/scala/raw/utils/RawTestSuite.scala | 8 ++++++--
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/utils/src/main/scala/raw/utils/RawService.scala b/utils/src/main/scala/raw/utils/RawService.scala
index a1f0c4b54..54b4c9ed6 100644
--- a/utils/src/main/scala/raw/utils/RawService.scala
+++ b/utils/src/main/scala/raw/utils/RawService.scala
@@ -44,7 +44,7 @@ trait RawService extends StrictLogging {
   import RawService._
   protected val stopped = new AtomicBoolean(false)
 
-  logger.debug(s"Adding service: $this. Caller:\n${Thread.currentThread().getStackTrace.take(10).mkString("\t\n")}")
+  logger.debug(s"Adding service: $this")
   services.add(this)
 
   /**
@@ -63,9 +63,7 @@ trait RawService extends StrictLogging {
       } finally {
         val removed = services.remove(this)
         if (removed) {
-          logger.debug(
-            s"Stopping service: $this. Caller:\n${Thread.currentThread().getStackTrace.take(10).mkString("\t\n")}"
-          )
+          logger.debug(s"Stopping service: $this")
         } else {
           logger.warn(s"Service was not found on active service list: $this")
         }
diff --git a/utils/src/test/scala/raw/utils/RawTestSuite.scala b/utils/src/test/scala/raw/utils/RawTestSuite.scala
index c585d309e..29c220516 100644
--- a/utils/src/test/scala/raw/utils/RawTestSuite.scala
+++ b/utils/src/test/scala/raw/utils/RawTestSuite.scala
@@ -44,20 +44,24 @@ trait RawTestSuite extends FixtureAnyFunSuite with BeforeAndAfterAll with Strict
     test(TestData(test.name))
   }
 
+  var services: Seq[RawService] = _
   override def beforeAll(): Unit = {
     // If a previous test suite crashed, clean its leftovers anyway.
     RawService.stopAll()
+    services = RawService.services.asScala.toSeq
+    logger.debug(s"Services up at startup: ${services.map(_.toString).mkString(", ")}")
     super.beforeAll()
   }
 
   override def afterAll(): Unit = {
     logger.info("Checking if all services have stopped")
     var attempts = 10
-    while (!RawService.isStopped() && attempts > 0) {
+    while (RawService.services.size != services.size && attempts > 0) {
+      logger.debug(s"Services up at teardown: ${RawService.services.asScala.map(_.toString).mkString(", ")}")
       attempts -= 1
       logger.debug(s"Waiting for services to terminate gracefully. Attempts left: $attempts")
       Thread.sleep(1000)
     }
-    assert(RawService.isStopped(), s"Not all services stopped properly. Still running: ${RawService.services}")
+    assert(RawService.services.size == services.size, s"Not all services stopped properly. Still running: ${RawService.services}")
   }
 }

From d28d3530d45a697292954cca91bdbbe2633c8552 Mon Sep 17 00:00:00 2001
From: Benjamin Gaidioz <ben@raw-labs.com>
Date: Tue, 20 Feb 2024 13:44:36 +0100
Subject: [PATCH 08/11] wip

---
 utils/src/test/scala/raw/utils/RawTestSuite.scala | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/utils/src/test/scala/raw/utils/RawTestSuite.scala b/utils/src/test/scala/raw/utils/RawTestSuite.scala
index 29c220516..e2864f4e8 100644
--- a/utils/src/test/scala/raw/utils/RawTestSuite.scala
+++ b/utils/src/test/scala/raw/utils/RawTestSuite.scala
@@ -46,22 +46,21 @@ trait RawTestSuite extends FixtureAnyFunSuite with BeforeAndAfterAll with Strict
 
   var services: Seq[RawService] = _
   override def beforeAll(): Unit = {
-    // If a previous test suite crashed, clean its leftovers anyway.
-    RawService.stopAll()
-    services = RawService.services.asScala.toSeq
-    logger.debug(s"Services up at startup: ${services.map(_.toString).mkString(", ")}")
     super.beforeAll()
   }
 
   override def afterAll(): Unit = {
     logger.info("Checking if all services have stopped")
     var attempts = 10
-    while (RawService.services.size != services.size && attempts > 0) {
+    while (RawService.services.size() > 1 && attempts > 0) {
       logger.debug(s"Services up at teardown: ${RawService.services.asScala.map(_.toString).mkString(", ")}")
       attempts -= 1
       logger.debug(s"Waiting for services to terminate gracefully. Attempts left: $attempts")
       Thread.sleep(1000)
     }
-    assert(RawService.services.size == services.size, s"Not all services stopped properly. Still running: ${RawService.services}")
+    assert(
+      RawService.services.size == 1,
+      s"Not all services stopped properly. Still running: ${RawService.services}"
+    )
   }
 }

From 03eda44b9bffa3291f250177649d7c50e072dc09 Mon Sep 17 00:00:00 2001
From: Benjamin Gaidioz <ben@raw-labs.com>
Date: Wed, 21 Feb 2024 14:34:52 +0100
Subject: [PATCH 09/11] wip

---
 utils/src/test/scala/raw/utils/RawTestSuite.scala | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/utils/src/test/scala/raw/utils/RawTestSuite.scala b/utils/src/test/scala/raw/utils/RawTestSuite.scala
index e2864f4e8..1bdf7c009 100644
--- a/utils/src/test/scala/raw/utils/RawTestSuite.scala
+++ b/utils/src/test/scala/raw/utils/RawTestSuite.scala
@@ -44,23 +44,23 @@ trait RawTestSuite extends FixtureAnyFunSuite with BeforeAndAfterAll with Strict
     test(TestData(test.name))
   }
 
-  var services: Seq[RawService] = _
   override def beforeAll(): Unit = {
     super.beforeAll()
   }
 
+  protected def serviceCanBeRunning(service: RawService) = false
+
+  private def servicesAreStopped = RawService.services.asScala.forall(s => serviceCanBeRunning(s))
+
   override def afterAll(): Unit = {
     logger.info("Checking if all services have stopped")
     var attempts = 10
-    while (RawService.services.size() > 1 && attempts > 0) {
-      logger.debug(s"Services up at teardown: ${RawService.services.asScala.map(_.toString).mkString(", ")}")
+    while (!servicesAreStopped && attempts > 0) {
       attempts -= 1
       logger.debug(s"Waiting for services to terminate gracefully. Attempts left: $attempts")
       Thread.sleep(1000)
     }
-    assert(
-      RawService.services.size == 1,
-      s"Not all services stopped properly. Still running: ${RawService.services}"
-    )
+    assert(servicesAreStopped, s"Not all services stopped properly. Still running:\n"
+      + RawService.services.asScala.map(s => s"- $s: can be running = ${serviceCanBeRunning(s)}").mkString("\n"))
   }
 }

From 309f3364710ce709d486f1f33a0b09ea5a415445 Mon Sep 17 00:00:00 2001
From: Benjamin Gaidioz <ben@raw-labs.com>
Date: Wed, 21 Feb 2024 15:53:31 +0100
Subject: [PATCH 10/11] wip

---
 utils/src/test/scala/raw/utils/RawTestSuite.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/src/test/scala/raw/utils/RawTestSuite.scala b/utils/src/test/scala/raw/utils/RawTestSuite.scala
index 1bdf7c009..0720bfbd3 100644
--- a/utils/src/test/scala/raw/utils/RawTestSuite.scala
+++ b/utils/src/test/scala/raw/utils/RawTestSuite.scala
@@ -48,7 +48,7 @@ trait RawTestSuite extends FixtureAnyFunSuite with BeforeAndAfterAll with Strict
     super.beforeAll()
   }
 
-  protected def serviceCanBeRunning(service: RawService) = false
+  protected def serviceCanBeRunning(service: RawService) = true
 
   private def servicesAreStopped = RawService.services.asScala.forall(s => serviceCanBeRunning(s))
 

From c39fc6804588abd7370880d0337565290dcc53b9 Mon Sep 17 00:00:00 2001
From: Benjamin Gaidioz <ben@raw-labs.com>
Date: Wed, 21 Feb 2024 16:24:37 +0100
Subject: [PATCH 11/11] wip

---
 utils/src/test/scala/raw/utils/RawTestSuite.scala | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/utils/src/test/scala/raw/utils/RawTestSuite.scala b/utils/src/test/scala/raw/utils/RawTestSuite.scala
index 0720bfbd3..daae48e4a 100644
--- a/utils/src/test/scala/raw/utils/RawTestSuite.scala
+++ b/utils/src/test/scala/raw/utils/RawTestSuite.scala
@@ -60,7 +60,10 @@ trait RawTestSuite extends FixtureAnyFunSuite with BeforeAndAfterAll with Strict
       logger.debug(s"Waiting for services to terminate gracefully. Attempts left: $attempts")
       Thread.sleep(1000)
     }
-    assert(servicesAreStopped, s"Not all services stopped properly. Still running:\n"
-      + RawService.services.asScala.map(s => s"- $s: can be running = ${serviceCanBeRunning(s)}").mkString("\n"))
+    assert(
+      servicesAreStopped,
+      s"Not all services stopped properly. Still running:\n"
+        + RawService.services.asScala.map(s => s"- $s: can be running = ${serviceCanBeRunning(s)}").mkString("\n")
+    )
   }
 }