Add SPECjvm98 support (#63)

Co-authored-by: Zixian Cai <u5937495@anu.edu.au>

Author: tianleq

docs/src/changelog.md

@@ -5,6 +5,9 @@
 - `JVMClasspathAppend`
 - `JVMClasspathPrepend`
 
+#### Benchmark Suites
+- `SPECjvm98`
+
 ### Changed
 #### Modifiers
 - `JVMClasspath` is now an alias of `JVMClasspathAppend`. This is backward compatible.

docs/src/references/suite.md

@@ -101,6 +101,52 @@ Note that the property file should reside in `path/../config/specjbb2015.props`
 Only strings are allowed, which should correspond to the the mode of the SPECjbb2015 controller.
 Right now, only `"composite"` is supported.
 
+## `SPECjvm98` (preview ⚠️)
+[SPECjvm98](https://www.spec.org/jvm98/).
+
+Note that you will need to prepend probes to the classpaths, so that the [modified](https://github.com/anupli/probes/blob/master/SpecApplication.java) `SpecApplication` can be used.
+
+Here is an example configuration file.
+```yaml
+includes:
+  - "/home/zixianc/running-ng/src/running/config/base/runbms.yml"
+
+modifiers:
+  probes_cp:
+    type: JVMClasspathPrepend
+    val: "/home/zixianc/MMTk-Dev/evaluation/probes /home/zixianc/MMTk-Dev/evaluation/probes/probes.jar"
+
+benchmarks:
+  specjvm98:
+    - _213_javac
+
+configs:
+  - "adoptopenjdk-8|probes_cp"
+```
+
+### Keys
+`release`: one of the possible values `["1.03_05"]`.
+The value is required.
+
+`path`: path to the SPECjvm98 folder, where you can find `SpecApplication.class`.
+The value is required.
+
+`timing_iteration`: specifying the timing iteration.
+It can only be a number, which is passed to SpecApplication as `-i`.
+The value is required.
+
+### Benchmark Specification
+Only strings are allowed, which should correspond to benchmark program of SPECjvm98.
+The following are the benchmarks:
+- _200_check
+- _201_compress
+- _202_jess
+- _209_db
+- _213_javac
+- _222_mpegaudio
+- _227_mtrt
+- _228_jack
+
 ## `Octane` (preview ⚠️)
 ### Keys
 `path`: path to the Octane benchmark folder.

src/running/benchmark.py

@@ -20,7 +20,7 @@ class SubprocessrExit(Enum):
 
 
 class Benchmark(object):
-    def __init__(self, suite_name: str, name: str, wrapper: Optional[str] = None, timeout: Optional[int] = None, **kwargs):
+    def __init__(self, suite_name: str, name: str, wrapper: Optional[str] = None, timeout: Optional[int] = None, override_cwd: Optional[Path] = None, **kwargs):
         self.name = name
         self.suite_name = suite_name
         self.env_args: Dict[str, str]
@@ -31,6 +31,9 @@ def __init__(self, suite_name: str, name: str, wrapper: Optional[str] = None, ti
         else:
             self.wrapper = []
         self.timeout = timeout
+        # ignore the current working directory provided by commands like runbms or minheap
+        # certain benchmarks expect to be invoked from certain directories
+        self.override_cwd = override_cwd
 
     def get_env_str(self) -> str:
         return " ".join([
@@ -85,7 +88,7 @@ def run(self, runtime: Runtime, cwd: Path = None) -> Tuple[bytes, SubprocessrExi
                     stdout=subprocess.PIPE,
                     stderr=subprocess.STDOUT,
                     timeout=self.timeout,
-                    cwd=cwd
+                    cwd=self.override_cwd if self.override_cwd else cwd
                 )
                 return p.stdout, SubprocessrExit.Normal
             except subprocess.CalledProcessError as e:

src/running/config/base/specjvm98.yml

@@ -0,0 +1,6 @@
+suites:
+  specjvm98:
+    type: SPECjvm98
+    path: "/usr/share/benchmarks/SPECjvm98"
+    release: "1.03_05"
+    timing_iteration: 3

src/running/config/base/suites.yml

@@ -1,3 +1,4 @@
 includes:
   - "./dacapo.yml"
   - "./specjbb2015.yml"
+  - "./specjvm98.yml"

src/running/suite.py

@@ -186,7 +186,7 @@ def get_benchmark(self, bm_spec: Union[str, Dict[str, Any]]) -> 'JavaBenchmark':
                 program_args.append("--converge")
         # Input size
         program_args.extend(["-s", size])
-        # Naem of the benchmark
+        # Name of the benchmark
         program_args.append(bm_name)
         return JavaBenchmark(
             jvm_args=[],
@@ -342,3 +342,55 @@ def get_minheap(self, bm: Benchmark) -> int:
 
     def is_passed(self, output: bytes) -> bool:
         return b"PASSED" in output
+
+
+@register(BenchmarkSuite)
+class SPECjvm98(JavaBenchmarkSuite):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.release: str
+        self.release = kwargs["release"]
+        if self.release not in ["1.03_05"]:
+            raise ValueError(
+                "SPECjvm98 release {} not recongized".format(self.release))
+        self.path: Path
+        self.path = Path(kwargs["path"]).resolve()
+
+        if not self.path.exists():
+            logging.info("SPECjvm98 {} not found".format(self.path))
+        if not (self.path / "SpecApplication.class").exists():
+            logging.info(
+                "SpecApplication.class not found under SPECjvm98 {}".format(self.path))
+        try:
+            self.timing_iteration = int(kwargs.get("timing_iteration"))
+        except ValueError:
+            timing_iteration = kwargs.get("timing_iteration")
+            raise TypeError("The timing iteration of the SPECjvm98 benchmark suite `{}` is {}, which is not an "
+                            "integer".format(self.path.parent, repr(timing_iteration)))
+
+    def __str__(self) -> str:
+        return "{} SPECjvm98 {} {}".format(super().__str__(), self.release, self.path)
+
+    def get_benchmark(self, bm_spec: Union[str, Dict[str, Any]]) -> 'JavaBenchmark':
+        assert type(bm_spec) is str
+        program_args = [
+            "SpecApplication",
+            "-i{}".format(self.timing_iteration),
+            bm_spec
+        ]
+        return JavaBenchmark(
+            jvm_args=[],
+            program_args=program_args,
+            cp=[str(self.path)],
+            suite_name=self.name,
+            name=bm_spec,
+            override_cwd=self.path
+        )
+
+    def get_minheap(self, _bm: Benchmark) -> int:
+        # FIXME allow user to measure and specify minimum heap sizes
+        return 32  # SPEC recommends running with minimum 32MB of heap
+
+    def is_passed(self, output: bytes) -> bool:
+        # FIXME
+        return b"**NOT VALID**" not in output