(1) added argument to path of perfspect binary; (2) disabled perfspect on aarch64 and windows; (3) fixed case when perfspect metrics are not collected

Author: mlim19

gprofiler/main.py

@@ -57,7 +57,7 @@
 from gprofiler.metadata.external_metadata import ExternalMetadataStaleError, read_external_metadata
 from gprofiler.metadata.metadata_collector import get_current_metadata, get_static_metadata
 from gprofiler.metadata.system_metadata import get_hostname, get_run_mode, get_static_system_info
-from gprofiler.platform import is_linux, is_windows
+from gprofiler.platform import is_linux, is_windows, is_aarch64
 from gprofiler.profiler_state import ProfilerState
 from gprofiler.profilers.factory import get_profilers
 from gprofiler.profilers.profiler_base import NoopProfiler, ProcessProfilerBase, ProfilerInterface
@@ -137,6 +137,7 @@ def __init__(
         controller_process: Optional[Process] = None,
         external_metadata_path: Optional[Path] = None,
         heartbeat_file_path: Optional[Path] = None,
+        perfspect_path: Optional[Path] = None,
     ):
         self._output_dir = output_dir
         self._flamegraph = flamegraph
@@ -157,6 +158,7 @@ def __init__(
         self._duration = duration
         self._external_metadata_path = external_metadata_path
         self._heartbeat_file_path = heartbeat_file_path
+        self._perfspect_path = perfspect_path
         if self._collect_metadata:
             self._static_metadata = get_static_metadata(self._spawn_time, user_args, self._external_metadata_path)
         self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=10)
@@ -179,7 +181,8 @@ def __init__(
         self._usage_logger = usage_logger
         if self._collect_metrics:
             self._system_metrics_monitor: SystemMetricsMonitorBase = SystemMetricsMonitor(
-                self._profiler_state.stop_event
+                stop_event = self._profiler_state.stop_event,
+                perfspect_path = perfspect_path
             )
         else:
             self._system_metrics_monitor = NoopSystemMetricsMonitor()
@@ -834,6 +837,17 @@ def parse_cmd_args() -> configargparse.Namespace:
         "The file modification indicates the last snapshot time.",
     )
 
+    if is_linux() and not is_aarch64():
+        hw_metrics_options = parser.add_argument_group("hardware metrics")
+        hw_metrics_options.add_argument(
+            "--perfspect-path",
+            type=str,
+            dest="intel_perfspect_path",
+            default=None,
+            help="Enable HW metrics collection with Intel PerfSpect."
+            " Provide path to PerfSpect binary to enable collection.",
+        )
+
     args = parser.parse_args()
 
     args.perf_inject = args.nodejs_mode == "perf"
@@ -1103,6 +1117,13 @@ def main() -> None:
         if args.heartbeat_file is not None:
             heartbeat_file_path = Path(args.heartbeat_file)
 
+        perfspect_path: Optional[Path] = None
+        if args.intel_perfspect_path is not None:
+            perfspect_path = Path(args.intel_perfspect_path)
+            if not perfspect_path.is_file():
+                logger.error(f"Perfspect binary file {args.perfspect_path} does not exist!")
+                sys.exit(1)
+
         try:
             log_system_info()
         except Exception:
@@ -1187,6 +1208,7 @@ def main() -> None:
             processes_to_profile=processes_to_profile,
             external_metadata_path=external_metadata_path,
             heartbeat_file_path=heartbeat_file_path,
+            perfspect_path=perfspect_path,
         )
         logger.info("gProfiler initialized and ready to start profiling")
         if args.continuous:

gprofiler/platform.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import platform
 import sys
 from functools import lru_cache
 
@@ -28,3 +29,7 @@ def is_windows() -> bool:
 @lru_cache(maxsize=None)
 def is_linux() -> bool:
     return sys.platform == LINUX_PLATFORM_NAME
+
+@lru_cache(maxsize=None)
+def is_aarch64() -> bool:
+    return platform.machine() == "aarch64"

gprofiler/system_metrics.py

@@ -4,6 +4,7 @@
 from threading import Event, RLock, Thread
 from typing import List, Optional
 
+import os
 import subprocess
 import psutil
 
@@ -20,17 +21,17 @@ class Metrics:
     # The average RAM usage between gProfiler cycles
     mem_avg: Optional[float]
     # The CPU frequency between gProfiler cycles
-    cpu_freq: Optional[float]
+    cpu_freq: Optional[float] = None
     # The CPI between gProfiler cycles
-    cpu_cpi: Optional[float]
+    cpu_cpi: Optional[float] = None
     # The CPU TMA frontend bound between gProfiler cycles
-    cpu_tma_fe_bound: Optional[float]
+    cpu_tma_fe_bound: Optional[float] = None
     # The CPU TMA backend bound between gProfiler cycles
-    cpu_tma_be_bound: Optional[float]
+    cpu_tma_be_bound: Optional[float] = None
     # The CPU TMA bad speculation between gProfiler cycles
-    cpu_tma_bad_spec: Optional[float]
+    cpu_tma_bad_spec: Optional[float] = None
     # The CPU TMA retiring between gProfiler cycles
-    cpu_tma_retiring: Optional[float]
+    cpu_tma_retiring: Optional[float] = None
 
 class SystemMetricsMonitorBase(metaclass=ABCMeta):
     @abstractmethod
@@ -61,11 +62,19 @@ def _get_hw_metrics(self) -> Optional[List[float]]:
 
     def get_metrics(self) -> Metrics:
         hw_metrics = self._get_hw_metrics()
-        return Metrics(self._get_cpu_utilization(), self._get_average_memory_utilization(), hw_metrics[0], hw_metrics[1], hw_metrics[2], hw_metrics[3], hw_metrics[4], hw_metrics[5])
+        if hw_metrics and len(hw_metrics) == 6:
+            return Metrics(self._get_cpu_utilization(), self._get_average_memory_utilization(), hw_metrics[0], hw_metrics[1], hw_metrics[2], hw_metrics[3], hw_metrics[4], hw_metrics[5])
+        else:
+            return Metrics(self._get_cpu_utilization(), self._get_average_memory_utilization())
 
 
 class SystemMetricsMonitor(SystemMetricsMonitorBase):
-    def __init__(self, stop_event: Event, polling_rate_seconds: int = DEFAULT_POLLING_INTERVAL_SECONDS):
+    def __init__(
+        self,
+        stop_event: Event,
+        polling_rate_seconds: int = DEFAULT_POLLING_INTERVAL_SECONDS,
+        perfspect_path: str = None
+    ):
         self._polling_rate_seconds = polling_rate_seconds
         self._mem_percentages: List[float] = []
         self._stop_event = stop_event
@@ -74,42 +83,46 @@ def __init__(self, stop_event: Event, polling_rate_seconds: int = DEFAULT_POLLIN
         self._perfspect_thread: Optional[Thread] = None
         self._hw_metrics = {'cpu_freq':[], 'cpu_cpi':[], 'cpu_tma_fe_bound':[], 'cpu_tma_be_bound':[], 'cpu_tma_bad_spec':[], 'cpu_tma_retiring':[]}
         self._ps_process = None
+        self._perfspect_path = perfspect_path
 
         self._get_cpu_utilization()  # Call this once to set the necessary data
 
     def start(self) -> None:
         assert self._thread is None, "SystemMetricsMonitor is already running"
-        assert self._perfspect_thread is None, "Perfspect is already running"
         assert not self._stop_event.is_set(), "Stop condition is already set (perhaps gProfiler was already stopped?)"
         self._thread = Thread(target=self._continuously_poll_memory, args=(self._polling_rate_seconds,))
         self._thread.start()
 
-        ps_cmd = ['/tmp/perfspect', 'metrics', '--metrics', '"CPU operating frequency (in GHz)","CPI","TMA_Frontend_Bound(%)","TMA_Bad_Speculation(%)","TMA_Backend_Bound(%)","TMA_Retiring(%)"', '--duration', '0', '--live', '--format', 'csv', '--interval', '10']
-        self._ps_process = subprocess.Popen(ps_cmd, stdout=subprocess.PIPE)
-#        ps_stdout, ps_stderr = ps_process.communicate()
-#        try:
-#            # wait 2 seconds to ensure it starts
-#            ps_process.wait(2)
-#        except subprocess.TimeoutExpired:
-#            pass
-#        else:
-#            raise Exception(f"Command {ps_cmd} exited unexpectedly with {ps_process.returncode}")
-        self._perfspect_thread = Thread(target=self._continuously_poll_perfspect, args=(self._polling_rate_seconds,))
-        self._perfspect_thread.start()
+        if self._perfspect_path:
+            assert self._perfspect_thread is None, "Perfspect is already running"
+            ps_cmd = [self._perfspect_path, 'metrics', '--metrics', '"CPU operating frequency (in GHz)","CPI","TMA_Frontend_Bound(%)","TMA_Bad_Speculation(%)","TMA_Backend_Bound(%)","TMA_Retiring(%)"', '--duration', '0', '--live', '--format', 'csv', '--interval', '10']
+            self._ps_process = subprocess.Popen(ps_cmd, stdout=subprocess.PIPE)
+        #    ps_stdout, ps_stderr = ps_process.communicate()
+        #    try:
+        #        # wait 2 seconds to ensure it starts
+        #        ps_process.wait(2)
+        #    except subprocess.TimeoutExpired:
+        #        pass
+        #    else:
+        #        raise Exception(f"Command {ps_cmd} exited unexpectedly with {ps_process.returncode}")
+            self._perfspect_thread = Thread(target=self._continuously_poll_perfspect, args=(self._polling_rate_seconds,))
+            self._perfspect_thread.start()
 
     def stop(self) -> None:
         assert self._thread is not None, "SystemMetricsMonitor is not running"
-        assert self._perfspect_thread is not None, "Perfspect is not running"
         assert self._stop_event.is_set(), "Stop event was not set before stopping the SystemMetricsMonitor"
         self._thread.join(STOP_TIMEOUT_SECONDS)
         if self._thread.is_alive():
             raise ThreadStopTimeoutError("Timed out while waiting for the SystemMetricsMonitor internal thread to stop")
         self._thread = None
-        self._ps_process.kill()
-        self._perfspect_thread.join(STOP_TIMEOUT_SECONDS)
-        if self._perfspect_thread.is_alive():
-            raise ThreadStopTimeoutError("Timed out while waiting for the SystemMetricsMonitor Perfspect thread to stop")
-        self._perfspect_thread = None
+
+        if self._perfspect_path:
+            self._ps_process.kill()
+            assert self._perfspect_thread is not None, "Perfspect is not running"
+            self._perfspect_thread.join(STOP_TIMEOUT_SECONDS)
+            if self._perfspect_thread.is_alive():
+                raise ThreadStopTimeoutError("Timed out while waiting for the SystemMetricsMonitor Perfspect thread to stop")
+            self._perfspect_thread = None
 
     def _continuously_poll_memory(self, polling_rate_seconds: int) -> None:
         while not self._stop_event.is_set():
@@ -125,7 +138,7 @@ def _continuously_poll_perfspect(self, polling_rate_seconds: int) -> None:
             if metrics_str.startswith('TS,SKT,CPU,CID'):
                 continue
             metric_values = metrics_str.split(',')
-            if len(metric_values) > 0:
+            if len(metric_values) > 0 and metric_values[0] != '':
                 self._hw_metrics['cpu_freq'].append(float(metric_values[4]))
                 self._hw_metrics['cpu_cpi'].append(float(metric_values[5]))
                 self._hw_metrics['cpu_tma_fe_bound'].append(float(metric_values[6]))
@@ -172,7 +185,7 @@ def _get_hw_metrics(self) -> List[float]:
         self._hw_metrics['cpu_tma_retiring'] = []
 
         return metric_list
-    
+
 
 class NoopSystemMetricsMonitor(SystemMetricsMonitorBase):
     def start(self) -> None: