Sync with 'develop'

Signed-off-by: Abolfazl Shahbazi <abolfazl.shahbazi@intel.com>

Author: ashahba

benchmarks/README.md

@@ -21,6 +21,7 @@
 import glob
 import json
 import os
+import re
 import sys
 import time
 
@@ -153,6 +154,16 @@ def run_numactl_multi_instance(self, cmd, replace_unique_output_dir=None):
         so that each instance uses a unique output folder.
         """
 
+        # Find LD_PRELOAD vars, remove them from the cmd, and save them to add on to the prefix
+        ld_preload_strs = re.findall(r'\bLD_PRELOAD=\S*', cmd)
+        ld_preload_prefix = ""
+        for ld_preload_str in ld_preload_strs:
+            cmd = cmd.replace(ld_preload_str, "")
+            ld_preload_prefix += ld_preload_str + " "
+
+        # Remove leading/trailing whitespace
+        cmd = cmd.strip()
+
         if self.args.numa_cores_per_instance != "socket":
             # Get the cores list and group them according to the number of cores per instance
             cores_per_instance = int(self.args.numa_cores_per_instance)
@@ -208,9 +219,9 @@ def run_numactl_multi_instance(self, cmd, replace_unique_output_dir=None):
             if len(core_list) == 0:
                 continue
 
-            prefix = ("OMP_NUM_THREADS={0} "
-                      "numactl --localalloc --physcpubind={1}").format(
-                len(core_list), ",".join(core_list))
+            prefix = ("{0}OMP_NUM_THREADS={1} "
+                      "numactl --localalloc --physcpubind={2}").format(
+                ld_preload_prefix, len(core_list), ",".join(core_list))
             instance_logfile = log_filename_format.format("instance" + str(instance_num))
 
             unique_command = cmd
@@ -282,6 +293,7 @@ def get_command_prefix(self, socket_id, numactl=True):
         Should be used only for single instance.
         """
         command = ""
+        ld_preload = ""
 
         if not self.args.disable_tcmalloc:
             # Try to find the TCMalloc library file
@@ -290,16 +302,53 @@ def get_command_prefix(self, socket_id, numactl=True):
             if len(matches) == 0:
                 matches = glob.glob("/usr/lib64/libtcmalloc.so*")
 
+            if len(matches) == 0:
+                matches = glob.glob("/usr/lib/*/libtcmalloc.so*")
+
+            if len(matches) == 0:
+                matches = glob.glob("/usr/lib64/*/libtcmalloc.so*")
+
             if len(matches) > 0:
-                command += "LD_PRELOAD={} ".format(matches[0])
+                ld_preload += "LD_PRELOAD={} ".format(matches[0])
             else:
                 # Unable to find the TCMalloc library file
-                print("Warning: Unable to find the TCMalloc library file (libtcmalloc.so) in /usr/lib or /usr/lib64, "
-                      "so the LD_PRELOAD environment variable will not be set.")
+                print("Warning: Unable to find the TCMalloc library file (libtcmalloc.so) in /usr/lib, /usr/lib64, "
+                      "/usr/lib/*, or /usr/lib64/* so the LD_PRELOAD environment variable will not be set.")
 
         num_numas = self.platform_util.num_numa_nodes
         if num_numas and socket_id != -1 and numactl and not self.args.numa_cores_per_instance:
-            command += "numactl --cpunodebind={0} --membind={0} ".format(str(socket_id))
+            if self.args.num_cores == -1:
+                # Running on the whole socket
+                command += "numactl --cpunodebind={0} --membind={0} ".format(
+                    str(socket_id))
+            else:
+                # Running on specific number of cores
+                first_physical_core = self.platform_util.cpuset_cpus[0][0]
+                num_sockets = len(self.platform_util.cpuset_cpus.keys())
+                num_cores_in_socket0 = len(self.platform_util.cpuset_cpus[0])
+                for i in range(num_sockets):
+                    if num_cores_in_socket0 != len(
+                            self.platform_util.cpuset_cpus[i]):
+                        raise ValueError(
+                            "Error: Identifying logical core id assumes all sockets have same number of cores"
+                        )
+                first_logical_core = num_cores_in_socket0 * num_sockets
+                if self.platform_util.num_threads_per_core == 1:
+                    # HT is off
+                    cpus_range = "{0}-{1}".format(
+                        first_physical_core,
+                        first_physical_core + self.args.num_cores - 1)
+                else:
+                    # HT is on.
+                    cpus_range = "{0}-{1},{2}-{3}".format(
+                        first_physical_core,
+                        first_physical_core + self.args.num_cores - 1,
+                        first_logical_core,
+                        first_logical_core + self.args.num_cores - 1)
+                command += "numactl -C{0} --membind=0 ".format(cpus_range)
+
+        # Add LD_PRELOAD to the front of the command
+        command = ld_preload + command
 
         return command
 

benchmarks/common/base_model_init.py

@@ -249,7 +249,7 @@ def _get_cpuset(self):
             with open(cpuset_cpus_file, "r") as f:
                 cpuset = f.read()
 
-            if self.args.verbose:
+            if hasattr(self.args, "verbose") and self.args.verbose:
                 print("cpuset.cpus: {}".format(cpuset))
         return cpuset
 
@@ -301,10 +301,16 @@ def linux_init(self):
 
         # Try to get the cpuset.cpus info, since lscpu does not know if the cpuset is limited
         cpuset = self._get_cpuset()
+
         if cpuset:
+            num_cores_arg = -1
+            if hasattr(self.args, "num_cores"):
+                num_cores_arg = self.args.num_cores
             # If the cpuset is the same as the online_cpus_list, then we are using the whole
-            # machine, so let's avoid unnecessary complexity and don't bother with the cpuset_cpu list
-            if (online_cpus_list != "" and online_cpus_list != cpuset) or online_cpus_list == "":
+            # machine, so let's avoid unnecessary complexity and don't bother with the cpuset_cpu list.
+            # The cpuset_cpus list will also get populated if the num_cores arg is being specified,
+            # since this list will be used to create the numactl args in base_model_init.py
+            if (online_cpus_list != "" and online_cpus_list != cpuset) or online_cpus_list == "" or num_cores_arg != -1:
                 self.cpuset_cpus = self._get_list_from_string_ranges(cpuset)
 
         # Uses numactl get the core number for each numa node and adds the cores for each
@@ -316,33 +322,34 @@ def linux_init(self):
             cores_per_node = int(num_physical_cores / self.num_numa_nodes)
         else:
             cores_per_node = self.num_cores_per_socket
-
-        if self.num_numa_nodes > 0 and self.args.numa_cores_per_instance is not None:
-            try:
-                # Get the list of cores
-                cpu_array_command = \
-                    "numactl -H | grep 'node [0-9]* cpus:' |" \
-                    "sed 's/.*node [0-9]* cpus: *//' | head -{0} |cut -f1-{1} -d' '".format(
-                        self.num_numa_nodes, int(cores_per_node))
-                cpu_array = subprocess.Popen(
-                    cpu_array_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.readlines()
-
-                for node_cpus in cpu_array:
-                    node_cpus = str(node_cpus).lstrip("b'").replace("\\n'", " ")
-                    self.cpu_core_list.append([x for x in node_cpus.split(" ") if x != ''])
-
-                # If we have the cpuset list, cross check that list with our core list and
-                # remove cores that are not part of the cpuset list
-                if self.cpuset_cpus is not None:
-                    for socket, core_list in enumerate(self.cpu_core_list):
-                        self.cpu_core_list[socket] = [x for x in core_list if int(x) in self.cpuset_cpus]
-
-                if (self.args.verbose):
-                    print("Core list: {}".format(self.cpu_core_list), flush=True)
-
-            except Exception as e:
-                print("Warning: An error occured when getting the list of cores using '{}':\n {}".
-                      format(cpu_array_command, e))
+        if hasattr(self.args, "numa_cores_per_instance"):
+            if self.num_numa_nodes > 0 and self.args.numa_cores_per_instance is not None:
+                try:
+                    # Get the list of cores
+                    cpu_array_command = \
+                        "numactl -H | grep 'node [0-9]* cpus:' |" \
+                        "sed 's/.*node [0-9]* cpus: *//' | head -{0} |cut -f1-{1} -d' '".format(
+                            self.num_numa_nodes, int(cores_per_node))
+                    cpu_array = subprocess.Popen(
+                        cpu_array_command, shell=True, stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE).stdout.readlines()
+
+                    for node_cpus in cpu_array:
+                        node_cpus = str(node_cpus).lstrip("b'").replace("\\n'", " ")
+                        self.cpu_core_list.append([x for x in node_cpus.split(" ") if x != ''])
+
+                    # If we have the cpuset list, cross check that list with our core list and
+                    # remove cores that are not part of the cpuset list
+                    if self.cpuset_cpus is not None:
+                        for socket, core_list in enumerate(self.cpu_core_list):
+                            self.cpu_core_list[socket] = [x for x in core_list if int(x) in self.cpuset_cpus]
+
+                    if hasattr(self.args, "verbose") and self.args.verbose:
+                        print("Core list: {}".format(self.cpu_core_list), flush=True)
+
+                except Exception as e:
+                    print("Warning: An error occured when getting the list of cores using '{}':\n {}".
+                          format(cpu_array_command, e))
 
         if self.cpuset_cpus is not None:
             # Reformat the cpuset_cpus list so that it's split up by node