update

Author: sbalandi

tools/llm_bench/benchmark.py

@@ -9,7 +9,7 @@
 from openvino import get_version
 import torch
 import traceback
-from llm_bench_utils.memory_profile import MemMonitorWrapper
+from llm_bench_utils.memory_monitor import MemMonitorWrapper
 import llm_bench_utils.output_csv
 import llm_bench_utils.output_json
 import task.visual_language_generation as bench_vlm

tools/llm_bench/llm_bench_utils/gen_output_data.py

@@ -12,7 +12,9 @@ def gen_iterate_data(
     latency='',
     res_md5='',
     max_rss_mem='',
+    max_rss_mem_increase='',
     max_sys_mem='',
+    max_sys_mem_increase='',
     prompt_idx='',
     tokenization_time=[],
     mm_embeddings_preparation_time=''
@@ -30,7 +32,9 @@ def gen_iterate_data(
     iter_data['first_token_infer_latency'] = -1
     iter_data['other_tokens_infer_avg_latency'] = -1
     iter_data['max_rss_mem_consumption'] = max_rss_mem
+    iter_data['max_rss_mem_increase'] = max_rss_mem_increase
     iter_data['max_sys_mem_consumption'] = max_sys_mem
+    iter_data['max_sys_mem_increase'] = max_sys_mem_increase
     iter_data['prompt_idx'] = prompt_idx
     iter_data['tokenization_time'] = tokenization_time[0] if len(tokenization_time) > 0 else ''
     iter_data['detokenization_time'] = tokenization_time[1] if len(tokenization_time) > 1 else ''

tools/llm_bench/llm_bench_utils/memory_monitor.py

@@ -10,7 +10,6 @@
 # limitations under the License.
 import argparse
 import atexit
-import logging
 import queue
 import subprocess
 import threading
@@ -26,7 +25,7 @@
 import psutil
 from tabulate import tabulate
 
-logger = logging.getLogger("memory_monitor")
+import logging as log
 
 # CUSTOM FIX TO AVOID ISSUE: RuntimeError: main thread is not in main loop
 matplotlib.use('Agg')
@@ -50,7 +49,7 @@ class MemoryUnit(Enum):
 @lru_cache
 def system_memory_warning():
     # Log once
-    logger.warning(
+    log.warning(
         "Please note that MemoryType.SYSTEM in general is affected by other processes that change RAM availability."
     )
 
@@ -265,7 +264,7 @@ def _monitor_memory(self):
 class memory_monitor_context:
     def __init__(
         self,
-        interval: Optional[float] = 0.1,
+        interval: Optional[float] = 0.01,
         memory_unit: Optional[MemoryUnit] = MemoryUnit.MiB,
         return_max_value: Optional[bool] = True,
         save_dir: Optional[Path] = None,
@@ -296,7 +295,7 @@ def __init__(
         self.return_max_value = return_max_value
         self.save_dir = save_dir
 
-        self.memory_data = {}
+        self.memory_data = {'full_mem': {}, 'from_zero': {}}
 
     def __enter__(self):
         for mm in self.memory_monitors.values():
@@ -312,18 +311,96 @@ def __exit__(self, exc_type, exc_val, exc_tb):
             mm.stop()
             for fz in [False, True]:
                 time_values, memory_values = mm.get_data(memory_from_zero=fz)
-                if fz:
-                    self.memory_data[mt] = max(memory_values) if self.return_max_value else (time_values, memory_values)
+
+                mm_measure_type = 'from_zero' if fz else 'full_mem'
+                self.memory_data[mm_measure_type][mt] = max(memory_values) if self.return_max_value else (time_values, memory_values)
 
                 if self.save_dir:
                     mm.save_memory_logs(
                         time_values,
                         memory_values,
                         save_dir=self.save_dir,
-                        filename_suffix="_from-zero" if fz else "",
+                        filename_suffix="_mem_increase" if fz else "",
                     )
 
 
+class MemMonitorWrapper():
+    def __init__(self):
+        self.save_dir = None
+
+        self.interval = 0.01
+        self.memory_unit = MemoryUnit.MiB
+        
+        self.memory_types = [MemoryType.RSS, MemoryType.SYSTEM]
+
+        self.memory_monitors = {}
+        self.memory_data = {'full_mem': {}, 'from_zero': {}}
+
+    def create_monitors(self):
+        for memory_type in self.memory_types:
+            self.memory_monitors[memory_type] = MemoryMonitor(
+                interval=self.interval, memory_type=memory_type, memory_unit=self.memory_unit
+            )
+
+    def set_dir(self, dir):
+        if not Path(dir).exists():
+            log.warning(f"Path to dir for memory consamption data is not exists {dir}, run without it.")
+        else:
+            self.save_dir = Path(dir)
+
+    def start(self, delay=None):
+        self.memory_data = {'full_mem': {}, 'from_zero': {}}
+        for mm in self.memory_monitors.values():
+            mm.start()
+
+        # compilation could be very fast, apply delay
+        if delay:
+            time.sleep(delay)
+        else:
+            time.sleep(self.interval * 3)
+
+    def stop_and_collect_data(self, dir_name='mem_monitor_log'):
+        self.stop()
+
+        for mt, mm in self.memory_monitors.items():
+            if not mm._memory_values_queue or len(mm._memory_values_queue.queue) == 0:
+                continue
+
+            for from_zero in [False, True]:
+                time_values, memory_values = mm.get_data(memory_from_zero=from_zero)
+
+                mm_measure_type = 'from_zero' if from_zero else 'full_mem'
+                self.memory_data[mm_measure_type][mt] = max(memory_values)
+
+                if self.save_dir:
+                    mm.save_memory_logs(
+                        time_values,
+                        memory_values,
+                        save_dir=self.save_dir / dir_name,
+                        filename_suffix="_mem_increase" if from_zero else "",
+                    )    
+
+    def stop(self):
+        # Stop addition of new values as soon as possible
+        for mm in self.memory_monitors.values():
+            mm._monitoring_thread_should_stop = True
+
+        for mm in self.memory_monitors.values():
+            mm.stop()
+
+    def get_data(self):
+        return (self.memory_data['full_mem'].get(MemoryType.RSS, -1), self.memory_data['from_zero'].get(MemoryType.RSS, -1),
+                self.memory_data['full_mem'].get(MemoryType.SYSTEM, -1), self.memory_data['from_zero'].get(MemoryType.SYSTEM, -1))
+
+    def log_data(self, comment):
+        max_rss_mem, max_rss_increase, max_sys_mem, max_sys_increase = self.get_data()
+        msg = (f"Max rss memory cost {comment}: {max_rss_mem:.2f}{self.memory_unit.value}, "
+               f"rss memory increase {comment}: {max_rss_increase:.2f}{self.memory_unit.value}, "
+               f"max system memory cost {comment}: {max_sys_mem:.2f}{self.memory_unit.value}, "
+               f"system memory increase {comment}: {max_sys_increase:.2f}{self.memory_unit.value}")
+        log.info(msg)
+
+
 def _cast_bytes_to(bytes, memory_unit, round_to_int=False):
     memory_unit_divisors = {
         MemoryUnit.B: 1,

tools/llm_bench/llm_bench_utils/memory_profile.py

@@ -63,8 +63,12 @@ def print_metrics(
     output_str = ''
     if iter_data['max_rss_mem_consumption'] != '' and iter_data['max_rss_mem_consumption'] > -1:
         output_str += f"Max rss memory cost: {iter_data['max_rss_mem_consumption']:.2f}MBytes, "
+    if iter_data['max_rss_mem_increase'] != '' and iter_data['max_rss_mem_increase'] > -1:
+        output_str += f"rss memory increase: {iter_data['max_rss_mem_increase']:.2f}MBytes, "
     if iter_data['max_sys_mem_consumption'] != '' and iter_data['max_sys_mem_consumption'] > -1:
-        output_str += f"max system memory memory cost: {iter_data['max_sys_mem_consumption']:.2f}MBytes"
+        output_str += f"max system memory memory cost: {iter_data['max_sys_mem_consumption']:.2f}MBytes, "
+    if iter_data['max_sys_mem_increase'] != '' and iter_data['max_sys_mem_increase'] > -1:
+        output_str += f"system memory increase: {iter_data['max_sys_mem_increase']:.2f}MBytes "
     if output_str != '':
         output_str = ' '.join([prefix, output_str])
         log.info(output_str)

tools/llm_bench/llm_bench_utils/metrics_print.py

@@ -91,7 +91,8 @@ def run_image_generation(image_param, num, image_id, pipe, args, iter_data_list,
     result_md5_list = []
     max_rss_mem_consumption = ''
     max_sys_mem_consumption = ''
-    print("MEM mem_consumption MEASURES ")
+    max_rss_mem_increase = ''
+    max_sys_mem_increase = ''
     if (args['mem_consumption'] == 1 and num == 0) or args['mem_consumption'] == 2:
         mem_consumption.start()
 
@@ -109,9 +110,7 @@ def run_image_generation(image_param, num, image_id, pipe, args, iter_data_list,
     end = time.perf_counter()
     if (args['mem_consumption'] == 1 and num == 0) or args['mem_consumption'] == 2:
         mem_consumption.stop_and_collect_data(f"{'P' + str(num) if num > 0 else 'warm-up'}_{proc_id}")
-        print("MEM mem_consumption MEASURES FINISH 1")
-        max_rss_mem_consumption, max_sys_mem_consumption = mem_consumption.get_data()
-        print("MEM mem_consumption MEASURES FINISH 2")
+        max_rss_mem_consumption, max_rss_mem_increase, max_sys_mem_consumption, max_sys_mem_increase = mem_consumption.get_data()
     for bs_idx in range(args['batch_size']):
         rslt_img_fn = llm_bench_utils.output_file.output_gen_image(res[bs_idx], args, image_id, num, bs_idx, proc_id, '.png')
         result_md5_list.append(hashlib.md5(Image.open(rslt_img_fn).tobytes(), usedforsecurity=False).hexdigest())
@@ -123,7 +122,9 @@ def run_image_generation(image_param, num, image_id, pipe, args, iter_data_list,
         gen_time=generation_time,
         res_md5=result_md5_list,
         max_rss_mem=max_rss_mem_consumption,
+        max_rss_mem_increase=max_rss_mem_increase,
         max_sys_mem=max_sys_mem_consumption,
+        max_sys_mem_increase=max_sys_mem_increase,
         prompt_idx=image_id,
     )
     iter_data_list.append(iter_data)
@@ -158,6 +159,8 @@ def run_image_generation_genai(image_param, num, image_id, pipe, args, iter_data
     result_md5_list = []
     max_rss_mem_consumption = ''
     max_sys_mem_consumption = ''
+    max_rss_mem_increase = ''
+    max_sys_mem_increase = ''
     if (args['mem_consumption'] == 1 and num == 0) or args['mem_consumption'] == 2:
         mem_consumption.start()
 
@@ -180,7 +183,7 @@ def run_image_generation_genai(image_param, num, image_id, pipe, args, iter_data
 
     if (args['mem_consumption'] == 1 and num == 0) or args['mem_consumption'] == 2:
         mem_consumption.stop_and_collect_data(f"{'P' + str(num) if num > 0 else 'warm-up'}_{proc_id}")
-        max_rss_mem_consumption, max_sys_mem_consumption = mem_consumption.get_data()
+        max_rss_mem_consumption, max_rss_mem_increase, max_sys_mem_consumption, max_sys_mem_increase = mem_consumption.get_data()
     for bs_idx in range(args['batch_size']):
         image = Image.fromarray(res[bs_idx])
         rslt_img_fn = llm_bench_utils.output_file.output_gen_image(image, args, image_id, num, bs_idx, proc_id, '.png')
@@ -193,7 +196,9 @@ def run_image_generation_genai(image_param, num, image_id, pipe, args, iter_data
         gen_time=generation_time,
         res_md5=result_md5_list,
         max_rss_mem=max_rss_mem_consumption,
+        max_rss_mem_increase=max_rss_mem_increase,
         max_sys_mem=max_sys_mem_consumption,
+        max_sys_mem_increase=max_sys_mem_increase,
         prompt_idx=image_id,
     )
     iter_data_list.append(iter_data)
@@ -233,7 +238,7 @@ def run_image_generation_benchmark(model_path, framework, device, args, num_iter
         if "guidance_scale" in static_input_args:
             args["guidance_scale"] = static_input_args["guidance_scale"]
 
-    pipe, pretrain_time, use_genai, callback = FW_UTILS[framework].create_image_gen_model(model_path, device, **args)
+    pipe, pretrain_time, use_genai, callback = FW_UTILS[framework].create_image_gen_model(model_path, device, mem_consumption, **args)
     iter_data_list = []
 
     if framework == "ov" and not use_genai:

tools/llm_bench/task/image_generation.py

@@ -25,6 +25,8 @@ def run_speech_2_txt_generation(input_param, args, md5_list, iter_data_list):
     result_md5_list = []
     max_rss_mem_consumption = ''
     max_sys_mem_consumption = ''
+    max_rss_mem_increase = ''
+    max_sys_mem_increase = ''
     pipe = input_param['pipe']
     raw_speech = input_param['raw_speech']
     num = input_param['iter_idx']
@@ -85,15 +87,17 @@ def run_speech_2_txt_generation(input_param, args, md5_list, iter_data_list):
         md5_list[num][speech_id] = result_md5_list
     if (args['mem_consumption'] == 1 and num == 0) or args['mem_consumption'] == 2:
         mem_consumption.stop_and_collect_data(f"{'P' + str(num) if num > 0 else 'warm-up'}")
-        max_rss_mem_consumption, max_sys_mem_consumption = mem_consumption.get_data()
+        max_rss_mem_consumption, max_rss_mem_increase, max_sys_mem_consumption, max_sys_mem_increase = mem_consumption.get_data()
 
     iter_data = gen_output_data.gen_iterate_data(
         iter_idx=num,
         out_size=out_token_size,
         gen_time=generation_time,
         res_md5=result_md5_list,
         max_rss_mem=max_rss_mem_consumption,
+        max_rss_mem_increase=max_rss_mem_increase,
         max_sys_mem=max_sys_mem_consumption,
+        max_sys_mem_increase=max_sys_mem_increase,
         prompt_idx=speech_id,
     )
     iter_data_list.append(iter_data)

tools/llm_bench/task/speech_to_text_generation.py

@@ -34,14 +34,16 @@ def run_ldm_super_resolution(img, num, pipe, args, framework, iter_data_list, im
     low_res_img = low_res_img.resize((resize_image_width, resize_image_height))
     max_rss_mem_consumption = ''
     max_sys_mem_consumption = ''
+    max_rss_mem_increase = ''
+    max_sys_mem_increase = ''
     if (args['mem_consumption'] == 1 and num == 0) or args['mem_consumption'] == 2:
         mem_consumption.start()
     start = time.perf_counter()
     res = pipe(low_res_img, num_inference_steps=nsteps, tm_list=tm_list)
     end = time.perf_counter()
     if (args['mem_consumption'] == 1 and num == 0) or args['mem_consumption'] == 2:
         mem_consumption.stop_and_collect_data(f"{'P' + str(num) if num > 0 else 'warm-up'}_{proc_id}")
-        max_rss_mem_consumption, max_sys_mem_consumption = mem_consumption.get_data()
+        max_rss_mem_consumption, max_rss_mem_increase, max_sys_mem_consumption, max_sys_mem_increase = mem_consumption.get_data()
     result_md5_list = []
     if framework == 'ov':
         rslt_img_fn = llm_bench_utils.output_file.output_gen_image(res[0], args, image_id, num, None, proc_id, '.png')
@@ -54,7 +56,9 @@ def run_ldm_super_resolution(img, num, pipe, args, framework, iter_data_list, im
         gen_time=generation_time,
         res_md5=result_md5_list,
         max_rss_mem=max_rss_mem_consumption,
+        max_rss_mem_increase=max_rss_mem_increase,
         max_sys_mem=max_sys_mem_consumption,
+        max_sys_mem_increase=max_sys_mem_increase,
         prompt_idx=image_id,
     )
     iter_data_list.append(iter_data)