TensorRT_YoloV5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "U3GsJ7kcaE-t"
   },
   "source": [
    "# **Speeding Up Inference with TensorRT**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## YOLO-V5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ik8U7U8sbjkj",
    "outputId": "4b437dc7-cc28-4eaf-b95b-bbd076bc0223"
   },
   "outputs": [],
   "source": [
    "!pip install opencv-python\n",
    "!pip install numpy\n",
    "!pip install torch-tensorrt\n",
    "!pip install pandas\n",
    "!pip install matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ik8U7U8sbjkj",
    "outputId": "4b437dc7-cc28-4eaf-b95b-bbd076bc0223"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/long/.cache/torch/hub/ultralytics_yolov5_master\n",
      "YOLOv5 🚀 2024-2-27 Python-3.10.12 torch-2.2.1+cu121 CUDA:0 (NVIDIA GeForce RTX 4080 Laptop GPU, 12282MiB)\n",
      "\n",
      "Fusing layers... \n",
      "YOLOv5s summary: 213 layers, 7225885 parameters, 0 gradients, 16.4 GFLOPs\n",
      "Adding AutoShape... \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "model = torch.hub.load('ultralytics/yolov5', 'yolov5s', pretrained=True)\n",
    "\n",
    "# Use GPU for inference\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "model.to(device)\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ZkqFbyfAcAbb",
    "outputId": "4a05c640-c0dd-4cb9-fa8b-cabc0ac292e0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-02-28 13:45:22--  http://images.cocodataset.org/zips/val2017.zip\n",
      "Resolving images.cocodataset.org (images.cocodataset.org)... 52.216.43.217, 3.5.30.157, 52.216.208.41, ...\n",
      "Connecting to images.cocodataset.org (images.cocodataset.org)|52.216.43.217|:80... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 815585330 (778M) [application/zip]\n",
      "Saving to: ‘val2017.zip.3’\n",
      "\n",
      "val2017.zip.3         0%[                    ]   2.30M  1.54MB/s               ^C\n",
      "replace val2017/000000212226.jpg? [y]es, [n]o, [A]ll, [N]one, [r]ename: ^C\n"
     ]
    }
   ],
   "source": [
    "# Download COCO validation data set\n",
    "!wget http://images.cocodataset.org/zips/val2017.zip\n",
    "!unzip val2017.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Iterate through the images in the dataset\n",
    "import os\n",
    "import cv2\n",
    "\n",
    "def prepare_images(image_paths):\n",
    "\n",
    "    images = []\n",
    "    for image_path in image_paths:        \n",
    "      image = cv2.imread(image_path)\n",
    "\n",
    "      # Preprocess the image\n",
    "      image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "      image = cv2.resize(image, (640, 640))\n",
    "        \n",
    "      image = image.transpose((2, 0, 1)) # W, H, C => C, H, W\n",
    "      image = torch.from_numpy(image).to(device)\n",
    "      image = image.float() / 255.0 # Normalize [0;1]\n",
    "      images.append(image)\n",
    "        \n",
    "    return images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load COCO dataset\n",
    "image_dir = 'val2017'\n",
    "image_paths = [os.path.join(image_dir, img) for img in os.listdir(image_dir)]\n",
    "\n",
    "# Only take a subset of COCO\n",
    "IMAGE_NUMS = 128\n",
    "images = prepare_images(image_paths[:IMAGE_NUMS])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "def run_inference(model, images):    \n",
    "  times = []\n",
    "  results = []\n",
    "  for image in images:\n",
    "    # Move image to GPU, if not already\n",
    "    image = image.unsqueeze(0)\n",
    "    image_gpu = image.to('cuda')\n",
    "    \n",
    "    # Run inference with no gradient calculation\n",
    "    start = time.time()\n",
    "    with torch.no_grad():\n",
    "        result = model(image_gpu)\n",
    "\n",
    "    results.append(result)\n",
    "    times.append(time.time() - start)\n",
    "\n",
    "  return results, times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average inference time (s): 0.008198220282793045\n",
      "Average inference time (s): 0.006218124181032181\n",
      "Average inference time (s): 0.007922634482383728\n",
      "Average inference time (s): 0.008371112868189812\n",
      "Average inference time (s): 0.007605014368891716\n",
      "Average inference time (s): 0.005643885582685471\n",
      "Average inference time (s): 0.0056277308613061905\n",
      "Average inference time (s): 0.005955127999186516\n",
      "Average inference time (s): 0.010654594749212265\n",
      "Average inference time (s): 0.010228211060166359\n",
      "Mean average inference time (s): 0.007642465643584728\n"
     ]
    }
   ],
   "source": [
    "NUM_TRIALS = 10\n",
    "avgs = []\n",
    "\n",
    "for i in range(NUM_TRIALS):  \n",
    "    results, times = run_inference(model, images)\n",
    "    avg = sum(times)/len(times)\n",
    "    print(\"Average inference time (s):\", avg)\n",
    "    avgs.append(avg)\n",
    "    \n",
    "mat = sum(avgs)/NUM_TRIALS\n",
    "print(\"Mean average inference time (s):\", mat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vKIN7zJi2kvY"
   },
   "source": [
    "# **Accelerating the model with TensorRT**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/long/.cache/torch/hub/ultralytics_yolov5_master/models/yolo.py:100: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  if self.dynamic or self.grid[i].shape[2:4] != x[i].shape[2:4]:\n",
      "/home/long/.cache/torch/hub/ultralytics_yolov5_master/models/yolo.py:100: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  if self.dynamic or self.grid[i].shape[2:4] != x[i].shape[2:4]:\n",
      "WARNING: [Torch-TensorRT] - Trying to record the value 0 with the ITensor input_0 again.\n",
      "WARNING: [Torch-TensorRT] - Dilation not used in Max pooling converter\n",
      "WARNING: [Torch-TensorRT] - Dilation not used in Max pooling converter\n",
      "WARNING: [Torch-TensorRT] - Dilation not used in Max pooling converter\n",
      "WARNING: [Torch-TensorRT] - Unable to process input type of at::kLong, truncate type to at::kInt in scalar_to_tensor_util \n",
      "WARNING: [Torch-TensorRT] - Trying to record the value 213 with the ITensor (Unnamed Layer* 121) [ElementWise]_output again.\n",
      "WARNING: [Torch-TensorRT] - Unable to process input type of at::kLong, truncate type to at::kInt in scalar_to_tensor_util \n",
      "WARNING: [Torch-TensorRT] - Trying to record the value 224 with the ITensor (Unnamed Layer* 126) [ElementWise]_output again.\n",
      "WARNING: [Torch-TensorRT] - Unable to process input type of at::kLong, truncate type to at::kInt in scalar_to_tensor_util \n",
      "WARNING: [Torch-TensorRT] - Trying to record the value 53 with the ITensor (Unnamed Layer* 23) [ElementWise]_output again.\n",
      "WARNING: [Torch-TensorRT] - Unable to process input type of at::kLong, truncate type to at::kInt in scalar_to_tensor_util \n",
      "WARNING: [Torch-TensorRT] - Trying to record the value 64 with the ITensor (Unnamed Layer* 28) [ElementWise]_output again.\n",
      "WARNING: [Torch-TensorRT] - Truncating weight (constant in the graph) from Int64 to Int32\n",
      "WARNING: [Torch-TensorRT] - Truncating weight (constant in the graph) from Int64 to Int32\n",
      "WARNING: [Torch-TensorRT] - Truncating weight (constant in the graph) from Int64 to Int32\n",
      "WARNING: [Torch-TensorRT] - Truncating weight (constant in the graph) from Int64 to Int32\n",
      "WARNING: [Torch-TensorRT] - Truncating weight (constant in the graph) from Int64 to Int32\n",
      "WARNING: [Torch-TensorRT] - Truncating weight (constant in the graph) from Int64 to Int32\n"
     ]
    }
   ],
   "source": [
    "import torch_tensorrt\n",
    "\n",
    "model.eval()  # Set the model to evaluation mode\n",
    "input_data = torch.randn(1, 3, 640, 640)\n",
    "\n",
    "\n",
    "trt_model = torch_tensorrt.compile(model, inputs=[input_data],\n",
    "                                   ir='ts',\n",
    "                                   enabled_precisions={torch.float32},\n",
    "                                   truncate_long_and_double=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorRT optimized\n",
      "Average inference time (s): 0.004738861694931984\n",
      "Average inference time (s): 0.003201013430953026\n",
      "Average inference time (s): 0.0034089460968971252\n",
      "Average inference time (s): 0.003065887838602066\n",
      "Average inference time (s): 0.002827117219567299\n",
      "Average inference time (s): 0.0027531571686267853\n",
      "Average inference time (s): 0.0029252376407384872\n",
      "Average inference time (s): 0.002684958279132843\n",
      "Average inference time (s): 0.0024376586079597473\n",
      "Average inference time (s): 0.0030416101217269897\n",
      "Mean average inference time (s): 0.0031084448099136354\n"
     ]
    }
   ],
   "source": [
    "print(\"TensorRT optimized\")\n",
    "\n",
    "avgs = []\n",
    "for i in range(NUM_TRIALS):  \n",
    "    trt_results, trt_time = run_inference(trt_model, images)\n",
    "    avg = sum(trt_time)/len(images)\n",
    "    print(\"Average inference time (s):\", avg)\n",
    "    avgs.append(avg)\n",
    "    \n",
    "trt_mat = sum(avgs)/NUM_TRIALS\n",
    "print(\"Mean average inference time (s):\", trt_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Speed up: 2.4586139085406717\n"
     ]
    }
   ],
   "source": [
    "speed_up = mat / trt_mat\n",
    "print(\"Speed up:\", speed_up)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **Batching**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/long/.cache/torch/hub/ultralytics_yolov5_master/models/yolo.py:100: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  if self.dynamic or self.grid[i].shape[2:4] != x[i].shape[2:4]:\n",
      "/home/long/.cache/torch/hub/ultralytics_yolov5_master/models/yolo.py:100: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  if self.dynamic or self.grid[i].shape[2:4] != x[i].shape[2:4]:\n",
      "WARNING:torch_tensorrt._compile:Input graph is a Torchscript module but the ir provided is default (dynamo). Please set ir=torchscript to suppress the warning. Compiling the module with ir=torchscript\n",
      "WARNING: [Torch-TensorRT] - Trying to record the value 0 with the ITensor input_0 again.\n",
      "WARNING: [Torch-TensorRT] - Dilation not used in Max pooling converter\n",
      "WARNING: [Torch-TensorRT] - Dilation not used in Max pooling converter\n",
      "WARNING: [Torch-TensorRT] - Dilation not used in Max pooling converter\n",
      "WARNING: [Torch-TensorRT] - Unable to process input type of at::kLong, truncate type to at::kInt in scalar_to_tensor_util \n",
      "WARNING: [Torch-TensorRT] - Trying to record the value 213 with the ITensor (Unnamed Layer* 121) [ElementWise]_output again.\n",
      "WARNING: [Torch-TensorRT] - Unable to process input type of at::kLong, truncate type to at::kInt in scalar_to_tensor_util \n",
      "WARNING: [Torch-TensorRT] - Trying to record the value 224 with the ITensor (Unnamed Layer* 126) [ElementWise]_output again.\n",
      "WARNING: [Torch-TensorRT] - Unable to process input type of at::kLong, truncate type to at::kInt in scalar_to_tensor_util \n",
      "WARNING: [Torch-TensorRT] - Trying to record the value 53 with the ITensor (Unnamed Layer* 23) [ElementWise]_output again.\n",
      "WARNING: [Torch-TensorRT] - Unable to process input type of at::kLong, truncate type to at::kInt in scalar_to_tensor_util \n",
      "WARNING: [Torch-TensorRT] - Trying to record the value 64 with the ITensor (Unnamed Layer* 28) [ElementWise]_output again.\n",
      "WARNING: [Torch-TensorRT] - There may be undefined behavior using dynamic shape and aten::size without setting allow_shape_tensors\n",
      "WARNING: [Torch-TensorRT] - Truncating weight (constant in the graph) from Int64 to Int32\n",
      "WARNING: [Torch-TensorRT] - Truncating weight (constant in the graph) from Int64 to Int32\n",
      "WARNING: [Torch-TensorRT] - There may be undefined behavior using dynamic shape and aten::size without setting allow_shape_tensors\n",
      "WARNING: [Torch-TensorRT] - Truncating weight (constant in the graph) from Int64 to Int32\n",
      "WARNING: [Torch-TensorRT] - Truncating weight (constant in the graph) from Int64 to Int32\n",
      "WARNING: [Torch-TensorRT] - There may be undefined behavior using dynamic shape and aten::size without setting allow_shape_tensors\n",
      "WARNING: [Torch-TensorRT] - Truncating weight (constant in the graph) from Int64 to Int32\n",
      "WARNING: [Torch-TensorRT] - Truncating weight (constant in the graph) from Int64 to Int32\n"
     ]
    }
   ],
   "source": [
    "import torch_tensorrt\n",
    "import torch\n",
    "\n",
    "# Example for dynamic batch size\n",
    "BATCH_SIZE = 16\n",
    "DYNAMIC_BATCH_ENABLED = True\n",
    "input_data = torch.randn(BATCH_SIZE, 3, 640, 640)\n",
    "\n",
    "if DYNAMIC_BATCH_ENABLED:\n",
    "    # Specify dynamic batch size\n",
    "    inputs = [torch_tensorrt.Input(\n",
    "        min_shape=[1, 3, 640, 640],\n",
    "        opt_shape=[BATCH_SIZE, 3, 640, 640],\n",
    "        max_shape=[32, 3, 640, 640],  # Example max batch size\n",
    "        dtype=torch.float32  # Ensure this matches your model's expected input type\n",
    "    )]\n",
    "else:\n",
    "    inputs = [input_data]\n",
    "\n",
    "ts = torch.jit.trace(model, input_data, strict=False)\n",
    "trt_model = torch_tensorrt.compile(ts,\n",
    "                                   inputs=inputs,\n",
    "                                   enabled_precisions={torch.float32},\n",
    "                                   truncate_long_and_double=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_inference_in_batches(model, images, batch_size=BATCH_SIZE):\n",
    "    times = []\n",
    "    results = []\n",
    "    \n",
    "    for i in range(0, len(images), batch_size):\n",
    "        batch_images = images[i:i + batch_size]\n",
    "        \n",
    "        # Stack images to create a batch\n",
    "        batch = torch.stack(batch_images).to('cuda')\n",
    "\n",
    "        start = time.time()\n",
    "        with torch.no_grad():\n",
    "            result = model(batch)\n",
    "\n",
    "        results.append(result)\n",
    "        times.append(time.time() - start)\n",
    "    \n",
    "    return results, times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorRT optimized - Batched  16\n",
      "Average inference time (s): 0.0031599905341863632\n",
      "Average inference time (s): 0.0016660261899232864\n",
      "Average inference time (s): 0.0016808975487947464\n",
      "Average inference time (s): 0.0016959533095359802\n",
      "Average inference time (s): 0.0017578788101673126\n",
      "Average inference time (s): 0.00174027681350708\n",
      "Average inference time (s): 0.00173121877014637\n",
      "Average inference time (s): 0.0017148610204458237\n",
      "Average inference time (s): 0.0016897749155759811\n",
      "Average inference time (s): 0.0017396770417690277\n",
      "Mean average inference time (s): 0.001857655495405197\n"
     ]
    }
   ],
   "source": [
    "print(\"TensorRT optimized - Batched \", BATCH_SIZE)\n",
    "avgs = []\n",
    "for i in range(NUM_TRIALS):\n",
    "    batched_results, batched_time = run_inference_in_batches(trt_model, images)\n",
    "    avg = sum(batched_time)/len(images)\n",
    "    print(\"Average inference time (s):\", avg)\n",
    "    avgs.append(avg)\n",
    "\n",
    "batched_mat = sum(avgs)/NUM_TRIALS\n",
    "print(\"Mean average inference time (s):\", batched_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.114038185491294"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mat/batched_mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEICAYAAABS0fM3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAcFklEQVR4nO3de5QdZZ3u8e9jDAdGwQhpJORCUMOooALGIOBxUFG5SUYXDqDICOOJIA64xstR5xyBWSo6Ol4gSiZHUPACOsJwkTCCF5SLCSRMCISARkZNSJQQICSCQOJz/qi3x83O7u6dTld3kno+a+3Vu6reeutXu3rvX9X71kW2iYiI5nrGSAcQEREjK4kgIqLhkggiIhouiSAiouGSCCIiGi6JICKi4ZIIYpsi6QZJ7x7pOLYGkhZLOmSk44iRl0QQQ0LSqyXdImmNpIck3SzplSMdVytJX5f0ibZxkyVZ0jMHWec4SRdIWilpraR7JJ0t6VlDE3V9bO9t+4aRjiNGXhJBbDZJOwHfB84DdgbGA2cDT4xkXHWTtDPwc2AH4EDbOwJvAMYALxjB0Po12KQX264kghgKewHYvsT2BtuP277O9iIASe8qRwjnlSOGeyS9vndmSc9p2au+X9InJI1qmX6ypCWSHpb0A0l7tEx7Q6lvjaSZgDZnRST9WtJHJd1dlvc1Sdv3UfwfgLXACbZ/XT6DZbbPaFn3gyTdVuK7TdJBLcu6oazrLZLWSbpa0i6SviXp0VJ+ckt5Szpd0n2SHpT0WUnPKNNeIOnHklaXad+SNKZtvf63pEXAHyQ9s4w7tEyfJml+We7vJX2+Zd6jSzPSIyXmF7fV+0FJi8o6fqefzyu2UEkEMRR+AWyQdJGkwyU9t0OZA4D7gLHAmcDlZY8a4CJgPfBCYD/gjcC7AST9NfAx4K1AD3AjcEmZNha4DPg/pd5fAQcPwfq8A3gT1V79XqX+Tg4FLrf9p04Ty/pdA5wL7AJ8HrhG0i4txY4D3kl1FPUCqiOMr1EdWS2h+qxavQWYCuwPTAdO7l0ccA6wO/BiYCJwVtu8xwNHAmNsr2+b9iXgS7Z3KnF8t6zDXlSf9/upPv85wNWStmuZ92+Aw4A9gZcB7+r0ecSWK4kgNpvtR4FXAwb+H7BK0lWSntdS7AHgi7afsv0d4F7gyFLmcOD9tv9g+wHgC1Q/kADvAc6xvaT8eH0K2LccFRwB3G37e7afAr4I/G4IVmlm2bN/CPgk1Q9oJ7sAK/up50jgl7a/YXu97UuAe4A3t5T5mu1f2V4DXAv8yvYPy7r+G1VibPUZ2w/Z/i3V+h4PYHup7ettP2F7FVXS+au2ec8t6/V4h1ifAl4oaaztdbbnlvHHAteUup8CPkfVFHZQy7zn2l5RPq+rgX37+UxiC5REEEOi/FC/y/YEYB+qPdMvthS530+/w+FvSpk9gNHAytL08Ajwr8CupdwewJdapj1Etfc7vsy/rCUGtw53sL4sq9Vo4E/l1au1jt44O1kNjOtnebuX+Vv9hir2Xr9vef94h+Fnt83fMTZJu0q6tDStPQp8k+ooqa952/0d1dHPPaVJ6qhO61COfpa1rUNr8n2sQ8yxhUsiiCFn+x7g61QJodd4Sa3t95OAFVQ/Kk8AY22PKa+dbO9dyi0D3tMybYztHWzfQrU3PrG3wlL/RPr2W2By27g9gWVtzTutdfTG2ckPgbf0ttN3sIIqkbWaBNzfT4wD6Su2c6iOyF5WmndOYOP+kj5vNWz7l7aPp0rAnwG+V858eto6tHzGm7MOsYVJIojNJulFkj4gaUIZnkjVZDG3pdiuwOmSRkt6G1U79hzbK4HrgH+RtJOkZ5SOz95mjVnARyXtXep+Tpkfqvb3vSW9VdWZMKcDu/UT6mVUzVFvlDRK0u5U7f+XtpU7TdKE0sb/MeA7fdT3eWAn4KLeDmxJ4yV9XtLLqNrT95L09tI5eyzwEqozrAbrQ5KeWz7jM1pi2xFYBzwiaTzwoU2pVNIJknpKQnykjN5A1VdwpKTXSxoNfIAqcd+yGesQW5gkghgKa6k6g+dJ+gNVAriL6kej1zxgCvAgVbv7MbZXl2knAtsBdwMPA9+jNLnY/neqPdRLS5PHXVR9Cth+EHgb8GmqZpopwM19BWl7MVWCOoeqiennJa6z24p+myo53Vden6CD0iZ+EFX7+jxJa4EfAWuApWX9jiqfw2rgw8BRJe7BuhJYACykSoQXlPFnU3UgrynjL9/Eeg8DFktaR9VxfJztP9q+l+ro4jyqbfdm4M22n9yMdYgtjPJgmqibpHcB77b96pGOZSCSfk0V6w9HOpZ2kgxMsb10pGOJbUuOCCIiGi6JICKi4dI0FBHRcDkiiIhouK3u5lNjx4715MmTRzqMiIityoIFCx603dNp2laXCCZPnsz8+fNHOoyIiK2KpPar3P9bmoYiIhouiSAiouGSCCIiGi6JICKi4ZIIIiIaLokgIqLhkggiIhouiSAiouGSCCIiGm6ru7I4IrZsOrv9CZkxVHxmPTcJzRFBRETD1Z4IyrNh/1PSRs9pVeVcSUslLZK0f93xRETE0w3HEcEZwJI+ph1O9ZzZKcAM4PxhiCciIlrUmggkTQCOBL7aR5HpwMWuzAXGSBpXZ0wREfF0dR8RfBH4MPCnPqaPB5a1DC8v455G0gxJ8yXNX7Vq1ZAHGRHRZLUlAklHAQ/YXtBfsQ7jNuoWtz3b9lTbU3t6Oj5XISIiBqnOI4KDgaMl/Rq4FHidpG+2lVkOTGwZngCsqDGmiIhoU1sisP1R2xNsTwaOA35s+4S2YlcBJ5azh14FrLG9sq6YIiJiY8N+QZmkUwBszwLmAEcAS4HHgJOGO56IiKYblkRg+wbghvJ+Vst4A6cNRwwREdFZriyOiGi4JIKIiIZLIoiIaLgkgoiIhksiiIhouCSCiIiGSyKIiGi4JIKIiIZLIoiIaLgkgoiIhksiiIhouCSCiIiGSyKIiGi4JIKIiIZLIoiIaLgkgoiIhqvz4fXbS7pV0h2SFks6u0OZQyStkbSwvD5eVzwREdFZnU8oewJ4ne11kkYDN0m61vbctnI32j6qxjgiIqIftSWC8hjKdWVwdHm5ruVFRMTg1NpHIGmUpIXAA8D1tud1KHZgaT66VtLedcYTEREbqzUR2N5ge19gAjBN0j5tRW4H9rD9cuA84IpO9UiaIWm+pPmrVq2qM+SIiMYZlrOGbD8C3AAc1jb+Udvryvs5wGhJYzvMP9v2VNtTe3p6hiHiiIjmqPOsoR5JY8r7HYBDgXvayuwmSeX9tBLP6rpiioiIjdV51tA44CJJo6h+4L9r+/uSTgGwPQs4BjhV0nrgceC40skcERHDpM6zhhYB+3UYP6vl/UxgZl0xRETEwHJlcUREwyURREQ0XBJBRETDJRFERDRcEkFERMMlEURENFwSQUREwyURREQ0XBJBRETDJRFERDRcEkFERMMlEURENFwSQUREwyURREQ0XBJBRETDJRFERDRcEkFERMPV+czi7SXdKukOSYslnd2hjCSdK2mppEWS9q8rnoiI6KzOZxY/AbzO9jpJo4GbJF1re25LmcOBKeV1AHB++RsREcOktiMCV9aVwdHl1f5g+unAxaXsXGCMpHF1xRQRERurtY9A0ihJC4EHgOttz2srMh5Y1jK8vIxrr2eGpPmS5q9ataq2eCMimqjWRGB7g+19gQnANEn7tBVRp9k61DPb9lTbU3t6emqINCKiuYblrCHbjwA3AIe1TVoOTGwZngCsGI6YIiKiUudZQz2SxpT3OwCHAve0FbsKOLGcPfQqYI3tlXXFFBERG6vzrKFxwEWSRlElnO/a/r6kUwBszwLmAEcAS4HHgJNqjCciIjqoLRHYXgTs12H8rJb3Bk6rK4aIiBhYriyOiGi4JIKIiIZLIoiIaLg+E4GkD5SO3vbxu0i6oN6wIiJiuPR3RPCXwAJJB/eOkPReYD5wZ92BRUTE8OjzrCHbMyQdBMyUtBh4EfBL4KCc6x8Rse0Y6PTRu4DbqK4IFvCBJIGIiG1Lf30EJwALgfuAFwBvAf5Z0sWSdh2e8CIiom79HRG8DXit7d+U4QWSDgROAeYCz687uIiIqF9/fQTTO4wzcL6k79UaVUREDJtBXUdgOw8FiIjYRuSCsoiIhksiiIhouK7uPlquJ5jcWt72xTXFFBERw2jARCDpG1Snjy4ENpTRBpIIIiK2Ad0cEUwFXlLOGIqIiG1MN30EdwG7bWrFkiZK+omkJZIWSzqjQ5lDJK2RtLC8Pr6py4mIiM3TzRHBWOBuSbcCT/SOtH30APOtp7olxe2SdqS6IO1623e3lbvR9lGbFHVERAyZbhLBWYOpuNyTaGV5v1bSEmA80J4IIiJiBA2YCGz/dHMXImky1fOL53WYfKCkO4AVwAdtL+4w/wxgBsCkSZM2N5yIiGjR303nbip/10p6tOW1VtKj3S5A0rOBy4D3226f73ZgD9svB84DruhUh+3ZtqfantrT09PtoiMiogv93Wvo1eXvjoOtXNJoqiTwLduXd1jGoy3v50j6iqSxth8c7DJjG/NtjXQE266350TAqNR2ZbEkARcAS2x/vo8yu5VySJpW4lldV0wREbGxrq4sHqSDgXcCd0paWMZ9DJgEYHsWcAxwqqT1wOPAcbleISJieNWWCGzfRPVUs/7KzARm1hVDREQMrNt7De0GTKO6tcRttn9Xa1QRETFsBuwjkPRu4FbgrVRNOXMlnVx3YBERMTy6OSL4ELCf7dUAknYBbgEurDOwiIgYHt2cNbQcWNsyvBZYVk84EREx3Lo5IrgfmCfpSqo+gunArZL+AaCvU0MjImLr0E0i+FV59bqy/B30hWYREbHl6OZeQ2cPRyARETEy+kwEkq6magrqZeBB4Ce2v1l3YBERMTz6OyL4XIdxOwMnSNrH9kdqiikiIoZRfzed63j7aUlXAQuAJIKIiG3AJt90zvaGgUtFRMTWor8+gp07jH4ucCKw0cNjIiJi69RfH8ECqg7i3hvH9XYW3wCcWm9YERExXPrrI9hzOAOJiIiRUduDaSIiYuuQRBAR0XB1PqpyoqSfSFoiabGkMzqUkaRzJS2VtEjS/nXFExERnfV31lC/P8q2bx+g7vXAB2zfLmlHYIGk623f3VLmcGBKeR0AnF/+RkTEMOnvrKF/KX+3B6YCd1CdQfQyYB7w6v4qtr0SWFner5W0BBgPtCaC6cDF5TnFcyWNkTSuzBsREcOgz6Yh26+1/VrgN8D+tqfafgWwH7B0UxYiaXKZb17bpPE8/dkGy8u49vlnSJovaf6qVas2ZdERETGAbvoIXmT7zt4B23cB+3a7AEnPBi4D3m/70fbJHWbxRiPs2SURTe3p6el20RER0YVunkewRNJXgW9S/UifACzppnJJo6mSwLdsX96hyHJgYsvwBGBFN3VHRMTQ6OaI4CSqW0qcAbyfqo3/pIFmkiTgAmBJP08xuwo4sZw99CpgTfoHIiKGVzcPpvmjpFnAHNv3bkLdBwPvBO6UtLCM+xgwqdQ7C5gDHEHV5/AYXSSYiIgYWgMmAklHA58FtgP2lLQv8E+2j+5vPts30bkPoLWMgdO6jjYiIoZcN01DZwLTgEcAbC8EJtcWUUREDKtuEsF622tqjyQiIkZEN2cN3SXp7cAoSVOA04Fb6g0rIiKGSzdHBH8P7A08AXwbWEN19lBERGwDujlr6DHgHyV9yvYfhiGmiIgYRgMeEUg6SNLdlIvIJL1c0ldqjywiIoZFN01DXwDeBKwGsH0H8Jo6g4qIiOHT1fMIbC9rG7WhhlgiImIEdHPW0DJJBwGWtB3VWUNd3WsoIiK2fN0cEZxCdfXveOB+qjuP5mrgiIhtRDdnDT0IvGMYYomIiBHQzVlDz5d0taRVkh6QdKWk5w9HcBERUb9umoa+DXwXGAfsDvwbcEmdQUVExPDpJhHI9jdsry+v3gfURETENqCbs4Z+IukjwKVUCeBY4BpJOwPYfqjG+CIiombdJIJjy9/3tI0/mSoxpL8gImIr1s1ZQ3sOpmJJFwJHAQ/Y3qfD9EOAK4H/KqMut/1Pg1lWREQMXp99BJJeKWm3luETyxlD5/Y2Cw3g68BhA5S50fa+5ZUkEBExAvrrLP5X4EkASa8BPg1cTHUb6tkDVWz7Z0D6DyIitnD9JYJRLR3BxwKzbV9m+/8CLxyi5R8o6Q5J10rau69CkmZImi9p/qpVq4Zo0RERAQMkAkm9fQivB37cMq2bTuaB3A7sYfvlwHnAFX0VtD3b9lTbU3t6eoZg0RER0au/RHAJ8FNJVwKPAzcCSHohVfPQZrH9qO115f0cYLSksZtbb0REbJo+9+xtf1LSj6iuKL7Odu9FZM+genzlZikd0b+3bUnTSr2rN7feiIjYNP028die22HcL7qpWNIlwCHAWEnLgTOB0aWOWcAxwKmS1lMdcRzXkmwiImKYDEVbf0e2jx9g+kxgZl3Lj4iI7nT1hLKIiNh2JRFERDRcEkFERMMlEURENFwSQUREwyURREQ0XBJBRETDJRFERDRcEkFERMMlEURENFwSQUREwyURREQ0XBJBRETDJRFERDRcEkFERMMlEURENFxtiUDShZIekHRXH9Ml6VxJSyUtkrR/XbFERETf6jwi+DpwWD/TDwemlNcM4PwaY4mIiD7Ulghs/wx4qJ8i04GLXZkLjJE0rq54IiKis9qeWdyF8cCyluHlZdzK9oKSZlAdNTBp0qRBL1Aa9KwxAHukI4iIwRrJzuJOP8sdf05sz7Y91fbUnp6emsOKiGiWkUwEy4GJLcMTgBUjFEtERGONZCK4CjixnD30KmCN7Y2ahSIiol619RFIugQ4BBgraTlwJjAawPYsYA5wBLAUeAw4qa5YIiKib7UlAtvHDzDdwGl1LT8iIrqTK4sjIhouiSAiouGSCCIiGi6JICKi4ZIIIiIaLokgIqLhkggiIhouiSAiouGSCCIiGi6JICKi4ZIIIiIaLokgIqLhkggiIhouiSAiouGSCCIiGi6JICKi4WpNBJIOk3SvpKWSPtJh+iGS1khaWF4frzOeiIjYWJ2PqhwFfBl4A9WD6m+TdJXtu9uK3mj7qLriiIiI/tV5RDANWGr7PttPApcC02tcXkREDEKdiWA8sKxleHkZ1+5ASXdIulbS3jXGExERHdTWNASowzi3Dd8O7GF7naQjgCuAKRtVJM0AZgBMmjRpiMOMiGi2Oo8IlgMTW4YnACtaC9h+1Pa68n4OMFrS2PaKbM+2PdX21J6enhpDjohonjoTwW3AFEl7StoOOA64qrWApN0kqbyfVuJZXWNMERHRpramIdvrJb0P+AEwCrjQ9mJJp5Tps4BjgFMlrQceB46z3d58FBERNaqzj6C3uWdO27hZLe9nAjPrjCEiIvqXK4sjIhouiSAiouGSCCIiGi6JICKi4ZIIIiIaLokgIqLhkggiIhouiSAiouGSCCIiGi6JICKi4ZIIIiIaLokgIqLhkggiIhouiSAiouGSCCIiGi6JICKi4ZIIIiIartZEIOkwSfdKWirpIx2mS9K5ZfoiSfvXGU9ERGystkQgaRTwZeBw4CXA8ZJe0lbscGBKec0Azq8rnoiI6KzOI4JpwFLb99l+ErgUmN5WZjpwsStzgTGSxtUYU0REtKnz4fXjgWUtw8uBA7ooMx5Y2VpI0gyqIwaAdZLuHdpQt1hjgQdHOohuSCMdwRZjq9lmvCMbja1pewE6a7O22R59TagzEXSK2IMog+3ZwOyhCGprImm+7akjHUd0L9ts65LtVamzaWg5MLFleAKwYhBlIiKiRnUmgtuAKZL2lLQdcBxwVVuZq4ATy9lDrwLW2F7ZXlFERNSntqYh2+slvQ/4ATAKuND2YkmnlOmzgDnAEcBS4DHgpLri2Uo1rjlsG5BttnXJ9gJkb9QkHxERDZIriyMiGi6JICKi4ZIIhpCkDZIWSrpD0u2SDhri+r8u6Zjy/qsdrtSOmkjapWzbhZJ+J+n+lmGXv3dJulrSGEnzyrjfSlrVUnbySK/LcNjU70L5zN7bRb03SBr06Z6t36Ey3N923a5lPQa1bSV9T9Lzy/tPSlomaV2Hcn8j6W5JiyV9u4zrkfQfg13XTVHndQRN9LjtfQEkvQk4B/irOhZk+9111Bud2V4N7Asg6Sxgne3PleF1Ldv9IuA02weU4XcBU22/b/ijHlGb+l0YA7wX+ErtkbXob7uWcY8PdttK2hsYZfu+MupqYCbwy7ZyU4CPAgfbfljSriW2VZJWSjrY9s1Ds8ad5YigPjsBDwNIerakH5U9ozslTS/jnyXpmrLXdJekY8v4V0j6qaQFkn7Q6bYbrXtGktaVvY07JM2V9LwyvkfSZZJuK6+Dh23tm+vnVFfHx58N+F0APg28oOxZf7aU/XApc4ekT7fU9zZJt0r6haT/WcqOkvTZ8n++SNJ7ynhJmln2tq8Bdt2M9djUbfsO4MreAdtz+zg9/n8BX7b9cCn3QMu0K0o9tcoRwdDaQdJCYHtgHPC6Mv6PwFtsPyppLDBX0lXAYcAK20cCSHqOpNHAecD0skdwLPBJ4OR+lvssYK7tf5T0z1T/WJ8AvgR8wfZNkiZRncr74iFe5yhU3Wjx9cAFIx3LFmBTvwsfAfZp2fs+HPhr4ADbj0nauaXuZ9qeJukI4EzgUODvqK5DeqWk/wHcLOk6YD/gL4GXAs8D7gYu3NSVGeS2PRi4pItye5Vl3Ex1qv1ZtnubhOZTfZdrlUQwtFoPIw8ELpa0D9WtND4l6TXAn6j2Kp4H3Al8TtJngO/bvrGU3we4XtUNfEbRdu+lDp4Evl/eLwDeUN4fCrxEf74R0E6SdrS9drPXNFr1/uhNpvr8rx/RaLYMm/pdaHco8DXbjwHYfqhl2uXl7wKqzxzgjcDL9Of2/+dQ3dX4NcAltjcAKyT9eBPXY3O27ThgVRflnkkV6yFUd1e4UdI+th8BHgB234RlDkqahmpi++dUN7TqoTq06wFeUb4cvwe2t/0L4BVUCeEcSR+n+qIstr1veb3U9hsHWNxT/vMFIRv4c4J/BnBgS13jkwRq0fujtwewHXDayIazZenmu9BhNtHhvmPFE+Vv6/+6gL9v+V/f0/Z1vSFsVLl0QEsn79H9hL852/ZxOq9bu+XAlbafsv1fwL1UiYEy/+ObsMxBSSKoiaQXUe3Nr6baO3nA9lOSXku5C6Ck3YHHbH8T+BywP9U/QU/Zi0LS6NLpNBjXAf/dkSVp30HWE12wvQY4HfhgaeILuvsuAGuBHVtmuw44WdJflDpam4Y6+QFwau/nLmkvSc8CfgYcV/oQxgGvBbA9ryVptN/6ZiOD3LZLgBd2Ue6K3rhKc9leQG8H817AXV0ub9DSNDS0eg8jodpD+VvbGyR9C7ha0nxgIXBPKfNS4LOS/gQ8BZxq+8lyeHuupOdQbaMvAosHEc/pwJclLSr1/Aw4ZVBrFl2x/Z+S7qC6t9Y3RjqeEbRJ3wXbqyXdLOku4FrbHyo7LvMlPUl1O5qP9bO8r1I139yuqi10FVUfw79T9U/cCfwC+OlgV2gQ2/YaquaeHwKU/ru3A38haTnwVdtnUSWxN0q6m+oo50PlbCaoEsQ1g425W7nFREREDSTtAPyE6rTQDYOs42dUJ448PKTBtS8niSAioh6qrqFYYvu3g5i3hyqJXDHkgbUvK4kgIqLZ0lkcEdFwSQQREQ2XRBAR0XBJBBERDZdEEBHRcP8fvx0Stn/EGpcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "speed_up_values = [mat / mat, mat / trt_mat, mat / batched_mat]\n",
    "labels = ['Baseline', 'TRT', 'Batched-TRT (16)']\n",
    "\n",
    "plt.bar(labels, speed_up_values, color=['blue', 'orange', 'green'])\n",
    "\n",
    "# Adding labels and title\n",
    "plt.ylabel('Speed Up in X')\n",
    "plt.title('Speed Up Comparison')\n",
    "\n",
    "# Display the plot\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}