04_Training_Schedule_and_Strategies/11_Dynamic_LR_Scheduling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9e0c293c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os \n",
    "os.environ['KMP_DUPLICATE_LIB_OK']='True'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "903f8943",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import dependencies\n",
    "from collections import OrderedDict\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "import torch\n",
    "import torchvision\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision.datasets.mnist import MNIST\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data.sampler import RandomSampler, SequentialSampler\n",
    "import torch.nn.init as init\n",
    "\n",
    "import random\n",
    "import numpy as np\n",
    "import psutil\n",
    "from torchsummary import summary\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import warnings \n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4b56f9ea-54b0-4175-972a-523290fdbe41",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random seed set to 42\n"
     ]
    }
   ],
   "source": [
    "def set_experiment_seed(seed=42):\n",
    "    \"\"\"\n",
    "    Set random seeds and CUDA-related flags for experiment reproducibility.\n",
    "    \"\"\"\n",
    "    # Set random seeds for reproducibility\n",
    "    torch.manual_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    random.seed(seed)\n",
    "\n",
    "    # If using GPU, set random seed for CUDA operations\n",
    "    if torch.cuda.is_available():\n",
    "        torch.cuda.manual_seed(seed)\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = False  # Set to False for reproducibility\n",
    "\n",
    "    # Optionally log seed information\n",
    "    print(f\"Random seed set to {seed}\")\n",
    "\n",
    "set_experiment_seed()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "079ce688",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download MNIST dataset\n",
    "data_train = MNIST(\n",
    "    \"./data/mnist\", download=True, \n",
    "    transform=transforms.Compose([\n",
    "        transforms.Resize((32, 32)),\n",
    "        transforms.RandomRotation(degrees=15),\n",
    "        transforms.RandomAffine(degrees=0, translate=(0.1, 0.1)),\n",
    "        transforms.RandomVerticalFlip(),\n",
    "        transforms.ColorJitter(brightness=0.2, contrast=0.2),\n",
    "        transforms.GaussianBlur(kernel_size=1),\n",
    "        transforms.ToTensor()\n",
    "    ])\n",
    ")\n",
    "        \n",
    "data_test = MNIST(\n",
    "    \"./data/mnist\", download=True, train=False,\n",
    "    transform=transforms.Compose([\n",
    "        transforms.Resize((32, 32)),\n",
    "        transforms.ToTensor()\n",
    "    ])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "372c5b4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define hyperparameters\n",
    "N_CLASSES = 10\n",
    "EPOCHS = 50\n",
    "BATCH_SIZE = 512\n",
    "DROPOUT_PROB = 0.2\n",
    "USE_BATCHNORM = True\n",
    "LABEL_SMOOTHING = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "93918f18",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plt.figure(figsize=(10,10))\n",
    "# for i in range(25):\n",
    "#     plt.subplot(5,5,i+1)\n",
    "#     plt.xticks([])\n",
    "#     plt.yticks([])\n",
    "#     plt.grid(False)\n",
    "#     plt.imshow(data_train[i][0].permute(1,2,0).numpy(), cmap=plt.cm.binary)\n",
    "#     plt.xlabel(data_train[i][1])\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce264b0f-c32e-4907-bc34-c7644df7d2c7",
   "metadata": {},
   "source": [
    "`torch.utils.data.DataLoader` supports asynchronous data loading and data augmentation in separate worker subprocesses. The default setting for DataLoader is `num_workers=0`, which means that the data loading is synchronous and done in the main process. As a result the main training process has to wait for the data to be available to continue the execution.\r\n",
    "\r\n",
    "Settin`g num_workers >` 0 enables asynchronous data loading and overlap between the training and data loading. num_workers should be tuned depending on the workload, CPU, GPU, and location of training data.`\r\n",
    "\r\n",
    "DataLo`ader acce`pts pin_me`mory argument, which defaults` to F`alse. When using a GPU it’s better to `set pin_memory=`True, this instru`cts DataLo`ader to use pinned memory and enables faster and asynchronous memory copy from the host to the GPU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "fb249649-163b-495a-bd21-94711d5ec411",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_num_workers():\n",
    "    # Calculate the number of CPU cores\n",
    "    num_cpu_cores = os.cpu_count()\n",
    "\n",
    "    # Set a safe maximum multiplier value (e.g., 0.5) to avoid using all available resources\n",
    "    max_multiplier = 0.5\n",
    "\n",
    "    # Calculate the number of workers based on the available resources\n",
    "    multiplier = min(sum(psutil.cpu_percent(interval=1, percpu=True)) / 100.0, max_multiplier)\n",
    "    \n",
    "    # Ensure that num_workers is at least 1\n",
    "    num_workers = max(1, int(num_cpu_cores * multiplier))\n",
    "\n",
    "    return num_workers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "5b752015",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define train dataloader\n",
    "train_dataloader = DataLoader(\n",
    "    data_train, batch_size=BATCH_SIZE, \n",
    "    sampler=RandomSampler(data_train), \n",
    "    pin_memory=True, num_workers=calculate_num_workers()\n",
    ")\n",
    "\n",
    "# Define test dataloader\n",
    "test_dataloader = DataLoader(\n",
    "    data_test, batch_size=BATCH_SIZE, \n",
    "    sampler=SequentialSampler(data_test),\n",
    "    pin_memory=True, num_workers=calculate_num_workers()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6864f541-00b1-4851-a1da-288992d8d709",
   "metadata": {},
   "source": [
    "`torch.nn.Conv2d()` has `bias` parameter which defaults to `True` (the same is true for `Conv1d` and `Conv3d` ).\n",
    "\n",
    "If a `nn.Conv2d` layer is directly followed by a `nn.BatchNorm2d` layer, then the bias in the convolution is not needed, instead use `nn.Conv2d(..., bias=False, ....)`. Bias is not needed because in the first step `BatchNorm` subtracts the mean, which effectively cancels out the effect of bias.\n",
    "\n",
    "This is also applicable to 1d and 3d convolutions as long as `BatchNorm`(or other normalization layer) normalizes on the same dimension as convolution’s bias."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "35af50e0-5861-444c-ab61-8581c61f9af0",
   "metadata": {},
   "outputs": [],
   "source": [
    "class LeNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(LeNet, self).__init__()\n",
    "                \n",
    "        self.features1 = nn.Sequential(\n",
    "            nn.Conv2d(1, 6, kernel_size=(5, 5), bias=not USE_BATCHNORM),\n",
    "            self._get_norm_layer(6),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=(2, 2), stride=2),\n",
    "            nn.Dropout2d(p=DROPOUT_PROB)\n",
    "        )\n",
    "        \n",
    "        self.features2 = nn.Sequential(\n",
    "            nn.Conv2d(6, 16, kernel_size=(5, 5), bias=not USE_BATCHNORM),\n",
    "            self._get_norm_layer(16),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=(2, 2), stride=2),\n",
    "            nn.Dropout2d(p=DROPOUT_PROB)\n",
    "        )\n",
    "\n",
    "        self.features3 = nn.Sequential(\n",
    "            nn.Conv2d(16, 120, kernel_size=(5, 5), bias=not USE_BATCHNORM),\n",
    "            self._get_norm_layer(120),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout2d(p=DROPOUT_PROB)\n",
    "        )\n",
    "\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Linear(120, 84, bias=True),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(p=DROPOUT_PROB),\n",
    "            nn.Linear(84, 10, bias=True)\n",
    "        )\n",
    "\n",
    "        # Initialize layers\n",
    "        self.apply(self._initialize_weights)\n",
    "\n",
    "    def _get_norm_layer(self, channels):\n",
    "        if USE_BATCHNORM:\n",
    "            return nn.BatchNorm2d(channels)\n",
    "        else:\n",
    "            return nn.Identity()\n",
    "\n",
    "    def _initialize_weights(self, m):\n",
    "        if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):\n",
    "            init.kaiming_uniform_(m.weight)\n",
    "            if m.bias is not None:\n",
    "                init.constant_(m.bias, 0)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.features1(x)\n",
    "        x = self.features2(x)\n",
    "        x = self.features3(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.classifier(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "621c6db2-e8a3-42cc-ab36-5694abdf5764",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "31f61bca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1           [512, 6, 28, 28]             150\n",
      "       BatchNorm2d-2           [512, 6, 28, 28]              12\n",
      "              ReLU-3           [512, 6, 28, 28]               0\n",
      "         MaxPool2d-4           [512, 6, 14, 14]               0\n",
      "         Dropout2d-5           [512, 6, 14, 14]               0\n",
      "            Conv2d-6          [512, 16, 10, 10]           2,400\n",
      "       BatchNorm2d-7          [512, 16, 10, 10]              32\n",
      "              ReLU-8          [512, 16, 10, 10]               0\n",
      "         MaxPool2d-9            [512, 16, 5, 5]               0\n",
      "        Dropout2d-10            [512, 16, 5, 5]               0\n",
      "           Conv2d-11           [512, 120, 1, 1]          48,000\n",
      "      BatchNorm2d-12           [512, 120, 1, 1]             240\n",
      "             ReLU-13           [512, 120, 1, 1]               0\n",
      "        Dropout2d-14           [512, 120, 1, 1]               0\n",
      "           Linear-15                  [512, 84]          10,164\n",
      "             ReLU-16                  [512, 84]               0\n",
      "          Dropout-17                  [512, 84]               0\n",
      "           Linear-18                  [512, 10]             850\n",
      "================================================================\n",
      "Total params: 61,848\n",
      "Trainable params: 61,848\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 2.00\n",
      "Forward/backward pass size (MB): 89.09\n",
      "Params size (MB): 0.24\n",
      "Estimated Total Size (MB): 91.32\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# Instantiate the model\n",
    "model = LeNet()\n",
    "model = model.to(device)\n",
    "\n",
    "# Test with a random input\n",
    "summary(model, input_size=(1, 32, 32), batch_size=BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d186351-babd-4d4e-886e-d9338ca71640",
   "metadata": {},
   "source": [
    "#### 1. Cross Entropy Loss\n",
    "The standard cross-entropy loss for classification tasks is given by:\n",
    "\n",
    "$$ \\text{Traditional Cross Entropy Loss: } H(y, \\hat{y}) = - \\sum_i y_i \\log(\\hat{y}_i) $$\n",
    "\n",
    " - $y_i$ is a binary indicator of whether class $i$ is the correct classification.  \n",
    " - $p_i$ is the predicted probability of class $i$.\n",
    "\n",
    "#### 2. Label Smoothed Cross Entropy\n",
    "Label Smoothing Cross Entropy Loss introduces a modification to the target distribution:\n",
    "\n",
    " $$ \\text{Label Smoothed Cross Entropy Loss} = - \\sum_i \\left( (1 - \\text{smoothing}) \\cdot 1_{\\{y_i\\}} + \\frac{\\text{smoothing}}{C-1} \\cdot 1_{\\{1 - y_i\\}} \\right) \\cdot \\log(p_i) $$\n",
    "\n",
    "Where:\n",
    "- $1_{\\{y_i\\}}$ is a binary indicator of whether class $i$ is the correct classification.\n",
    "- $p_i$ is the predicted probability of class $i$.\n",
    "- $C$ is the number of classes.\n",
    "- $\\text{smoothing}$  is the smoothing factor.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "5556bcc8-3ff2-4169-b316-0a7b17aa1d2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "class LabelSmoothedCrossEntropy(nn.Module):\n",
    "    def __init__(self, num_classes, smoothing):\n",
    "        super(LabelSmoothedCrossEntropy, self).__init__()\n",
    "        self.num_classes = num_classes\n",
    "        self.smoothing = smoothing\n",
    "        self.confidence = 1.0 - smoothing\n",
    "\n",
    "    def forward(self, input_logits, target):\n",
    "        log_probs = F.log_softmax(input_logits, dim=-1)\n",
    "        true_dist = torch.zeros_like(log_probs)\n",
    "        true_dist.fill_(self.smoothing / (self.num_classes - 1))\n",
    "        true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)\n",
    "        loss = -torch.sum(true_dist * log_probs) / input_logits.size(0)\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "262d50c6-de05-416f-8ad7-14792edf403c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize loss\n",
    "# criterion = nn.CrossEntropyLoss()\n",
    "criterion = LabelSmoothedCrossEntropy(num_classes=N_CLASSES, smoothing=LABEL_SMOOTHING)\n",
    "\n",
    "# Initialize optimizer\n",
    "optimizer = optim.Adam(model.parameters())\n",
    "\n",
    "# Initialize scheduler\n",
    "scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', patience=1, factor=1e-6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "34cd9acf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epoch):\n",
    "    model.train()\n",
    "    total_loss = 0.0\n",
    "\n",
    "    with tqdm(total=len(train_dataloader), desc=f\"Epoch {epoch}/Training\", unit=\"batch\") as pbar:\n",
    "        for i, (images, labels) in enumerate(train_dataloader):\n",
    "            images, labels = images.to(device), labels.to(device)\n",
    "            \n",
    "            optimizer.zero_grad()\n",
    "            output = model(images)\n",
    "            loss = criterion(output, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            \n",
    "            total_loss += loss.item()\n",
    "            pbar.update(1)\n",
    "            pbar.set_postfix(loss=total_loss / (i+1))\n",
    "\n",
    "    avg_loss = total_loss / len(train_dataloader.dataset)\n",
    "    scheduler.step(metrics=avg_loss)\n",
    "    print(f\"[Train][Epoch {epoch}] Average Loss: {avg_loss:.5f}, Updated Learning Rate: {optimizer.param_groups[0]['lr']}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "de2f38e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(epoch):\n",
    "    model.eval()\n",
    "    total_correct = 0\n",
    "    total_loss = 0.0\n",
    "\n",
    "    with tqdm(total=len(test_dataloader), desc=f\"Epoch {epoch}/Testing\", unit=\"batch\") as pbar: \n",
    "        with torch.no_grad():\n",
    "            for i, (images, labels) in enumerate(test_dataloader):\n",
    "                images, labels = images.to(device), labels.to(device)\n",
    "                \n",
    "                output = model(images)                \n",
    "                loss = criterion(output, labels)\n",
    "\n",
    "                total_loss += loss.item()\n",
    "                pbar.update(1)\n",
    "                pbar.set_postfix(loss=total_loss / (i+1))\n",
    "                \n",
    "                pred = output.detach().max(1)[1]\n",
    "                total_correct += pred.eq(labels.view_as(pred)).sum()\n",
    "            \n",
    "    avg_loss = total_loss / len(test_dataloader.dataset)\n",
    "    accuracy = total_correct / len(test_dataloader.dataset)\n",
    "    \n",
    "    print(f\"[Test][Epoch {epoch}] Loss: {avg_loss:.5f}, Accuracy: {accuracy:3f}\")\n",
    "    return avg_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "0c3cd024",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run():\n",
    "    for epoch in range(EPOCHS):\n",
    "        train(epoch)\n",
    "        val_loss = test(epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "04e628b2-39af-4e82-8d13-e94a3f4ffd86",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0c11f239d1f4fb3b768cddaaf21084a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00382, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "662c0565a36c465ca95cb6c1adef0d8a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] Loss: 0.00217, Accuracy: 0.802600\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "309c109cc13e4b12a18bcdd79c78159c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00281, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "08dd176c602a49baaff4f512ec85594f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] Loss: 0.00169, Accuracy: 0.909900\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ad93eac144540728ac3b144c19ffbb1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00242, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27e26e55bb9d4eb188266718b81906e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] Loss: 0.00157, Accuracy: 0.931300\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1690e5e614754011816a6df477a962b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00222, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df71b314dcd1471d9a1b05f8d8f0ac6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] Loss: 0.00150, Accuracy: 0.944300\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0507435269e47b9a9f442e8e32e69ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00210, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e2d5da09de8e4c07aa70b966e69e0faa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] Loss: 0.00147, Accuracy: 0.947900\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e9cd311ca7e46cca496dc4c6b31b910",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 5/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 5] Average Loss: 0.00201, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "312c09ea69a94db48032cad5f5b75d4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 5/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 5] Loss: 0.00144, Accuracy: 0.951600\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd78dcbc91dc4e40a8e02b2885598b9e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 6/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 6] Average Loss: 0.00195, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3e43faddac9f4c329960c30754f04242",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 6/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 6] Loss: 0.00141, Accuracy: 0.960300\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b3b56db0849d4ad18b03e9f23d3f0269",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 7/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 7] Average Loss: 0.00190, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c039797d35346a19ac2664c057105b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 7/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 7] Loss: 0.00140, Accuracy: 0.958600\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fca7b4035d71477dbf7c4c4a5aebe843",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 8/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 8] Average Loss: 0.00186, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f965ad0ca5b4c35afb207e1911cde4d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 8/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 8] Loss: 0.00138, Accuracy: 0.960300\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d785243b29ad4e12a4eca9f20196e8fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 9/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 9] Average Loss: 0.00182, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a001737b7354d9a974c1420f6b29e44",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 9/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 9] Loss: 0.00136, Accuracy: 0.965200\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40c633ca099446a38d24d9791a47938b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 10/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 10] Average Loss: 0.00180, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c91bd501a2f4c7e8d4dd04e1020ef72",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 10/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 10] Loss: 0.00136, Accuracy: 0.963700\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ea29868c3a144a1ebe9f662acc29c755",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 11/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 11] Average Loss: 0.00176, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ed21488cb4647bb870f7eec9ebc1f59",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 11/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 11] Loss: 0.00134, Accuracy: 0.966800\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78da0005a80c44b0874cf9a74a05f6a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 12/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 12] Average Loss: 0.00175, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0fdc5912d724b8ab94e95f2c4c4b2e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 12/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 12] Loss: 0.00133, Accuracy: 0.966700\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0e4bd6552414f5e9483d27124f578db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 13/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 13] Average Loss: 0.00173, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee528b7db74f40489f00e6294cd067d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 13/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 13] Loss: 0.00132, Accuracy: 0.969500\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4a39076329c423bbdf39303c834497f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 14/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 14] Average Loss: 0.00171, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e036a0c193c4f4c9d7878458791c4d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 14/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 14] Loss: 0.00132, Accuracy: 0.968700\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a91e09146fe748fd826d8f2d64b15ed1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 15/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 15] Average Loss: 0.00170, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1b7ecfcb2934d3da33eb6f00d1398aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 15/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 15] Loss: 0.00131, Accuracy: 0.967300\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bde402466cba48baaa742ea29fb2ea61",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 16/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 16] Average Loss: 0.00168, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7056bf5a384243449e700d4305189cee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 16/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 16] Loss: 0.00131, Accuracy: 0.970700\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1871b6282aba48ae96d5e8d19c4b2378",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 17/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 17] Average Loss: 0.00167, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ae33e348abd41beab5f6da73885a242",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 17/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 17] Loss: 0.00130, Accuracy: 0.971200\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31f95b3c85a04657bd2e44930f67b4d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 18/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 18] Average Loss: 0.00166, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45e63ce1b05c4084bd628d32215ff23b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 18/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 18] Loss: 0.00129, Accuracy: 0.973200\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48d6ad1e644c40fe8c8692fbea12b3db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 19/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 19] Average Loss: 0.00165, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d3ae356a07549e5b02cb176a4f09ab5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 19/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 19] Loss: 0.00129, Accuracy: 0.970300\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf7198dbdd29478daa1ec70c70872f8a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 20/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 20] Average Loss: 0.00163, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e68a6887e974b9ba68728631f093eb2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 20/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 20] Loss: 0.00128, Accuracy: 0.972400\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b65ecf438e443be9c81f549b1f0fbfc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 21/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 21] Average Loss: 0.00162, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c500ea27e71d4568921cff68e4b288f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 21/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 21] Loss: 0.00127, Accuracy: 0.973800\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00a4bfb371f94abeaf31c54f4daa35b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 22/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 22] Average Loss: 0.00161, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1774a4a47a0b476c8f5807a460096d49",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 22/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 22] Loss: 0.00128, Accuracy: 0.970500\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f008ccba8bfc4a7183654961c92529e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 23/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 23] Average Loss: 0.00161, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ec644d0984941e2b289beb43b0b43a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 23/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 23] Loss: 0.00127, Accuracy: 0.972500\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d94c949ce8a04f9b96c533db06794a13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 24/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 24] Average Loss: 0.00160, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3eade7b378a5459c82fc9ded90abc64e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 24/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 24] Loss: 0.00127, Accuracy: 0.973500\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73e2c75b70284a0db21d100bd6930b00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 25/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 25] Average Loss: 0.00159, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a8cafba7abd444dafb899d14ba41533",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 25/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 25] Loss: 0.00126, Accuracy: 0.976200\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61cdf23c88ee44cda3f2d8b308d56567",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 26/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 26] Average Loss: 0.00159, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "182d09cdb2c14204a60a8a359e597d7e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 26/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 26] Loss: 0.00126, Accuracy: 0.973800\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f47ed20a5987497ebb5b00e111fc2b2e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 27/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 27] Average Loss: 0.00158, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96eec5615a414d76b847f8c482db616e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 27/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 27] Loss: 0.00126, Accuracy: 0.974400\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ad291189406647d7a6c0c23285b52284",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 28/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 28] Average Loss: 0.00156, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74a61808111a4f138130a3ee9ac80d1b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 28/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 28] Loss: 0.00126, Accuracy: 0.974100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7da5d9195d94883bf21f71a61b99a0b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 29/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 29] Average Loss: 0.00156, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c991c5f1417b4bd2ae2a24a8d6867087",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 29/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 29] Loss: 0.00126, Accuracy: 0.973300\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a9ca92cc94744c50a4d3a8944813e6a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 30/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 30] Average Loss: 0.00156, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73e5f6fd55d44f9e8bdf8192c7a55b5d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 30/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 30] Loss: 0.00125, Accuracy: 0.975300\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02d868512fcd452fa6ddd3f04a517ff7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 31/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 31] Average Loss: 0.00155, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "80b5e37521d64c9daa6a412bc334b548",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 31/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 31] Loss: 0.00126, Accuracy: 0.973400\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a25954fa6554f759a864cccc40172bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 32/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 32] Average Loss: 0.00155, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b5c9dbe03a0844c6bc3ab58f719bdcda",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 32/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 32] Loss: 0.00125, Accuracy: 0.974700\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d3cba9833cc34a4385ecafe294d8b2de",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 33/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 33] Average Loss: 0.00154, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae1a6a3ee5734867977a98feef789a4a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 33/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 33] Loss: 0.00125, Accuracy: 0.975400\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ecbf39f9293f4cadb87203664d9bcdb6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 34/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 34] Average Loss: 0.00154, Updated Learning Rate: 0.001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c995e3b8e8724f67939ce66c4d03a0b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 34/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 34] Loss: 0.00125, Accuracy: 0.976000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d311f2ee8c24416c8c30d689550c42b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 35/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 35] Average Loss: 0.00154, Updated Learning Rate: 1e-09\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1771e56dd86046e8bf1e3d7f1eef32d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 35/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 35] Loss: 0.00124, Accuracy: 0.977100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "618b62a307824f54aa209068678590b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 36/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 36] Average Loss: 0.00153, Updated Learning Rate: 1e-09\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b72d4a225054c89aec7f8a4001e47dd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 36/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 36] Loss: 0.00124, Accuracy: 0.976600\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d9c9ef3fa2d4208b617957ada5c2cac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 37/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 37] Average Loss: 0.00152, Updated Learning Rate: 1e-09\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a35abc0068e64eebab40406cfdacb701",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 37/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 37] Loss: 0.00124, Accuracy: 0.976800\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "09204a4375a74bbaa23b83aa6c55c864",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 38/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 38] Average Loss: 0.00153, Updated Learning Rate: 1e-09\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16edb308a7244cfbbfe962e6ea82d1a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 38/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 38] Loss: 0.00124, Accuracy: 0.976600\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e058fcd7591943ada7cefea369429ee3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 39/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 39] Average Loss: 0.00153, Updated Learning Rate: 1e-09\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48149ed87b8842cca993f537ab5eba54",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 39/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 39] Loss: 0.00124, Accuracy: 0.976500\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3316b1ed297c4ec3b0aac5084ff4be97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 40/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 40] Average Loss: 0.00153, Updated Learning Rate: 1e-09\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b6fb6f906f848cebb39fbd58ee526b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 40/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 40] Loss: 0.00124, Accuracy: 0.976700\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0d253d4a2184181acae4805313f05c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 41/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 41] Average Loss: 0.00153, Updated Learning Rate: 1e-09\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc58a5e954604249bcf0b18258cf5517",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 41/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 41] Loss: 0.00124, Accuracy: 0.977500\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "402553a7f9fe41f4bbe4364cd794d062",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 42/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 42] Average Loss: 0.00153, Updated Learning Rate: 1e-09\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a1b5598306146bfb30a9e99a850c271",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 42/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 42] Loss: 0.00124, Accuracy: 0.976900\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b1118cd019444d39ad4b0229fd1fdc4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 43/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 43] Average Loss: 0.00153, Updated Learning Rate: 1e-09\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e344814f09064213a1642034133fcd9d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 43/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 43] Loss: 0.00124, Accuracy: 0.977200\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5f73ec0bb1c43a2bb9956979f80996d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 44/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 44] Average Loss: 0.00153, Updated Learning Rate: 1e-09\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "685b2a6a60f74be19f819ce5cbc2982d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 44/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 44] Loss: 0.00124, Accuracy: 0.976800\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69109b67b4434745a33742def2718dfd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 45/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 45] Average Loss: 0.00153, Updated Learning Rate: 1e-09\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7bbe2a002e674074996e4fbdd725b001",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 45/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 45] Loss: 0.00124, Accuracy: 0.977000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b47bc187770f410d872f6fb0bd760217",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 46/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 46] Average Loss: 0.00153, Updated Learning Rate: 1e-09\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae7d7e6db3384b2795079561e46215ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 46/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 46] Loss: 0.00124, Accuracy: 0.977100\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cdd4487606de4e34a0f00cb5f8fc0f27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 47/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 47] Average Loss: 0.00153, Updated Learning Rate: 1e-09\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3fa927c76ea943ef801d8475e45b84b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 47/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 47] Loss: 0.00124, Accuracy: 0.976400\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca27bc676ecb4a10ac69af89bb2ff09f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 48/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 48] Average Loss: 0.00153, Updated Learning Rate: 1e-09\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "11128249126e42108c9e2992c96d0040",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 48/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 48] Loss: 0.00124, Accuracy: 0.976700\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b5bba19ea48f452383424c7935f1f0a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 49/Training:   0%|          | 0/118 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 49] Average Loss: 0.00153, Updated Learning Rate: 1e-09\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee66d2977f1a43f9bb2d48a3db12fb2e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 49/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 49] Loss: 0.00124, Accuracy: 0.977400\n"
     ]
    }
   ],
   "source": [
    "run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfe42492",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd33fbf9-1624-4450-bf58-347bc6ca0be0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7ee87b2-3663-4643-aeaf-05b7be98dd42",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}