train_a_better_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from subgroups.dataloading_ffcv import make_dataloaders\n",
    "from subgroups.resnet9 import construct_rn9\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "import time\n",
    "\n",
    "from tqdm.auto import tqdm\n",
    "from torch.amp import GradScaler, autocast\n",
    "from torch.nn import CrossEntropyLoss\n",
    "from torch.optim import SGD, lr_scheduler\n",
    "\n",
    "from torch.optim import Adam\n",
    "\n",
    "from typing import List\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import wandb\n",
    "\n",
    "import os\n",
    "\n",
    "from torch.optim.lr_scheduler import ExponentialLR, StepLR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def full_iteration_ffcv(train_dataset: str = None, test_dataset: str = None, batch_size: int = None, num_workers: int = None, \n",
    "                        seed: int = None, alpha: float = None, num_classes: int = None, lr: float = None, epochs: int = None, \n",
    "                        momentum: float = None, weight_decay: float = None, label_smoothing: float = None, \n",
    "                        length: int = None, get_val_samples: bool = None, no_transform: bool = None,  \n",
    "                        return_sequential: bool = None, test_batch_size: int = None, optimizer: str = None, gamma: float = None, \n",
    "                        step_size: float = None, lr_scheduler: str = None, lr_tta: bool = None) -> tuple:\n",
    "    \n",
    "    loaders, _ = make_dataloaders(\n",
    "        train_dataset=train_dataset, test_dataset=test_dataset, batch_size=batch_size,\n",
    "        num_workers=num_workers, seed=seed, alpha=alpha, length=length, test_batch_size=test_batch_size, get_val_samples=get_val_samples, no_transform=no_transform, return_sequential=return_sequential,\n",
    "    )  \n",
    "        \n",
    "    model = construct_rn9(num_classes=num_classes).to(memory_format=torch.channels_last).cuda()\n",
    "    \n",
    "    average_acc, all_margins = train(\n",
    "        model, loaders, lr=lr, epochs=epochs, momentum=momentum,\n",
    "        weight_decay=weight_decay, label_smoothing=label_smoothing,\n",
    "        optimizer=optimizer, gamma=gamma, step_size=step_size, \n",
    "        lr_scheduler=lr_scheduler, lr_tta=lr_tta\n",
    "    )\n",
    "\n",
    "    return [average_acc, all_margins, loaders['train'].indices]\n",
    "\n",
    "def eval_test(model, loader, lr_tta: bool = False):\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        temp_acc, N = 0.0, 0.0\n",
    "        all_margins = []\n",
    "        for it, (ims, labs) in enumerate(loader['test']):\n",
    "            with autocast('cuda'):\n",
    "                out = model(ims)\n",
    "                if lr_tta:\n",
    "                    out += model(torch.fliplr(ims))\n",
    "                    out /= 2\n",
    "\n",
    "                pred = out.argmax(1).eq(labs)\n",
    "                total_correct = pred.sum().cpu().item()\n",
    "                temp_acc+=total_correct/ims.shape[0]\n",
    "                N+=1\n",
    "\n",
    "                class_logits = out[torch.arange(out.shape[0]), labs].clone()\n",
    "                out[torch.arange(out.shape[0]), labs] = -1000\n",
    "                next_classes = out.argmax(1)\n",
    "                class_logits -= out[torch.arange(out.shape[0]), next_classes]\n",
    "                all_margins.append(class_logits.cpu())\n",
    "\n",
    "    all_margins = torch.cat(all_margins)                \n",
    "    average_acc = temp_acc/N\n",
    "    return average_acc, all_margins\n",
    "\n",
    "\n",
    "def train(model: torch.nn.Module, loader: dict = None, lr: float = None, epochs: int = None, momentum: float = None,\n",
    "          weight_decay: float = None, label_smoothing: float = None, optimizer: str = None, gamma: float = None, \n",
    "          step_size: float = None, lr_scheduler: str = None, lr_tta: bool = None) -> list:\n",
    "    \n",
    "    if optimizer=='SGD':\n",
    "        opt = SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=weight_decay)\n",
    "    elif optimizer=='adam':\n",
    "        opt = Adam(model.parameters(), lr=lr)\n",
    "\n",
    "    if lr_scheduler=='exponential':\n",
    "        scheduler = ExponentialLR(opt, gamma=gamma)\n",
    "    elif lr_scheduler=='step':\n",
    "        scheduler = StepLR(opt, step_size=step_size, gamma=gamma)\n",
    "\n",
    "    scaler = GradScaler('cuda')\n",
    "    \n",
    "    loss_fn = CrossEntropyLoss(label_smoothing=label_smoothing)\n",
    "\n",
    "    model.train()\n",
    "\n",
    "    for ep in tqdm(range(epochs)):\n",
    "        \n",
    "        for it, (ims, labs) in enumerate(loader['train']):\n",
    "            opt.zero_grad(set_to_none=True)\n",
    "\n",
    "            with autocast('cuda'):\n",
    "                out = model(ims)\n",
    "                loss = loss_fn(out, labs)\n",
    "            \n",
    "            scaler.scale(loss).backward()\n",
    "            scaler.step(opt)\n",
    "            scaler.update()\n",
    "\n",
    "        scheduler.step()\n",
    "\n",
    "        #average_acc, all_margins = eval_test(model, loader, lr_tta=lr_tta)\n",
    "\n",
    "#        wandb.log({\n",
    "#                \"epoch\": ep, \n",
    "#                \"test_accuracy\": average_acc,\n",
    "#                \"event\": 'model_update'\n",
    "#        })\n",
    "       # model.train()\n",
    "    \n",
    "    average_acc, all_margins = eval_test(model, loader, lr_tta=lr_tta)\n",
    "    #average_acc, all_margins = None, None\n",
    "    return average_acc, all_margins"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Specify your starting parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "hyperparameters = {\n",
    "    \"train_dataset\": './CIFAR10/cifar10_train_subset_binaryLabels.beton',\n",
    "    \"test_dataset\": './CIFAR10/cifar10_val_subset_binaryLabels.beton',\n",
    "    \"batch_size\": 32,\n",
    "    \"num_workers\": 1, \n",
    "    \"seed\": 1,\n",
    "    \"alpha\": 0.1,\n",
    "    \"num_classes\": 2,\n",
    "    \"lr\": 0.45,\n",
    "    \"epochs\": 205,\n",
    "    \"momentum\": 0.9,\n",
    "    \"weight_decay\": 5e-4,\n",
    "    \"label_smoothing\": 0.0,\n",
    "    \"test_batch_size\": 2000,\n",
    "    \"optimizer\": 'SGD',\n",
    "    \"lr_scheduler\": 'step',\n",
    "    \"step_size\": 60,\n",
    "    \"gamma\": 0.25,\n",
    "    \"lr_tta\": False, \n",
    "    \"length\": 25000,\n",
    "    \"get_val_samples\": False,\n",
    "    \"no_transform\": False,\n",
    "    \"return_sequential\": False\n",
    "    }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check your signal to noise ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 205/205 [01:09<00:00,  2.95it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 205/205 [01:09<00:00,  2.96it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 205/205 [01:07<00:00,  3.04it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 205/205 [01:06<00:00,  3.09it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 205/205 [01:06<00:00,  3.08it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 205/205 [00:56<00:00,  3.65it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 205/205 [00:55<00:00,  3.67it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 205/205 [00:56<00:00,  3.63it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 205/205 [00:56<00:00,  3.65it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 205/205 [00:56<00:00,  3.62it/s]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAG0CAYAAAAYQdwgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGhElEQVR4nO3de1wU5eI/8M9yW7ntEggsqAimghzBCyqu5tGURCK/ecROFgepPFkKppJmlAlaJ0zL6/HSFay8lCWmlhpi4i9FJZS8IV5CoeTiyWAVFXB5fn90mOMCKosrO+Dn/XrNK2bmmZlnnp11Pz1zUwghBIiIiIhkxMLcFSAiIiKqiwGFiIiIZIcBhYiIiGSHAYWIiIhkhwGFiIiIZIcBhYiIiGSHAYWIiIhkhwGFiIiIZIcBhYiIiGSHAYWIiIhkx6iA4u3tDYVCUW+IiYkBAFy/fh0xMTFwcXGBg4MDIiIiUFJSYrCOgoIChIeHw87ODm5ubpgxYwZu3Lhhuj0iIiKiFs/KmMJZWVnQ6/XS+LFjx/DII4/giSeeAABMmzYN3377LTZs2AC1Wo3Y2FiMHj0ae/fuBQDo9XqEh4dDo9Fg3759KCoqwrhx42BtbY2333670fWoqanBhQsX4OjoCIVCYcwuEBERkZkIIXD58mV4enrCwuIOfSTiLkyZMkU8+OCDoqamRpSVlQlra2uxYcMGaX5ubq4AIDIzM4UQQnz33XfCwsJCFBcXS2VWrlwpVCqVqKysbPR2CwsLBQAOHDhw4MCBQwscCgsL7/hbb1QPys2qqqrw+eefIy4uDgqFAtnZ2aiurkZISIhUxs/PD15eXsjMzET//v2RmZmJgIAAuLu7S2VCQ0MxceJEHD9+HL169WpwW5WVlaisrJTGxX9fwFxYWAiVStXUXSAiIqJmpNPp0KFDBzg6Ot6xbJMDyqZNm1BWVoZnnnkGAFBcXAwbGxs4OTkZlHN3d0dxcbFU5uZwUju/dt6tJCUlYc6cOfWmq1QqBhQiIqIWpjGXZzT5Lp6PP/4YYWFh8PT0bOoqGi0+Ph7l5eXSUFhYeM+3SURERObTpB6U8+fPY+fOndi4caM0TaPRoKqqCmVlZQa9KCUlJdBoNFKZgwcPGqyr9i6f2jINUSqVUCqVTakqERERtUBN6kFJTk6Gm5sbwsPDpWlBQUGwtrZGenq6NC0vLw8FBQXQarUAAK1Wi6NHj6K0tFQqk5aWBpVKBX9//6buAxEREbUyRveg1NTUIDk5GdHR0bCy+t/iarUa48ePR1xcHJydnaFSqTB58mRotVr0798fADB8+HD4+/sjKioK8+fPR3FxMWbNmoWYmBj2kBARNZFer0d1dbW5q0EEa2trWFpammRdRgeUnTt3oqCgAM8991y9eYsWLYKFhQUiIiJQWVmJ0NBQrFixQppvaWmJrVu3YuLEidBqtbC3t0d0dDTmzp17d3tBRHQfEkKguLgYZWVl5q4KkcTJyQkajeaun1OmELX37LYgOp0OarUa5eXlvIuHiO5bRUVFKCsrg5ubG+zs7PjgSjIrIQSuXr2K0tJSODk5wcPDo14ZY36/m3ybMRERmY9er5fCiYuLi7mrQwQAsLW1BQCUlpbCzc3trk738GWBREQtUO01J3Z2dmauCZGh2mPybq+LYkAhImrBeFqH5MZUxyQDChEREckOAwoREbUKiYmJ6Nmz523LnDt3DgqFAjk5Oc1SJ+DPHoVNmzY12/ZMxdvbG4sXLzbb9hlQiIhaGYWieQe5euaZZzBq1CiDaR06dEBRURG6d+9+y+VM/cNcVFSEsLAwk63vfsG7eIiI6L5haWl521erNJZer4dCoYCFxZ3/P98U27sfsQeFiIia1ZAhQzB58mRMnToVDzzwANzd3fHhhx+ioqICzz77LBwdHdG5c2ds27ZNWiYlJcXgPW8AsGnTpltekJmYmIjVq1fjm2++gUKhgEKhwO7du+94imfIkCE4f/48pk2bJi138/Y3b94Mf39/KJVKFBQUICsrC4888gjatm0LtVqNwYMH49ChQwbrvPkUT+32N27ciIcffhh2dnbo0aMHMjMzb9leQggkJibCy8sLSqUSnp6eeOmll6T5lZWVmD59Otq1awd7e3sEBwdj9+7dBuv48ccfMWjQINja2qJDhw546aWXUFFRIc0vLS3FyJEjYWtrCx8fH6xZs8aoOtwLDChERNTsVq9ejbZt2+LgwYOYPHkyJk6ciCeeeAIDBgzAoUOHMHz4cERFReHq1atNWv/06dPx97//HSNGjEBRURGKioowYMCAOy63ceNGtG/fHnPnzpWWq3X16lW88847+Oijj3D8+HG4ubnh8uXLiI6Oxo8//oj9+/ejS5cuePTRR3H58uXbbuf111/H9OnTkZOTg65du+Kpp57CjRs3Giz79ddfY9GiRXj//fdx+vRpbNq0CQEBAdL82NhYZGZmYv369Thy5AieeOIJjBgxAqdPnwYAnD17FiNGjEBERASOHDmCL774Aj/++CNiY2OldTzzzDMoLCzEDz/8gK+++gorVqwweG/enepwT4gWqLy8XAAQ5eXl5q6KUYD6AxFRU1y7dk2cOHFCXLt2rd68hv6tuZeDsQYPHiweeughafzGjRvC3t5eREVFSdOKiooEAJGZmSmEECI5OVmo1WqD9aSmpoqbf8YSEhJEjx49pPHo6Gjx+OOPGyyTn58vAIjDhw/fsn4dO3YUixYtMpiWnJwsAIicnJzb7pterxeOjo5iy5Yt0jQAIjU11WD7H330kTT/+PHjAoDIzc1tcJ3vvfee6Nq1q6iqqqo37/z588LS0lL89ttvBtOHDRsm4uPjhRBCjB8/XkyYMMFg/v/7f/9PWFhYiGvXrom8vDwBQBw8eFCan5ubKwBI7XC7OtR1u2PTmN9v9qAQEVGzCwwMlP62tLSEi4uLwf+Ru7u7A4DB/8Wbm42NjUG9AaCkpATPP/88unTpArVaDZVKhStXrqCgoOC267p5PbWPhL/Vvj7xxBO4du0aOnXqhOeffx6pqalSb8vRo0eh1+vRtWtXODg4SENGRgbOnj0LAPj555+RkpJiMD80NBQ1NTXIz89Hbm4urKysEBQUJG3Tz8/P4JTa7epwr/AiWSIianbW1tYG4wqFwmBa7bUfNTU1AAALCwuIOq+Oa+43ONva2ta75iU6Ohq///47lixZgo4dO0KpVEKr1aKqquq267rdvtbVoUMH5OXlYefOnUhLS8OkSZOwYMECZGRk4MqVK7C0tER2dna9x8o7ODgAAK5cuYIXXnihwWtGvLy8cOrUqTvu++3qUPezNBUGFCIikj1XV1dcvnwZFRUVsLe3B4A7PsvExsYGer3e6G0Zs9zevXuxYsUKPProowCAwsJC/Oc//zF6m3dia2uLkSNHYuTIkYiJiYGfnx+OHj2KXr16Qa/Xo7S0FIMGDWpw2d69e+PEiRPo3Llzg/P9/Pxw48YNZGdno2/fvgCAvLy8em/JvlUdevfubdJ9rcWAQkREshccHAw7Ozu89tpreOmll3DgwAGkpKTcdhlvb2/s2LEDeXl5cHFxgVqtbtS2vL29sWfPHowdOxZKpRJt27a9ZdkuXbrgs88+Q58+faDT6TBjxgzphXmmkpKSAr1eL7XB559/DltbW3Ts2BEuLi6IjIzEuHHj8N5776FXr164ePEi0tPTERgYiPDwcMycORP9+/dHbGws/vnPf8Le3h4nTpxAWloa/v3vf8PX1xcjRozACy+8gJUrV8LKygpTp0412I/b1eFe4TUoREQke87Ozvj888/x3XffISAgAOvWrUNiYuJtl3n++efh6+uLPn36wNXVFXv37m3UtubOnYtz587hwQcfhKur623Lfvzxx/jjjz/Qu3dvREVF4aWXXoKbm1tjd6tRnJyc8OGHH2LgwIEIDAzEzp07sWXLFukt1snJyRg3bhxefvll+Pr6YtSoUcjKyoKXlxeAP693ycjIwKlTpzBo0CD06tULs2fPhqenp7SN5ORkeHp6YvDgwRg9ejQmTJhgsB93qsO9oBB1T+q1ADqdDmq1GuXl5VCpVOauTqM1dLt+y2t9IpKD69evIz8/Hz4+PmjTpo25q0Mkud2xaczvN3tQiIiISHYYUIiIiEh2GFCIiIhIdhhQiIiISHYYUIiIiEh2GFCIiIhIdvigNjPjrcdERET1sQeFiIiIZIcBhYiIiGSHAYWIiFq1Z555BqNGjWr27aakpMDJycmoZYYMGYKpU6fek/q0NLwGhYiotbnDO2pa/PZaiCeffFJ6y3Fjbdy4EdbW1tK4t7c3pk6del+GFgYUIiIyq6qqKtjY2Ji7GiZna2tr9JuNnZ2d71FtWh6e4iEiomY1ZMgQxMbGYurUqWjbti1CQ0MBAAsXLkRAQADs7e3RoUMHTJo0CVeuXJGWqz1lsmPHDnTr1g0ODg4YMWIEioqKpDJ6vR5xcXFwcnKCi4sLXnnlFdR9J25lZaX01uE2bdrgoYceQlZWljR/9+7dUCgU2LFjB3r16gVbW1sMHToUpaWl2LZtG7p16waVSoWnn34aV69eveV+1j3Fk5iYiJ49e+Kzzz6Dt7c31Go1xo4di8uXLxu0TW1vyZAhQ3D+/HlMmzYNCoUCioZu+2zFGFCaIjGRXZpERHdh9erVsLGxwd69e7Fq1SoAgIWFBZYuXYrjx49j9erV2LVrF1555RWD5a5evYp3330Xn332Gfbs2YOCggJMnz5dmv/ee+8hJSUFn3zyCX788UdcunQJqampBut45ZVX8PXXX2P16tU4dOgQOnfujNDQUFy6dMmgXGJiIv79739j3759KCwsxN///ncsXrwYa9euxbfffovvv/8ey5YtM2q/z549i02bNmHr1q3YunUrMjIyMG/evAbLbty4Ee3bt8fcuXNRVFRkEMTuBwwoJqJQ1B+IiKhhXbp0wfz58+Hr6wtfX18AwNSpU/Hwww/D29sbQ4cOxVtvvYUvv/zSYLnq6mqsWrUKffr0Qe/evREbG4v09HRp/uLFixEfH4/Ro0ejW7duWLVqFdRqtTS/oqICK1euxIIFCxAWFgZ/f398+OGHsLW1xccff2ywrbfeegsDBw5Er169MH78eGRkZGDlypXo1asXBg0ahDFjxuCHH34war9ramqQkpKC7t27Y9CgQYiKijKo/82cnZ1haWkJR0dHaDQaaDQao7bV0jGgEBFRswsKCqo3befOnRg2bBjatWsHR0dHREVF4ffffzc4jWJnZ4cHH3xQGvfw8EBpaSkAoLy8HEVFRQgODpbmW1lZoU+fPtL42bNnUV1djYEDB0rTrK2t0a9fP+Tm5hrUJzAwUPrb3d0ddnZ26NSpk8G02m03lre3NxwdHRusPxliQCEiomZnb29vMH7u3Dk89thjCAwMxNdff43s7GwsX74cwJ8X0da6+Q4XAFAoFPWuMTGVm7elUCga3HZNTU2T19nUddwvGFBMITERCUg0dy2IiFqs7Oxs1NTU4L333kP//v3RtWtXXLhwwah1qNVqeHh44MCBA9K0GzduIDs7Wxp/8MEHpWtfalVXVyMrKwv+/v53vyMmZmNjA71eb+5qmAVvMyYiIrPr3LkzqqursWzZMowcOdLg4lljTJkyBfPmzUOXLl3g5+eHhQsXoqysTJpvb2+PiRMnYsaMGXB2doaXlxfmz5+Pq1evYvz48SbcI9Pw9vbGnj17MHbsWCiVSrRt29bcVWo27EEhIiKz69GjBxYuXIh33nkH3bt3x5o1a5CUlGT0el5++WVERUUhOjoaWq0Wjo6O+Nvf/mZQZt68eYiIiEBUVBR69+6NM2fOYMeOHXjggQdMtTsmM3fuXJw7dw4PPvggXF1dzV2dZqUQ9+rk3T2k0+mgVqtRXl4OlUrV/BWovcX4pv8mzgHm1DnNU7dlG3tnT8v7RIiouV2/fh35+fnw8fFBmzZtzF0dIsntjk1jfr/Zg0JERESyw4BCREREssOAYkIJ4N08REREpsC7eO4hPk2WiIioadiDQkTUgrXA+xyolTPVMWl0QPntt9/wj3/8Ay4uLrC1tUVAQAB++ukng4rNnj0bHh4esLW1RUhICE6fPm2wjkuXLiEyMhIqlQpOTk4YP368wRsriYjo9mqfSHq7t+kSmUPtMVn3qbnGMuoUzx9//IGBAwfi4YcfxrZt2+Dq6orTp08b3Ds+f/58LF26FKtXr4aPjw/eeOMNhIaG4sSJE9LtRpGRkSgqKkJaWhqqq6vx7LPPYsKECVi7du1d7cw9xzcYE5FMWFpawsnJSXqPi52dHRQ8r0xmJITA1atXUVpaCicnJ1haWt7V+owKKO+88w46dOiA5ORkaZqPj49B5RYvXoxZs2bh8ccfBwB8+umncHd3x6ZNmzB27Fjk5uZi+/btyMrKkl7gtGzZMjz66KN499134enpWW+7lZWVqKyslMZ1Op1xe0lE1ArVvt2WL5sjOXFycjLJm5eNCiibN29GaGgonnjiCWRkZKBdu3aYNGkSnn/+eQBAfn4+iouLERISIi2jVqsRHByMzMxMjB07FpmZmXBycjJ4u2RISAgsLCxw4MCBek/8A4CkpCTMmTOnqft477BHhYjMSKFQwMPDA25ubqiurjZ3dYhgbW191z0ntYwKKL/88gtWrlyJuLg4vPbaa8jKysJLL70EGxsbREdHo7i4GMCfr6C+mbu7uzSvuLgYbm5uhpWwsoKzs7NUpq74+HjExcVJ4zqdDh06dDCm6i1aQ722vC6OiGpZWlqa7EeBSC6MCig1NTXo06cP3n77bQBAr169cOzYMaxatQrR0dH3pIIAoFQqoVQq79n65YankYmI6H5n1F08Hh4e9V5H3a1bNxQUFAD43/nQkpISgzIlJSXSPI1GU+986Y0bN3Dp0iWTnLMiIiKils+ogDJw4EDk5eUZTDt16hQ6duwI4M8LZjUaDdLT06X5Op0OBw4cgFarBQBotVqUlZUhOztbKrNr1y7U1NQgODi4yTtCRERErYdRp3imTZuGAQMG4O2338bf//53HDx4EB988AE++OADAH9esDV16lS89dZb6NKli3SbsaenJ0aNGgXgzx6XESNG4Pnnn8eqVatQXV2N2NhYjB07tsE7eIiIiOj+Y1RA6du3L1JTUxEfH4+5c+fCx8cHixcvRmRkpFTmlVdeQUVFBSZMmICysjI89NBD2L59u8Erl9esWYPY2FgMGzYMFhYWiIiIwNKlS023V0RERNSiKUQLfE6yTqeDWq1GeXk5VCpV8234NrcVJ950F/ScZnhhYMv71IiI6H5nzO8338VDREREssOAQkRERLLDgEJERESyw4BCREREssOAQkRERLLDgEJERESyw4BCREREssOAQkRERLLDgEJERESyw4BCREREssOAQkRERLLDgEJERESyw4BCREREssOAQkRERLLDgEJERESyw4ByDyQgEQlINHc1iIiIWiwGFCIiIpIdBhQiIiKSHQYUIiIikh0GFCIiIpIdBhQiIiKSHQYUIiIikh0GFCIiIpIdBhQiIiKSHQYUIiIikh0GFCIiIpIdBhQiIiKSHQYUIiIikh0GFCIiIpIdK3NXoKVKnGPuGhAREbVe7EG5hxKQiAQkmrsaRERELQ4DChEREckOAwoRERHJDgMKERERyQ4DChEREckOAwoRERHJDgMKERERyQ4DChEREckOAwoRERHJDgMKERERyQ4fdd9CKRSG40KYpx5ERET3glE9KImJiVAoFAaDn5+fNP/69euIiYmBi4sLHBwcEBERgZKSEoN1FBQUIDw8HHZ2dnBzc8OMGTNw48YN0+wNERERtQpG96D85S9/wc6dO/+3Aqv/rWLatGn49ttvsWHDBqjVasTGxmL06NHYu3cvAECv1yM8PBwajQb79u1DUVERxo0bB2tra7z99tsm2B0iIiJqDYwOKFZWVtBoNPWml5eX4+OPP8batWsxdOhQAEBycjK6deuG/fv3o3///vj+++9x4sQJ7Ny5E+7u7ujZsyfefPNNzJw5E4mJibCxsbn7PSIiIqIWz+iLZE+fPg1PT0906tQJkZGRKCgoAABkZ2ejuroaISEhUlk/Pz94eXkhMzMTAJCZmYmAgAC4u7tLZUJDQ6HT6XD8+PFbbrOyshI6nc5gICIiotbLqIASHByMlJQUbN++HStXrkR+fj4GDRqEy5cvo7i4GDY2NnBycjJYxt3dHcXFxQCA4uJig3BSO7923q0kJSVBrVZLQ4cOHYypNhEREbUwRp3iCQsLk/4ODAxEcHAwOnbsiC+//BK2trYmr1yt+Ph4xMXFSeM6nY4hhYiIqBW7q+egODk5oWvXrjhz5gw0Gg2qqqpQVlZmUKakpES6ZkWj0dS7q6d2vKHrWmoplUqoVCqDgYiIiFqvuwooV65cwdmzZ+Hh4YGgoCBYW1sjPT1dmp+Xl4eCggJotVoAgFarxdGjR1FaWiqVSUtLg0qlgr+//91UhYiIiFoRo07xTJ8+HSNHjkTHjh1x4cIFJCQkwNLSEk899RTUajXGjx+PuLg4ODs7Q6VSYfLkydBqtejfvz8AYPjw4fD390dUVBTmz5+P4uJizJo1CzExMVAqlfdkB4mIiKjlMSqg/Prrr3jqqafw+++/w9XVFQ899BD2798PV1dXAMCiRYtgYWGBiIgIVFZWIjQ0FCtWrJCWt7S0xNatWzFx4kRotVrY29sjOjoac+fONe1eERERUYumEKLlPSRdp9NBrVajvLy8ea9HSUz8359zGr/YHCTesczdanmfIhER3W+M+f3mywKJiIhIdhhQiIiISHYYUIiIiEh2GFCIiIhIdhhQiIiISHYYUIiIiEh2GFCIiIhIdhhQiIiISHYYUIiIiEh2GFCIiIhIdhhQmkECEpHQDI+7JyIiai0YUIiIiEh2GFCIiIhIdhhQiIiISHYYUIiIiEh2rMxdgRYhMdHcNSAiIrqvsAeFiIiIZIcBhYiIiGSHp3haCYWi/jQhmr8eREREpsAeFCIiIpIdBhQiIiKSHQYUIiIikh1eg9KK1b0uhdekEBFRS8EeFCIiIpIdBhQiIiKSHZ7iaUYJSJT+nnPT30RERGSIPShEREQkOwwoREREJDsMKERERCQ7DChEREQkOwwoREREJDsMKERERCQ7DChEREQkOwwoREREJDsMKERERCQ7DChEREQkOwwoREREJDsMKERERCQ7DChEREQkOwwoREREJDsMKERERCQ7DChEREQkO3cVUObNmweFQoGpU6dK065fv46YmBi4uLjAwcEBERERKCkpMViuoKAA4eHhsLOzg5ubG2bMmIEbN27cTVWIiIioFWlyQMnKysL777+PwMBAg+nTpk3Dli1bsGHDBmRkZODChQsYPXq0NF+v1yM8PBxVVVXYt28fVq9ejZSUFMyePbvpe0FEREStSpMCypUrVxAZGYkPP/wQDzzwgDS9vLwcH3/8MRYuXIihQ4ciKCgIycnJ2LdvH/bv3w8A+P7773HixAl8/vnn6NmzJ8LCwvDmm29i+fLlqKqqMs1eERERUYvWpIASExOD8PBwhISEGEzPzs5GdXW1wXQ/Pz94eXkhMzMTAJCZmYmAgAC4u7tLZUJDQ6HT6XD8+PEGt1dZWQmdTmcwEBERUetlZewC69evx6FDh5CVlVVvXnFxMWxsbODk5GQw3d3dHcXFxVKZm8NJ7fzaeQ1JSkrCnDlzjK0qERERtVBG9aAUFhZiypQpWLNmDdq0aXOv6lRPfHw8ysvLpaGwsLDZtt2aKBT1ByIiIjkyKqBkZ2ejtLQUvXv3hpWVFaysrJCRkYGlS5fCysoK7u7uqKqqQllZmcFyJSUl0Gg0AACNRlPvrp7a8doydSmVSqhUKoOBiIiIWi+jAsqwYcNw9OhR5OTkSEOfPn0QGRkp/W1tbY309HRpmby8PBQUFECr1QIAtFotjh49itLSUqlMWloaVCoV/P39TbRbRERE1JIZdQ2Ko6MjunfvbjDN3t4eLi4u0vTx48cjLi4Ozs7OUKlUmDx5MrRaLfr37w8AGD58OPz9/REVFYX58+ejuLgYs2bNQkxMDJRKpYl2i4iIiFoyoy+SvZNFixbBwsICERERqKysRGhoKFasWCHNt7S0xNatWzFx4kRotVrY29sjOjoac+fONXVViIiIqIVSCCGEuSthLJ1OB7VajfLy8ua5HiUxsf6ku7ypaA7qr9McWt6nT0RELZUxv998Fw8RERHJDgMKERERyQ4DChEREckOAwoRERHJDgMKERERyQ4DChEREckOAwoRERHJDgMKERERyY7JnyTbqvz3AW13+1A2IiIiMg57UIiIiEh2GFDMJAGJSJDJ4+6JiIjkhgGFiIiIZIcBhYiIiGSHAYWIiIhkhwGFiIiIZIcBhYiIiGSHAYWIiIhkhwGFiIiIZIcBhYiIiGSHAYWIiIhkhwGFiIiIZIcBhYiIiGSHAYWIiIhkhwGFiIiIZIcBhYiIiGSHAcXMEpCIBCSauxpERESywoBCREREsmNl7gqQeSkUhuNCmKceREREN2MPikzwVA8REdH/MKAQERGR7DCgEBERkewwoBAREZHsMKAQERGR7DCgEBERkewwoBAREZHsMKAQERGR7DCgEBERkewwoBAREZHs8FH3dEd8HD4RETU3BhQyUDeMEBERmQNP8RAREZHsGBVQVq5cicDAQKhUKqhUKmi1Wmzbtk2af/36dcTExMDFxQUODg6IiIhASUmJwToKCgoQHh4OOzs7uLm5YcaMGbhx44Zp9oaIiIhaBaMCSvv27TFv3jxkZ2fjp59+wtChQ/H444/j+PHjAIBp06Zhy5Yt2LBhAzIyMnDhwgWMHj1aWl6v1yM8PBxVVVXYt28fVq9ejZSUFMyePdu0e0VEREQtmkKIu7vk0dnZGQsWLMCYMWPg6uqKtWvXYsyYMQCAkydPolu3bsjMzET//v2xbds2PPbYY7hw4QLc3d0BAKtWrcLMmTNx8eJF2NjYNGqbOp0OarUa5eXlUKlUd1P920tM/PM/c+7dJuqag8Tm21gT8SJZIiJqCmN+v5t8DYper8f69etRUVEBrVaL7OxsVFdXIyQkRCrj5+cHLy8vZGZmAgAyMzMREBAghRMACA0NhU6nk3phGlJZWQmdTmcwEBERUetldEA5evQoHBwcoFQq8eKLLyI1NRX+/v4oLi6GjY0NnJycDMq7u7ujuLgYAFBcXGwQTmrn1867laSkJKjVamno0KGDsdUmIiKiFsTogOLr64ucnBwcOHAAEydORHR0NE6cOHEv6iaJj49HeXm5NBQWFt7T7REREZF5Gf0cFBsbG3Tu3BkAEBQUhKysLCxZsgRPPvkkqqqqUFZWZtCLUlJSAo1GAwDQaDQ4ePCgwfpq7/KpLdMQpVIJpVJpbFWJiIiohbrr56DU1NSgsrISQUFBsLa2Rnp6ujQvLy8PBQUF0Gq1AACtVoujR4+itLRUKpOWlgaVSgV/f/+7rQoRERG1Ekb1oMTHxyMsLAxeXl64fPky1q5di927d2PHjh1Qq9UYP3484uLi4OzsDJVKhcmTJ0Or1aJ///4AgOHDh8Pf3x9RUVGYP38+iouLMWvWLMTExLCHhIiIiCRGBZTS0lKMGzcORUVFUKvVCAwMxI4dO/DII48AABYtWgQLCwtERESgsrISoaGhWLFihbS8paUltm7diokTJ0Kr1cLe3h7R0dGYO3euafeKiIiIWrS7fg6KOfA5KObV8o4YIiKSg2Z5DgoRERHRvcKAQkRERLJj9G3GRApF/Wk87UNERKbEHhQiIiKSHQYUIiIikh0GFCIiIpIdBhQiIiKSHQYUIiIikh0GFCIiIpIdBhQiIiKSHQYUIiIikh0GFCIiIpIdBhSZSkAiElrAiwOJiIjuBQYUIiIikh0GFCIiIpIdBhQiIiKSHQYUIiIikh0rc1eADPHCWCIiIgYUMhGFwnBcCPPUg4iIWgee4iEiIiLZYUAhIiIi2WFAISIiItlhQCEiIiLZYUAhIiIi2WFAISIiItlhQCEiIiLZYUAhIiIi2WFAISIiItlhQCEiIiLZYUAhIiIi2WFAISIiItlhQCEiIiLZYUAhIiIi2WFAISIiItlhQCEiIiLZYUBpIRKQiAQkmrsaREREzYIBhYiIiGSHAYWIiIhkhwGFiIiIZMfK3BWg2+N1J0REdD9iDwoRERHJDgMKERERyY5RASUpKQl9+/aFo6Mj3NzcMGrUKOTl5RmUuX79OmJiYuDi4gIHBwdERESgpKTEoExBQQHCw8NhZ2cHNzc3zJgxAzdu3Lj7vSEiIqJWwaiAkpGRgZiYGOzfvx9paWmorq7G8OHDUVFRIZWZNm0atmzZgg0bNiAjIwMXLlzA6NGjpfl6vR7h4eGoqqrCvn37sHr1aqSkpGD27Nmm2ysiIiJq0RRCCNHUhS9evAg3NzdkZGTgr3/9K8rLy+Hq6oq1a9dizJgxAICTJ0+iW7duyMzMRP/+/bFt2zY89thjuHDhAtzd3QEAq1atwsyZM3Hx4kXY2Njccbs6nQ5qtRrl5eVQqVRNrf6tJSYajs4x/Saaak4LuWi26UcVERG1Vsb8ft/VNSjl5eUAAGdnZwBAdnY2qqurERISIpXx8/ODl5cXMjMzAQCZmZkICAiQwgkAhIaGQqfT4fjx4w1up7KyEjqdzmAgIiKi1qvJAaWmpgZTp07FwIED0b17dwBAcXExbGxs4OTkZFDW3d0dxcXFUpmbw0nt/Np5DUlKSoJarZaGDh06NLXaRERE1AI0OaDExMTg2LFjWL9+vSnr06D4+HiUl5dLQ2Fh4T3fJhEREZlPkx7UFhsbi61bt2LPnj1o3769NF2j0aCqqgplZWUGvSglJSXQaDRSmYMHDxqsr/Yun9oydSmVSiiVyqZUtdWpfXBbS7kWhYiIqCmM6kERQiA2NhapqanYtWsXfHx8DOYHBQXB2toa6enp0rS8vDwUFBRAq9UCALRaLY4ePYrS0lKpTFpaGlQqFfz9/e9mX0hGFIr6AxERUWMZ1YMSExODtWvX4ptvvoGjo6N0zYharYatrS3UajXGjx+PuLg4ODs7Q6VSYfLkydBqtejfvz8AYPjw4fD390dUVBTmz5+P4uJizJo1CzExMewlISIiIgBGBpSVK1cCAIYMGWIwPTk5Gc888wwAYNGiRbCwsEBERAQqKysRGhqKFStWSGUtLS2xdetWTJw4EVqtFvb29oiOjsbcuXPvbk+IiIio1TAqoDTmkSlt2rTB8uXLsXz58luW6dixI7777jtjNk1ERET3Eb6Lh4iIiGSHAYWIiIhkhwGFiIiIZIcBpYVKQKL0TBQiIqLWhgGFiIiIZKdJT5Ilaoq6D2vjG4+JiOhW2INCREREssOAQkRERLLDgEJERESyw4BCREREssOAQkRERLLDgEJERESyw4DSwvGBbURE1BrxOSgkK3xWChERAexBISIiIhliQCEiIiLZ4SmeBiTOMXcNiIiI7m/sQSEiIiLZYUAhIiIi2eEpHjKbunfsEBER1WIPChEREckOAwoRERHJDgMKERERyQ4DChEREckOAwoRERHJDgNKK8GXBhIRUWvCgEJERESyw4BCREREssOAQkRERLLDgEJERESyw4BCREREssOAQkRERLLDgNJK8bZjIiJqyfg2Y5K1ht54LETz14OIiJoXe1CIiIhIdhhQiIiISHZ4iodanLqnfXjKh4io9WEPChEREckOe1BaGd65Q0RErQEDCrV4vNOHiKj14SkeIiIikh2jA8qePXswcuRIeHp6QqFQYNOmTQbzhRCYPXs2PDw8YGtri5CQEJw+fdqgzKVLlxAZGQmVSgUnJyeMHz8eV65cuasdISIiotbD6IBSUVGBHj16YPny5Q3Onz9/PpYuXYpVq1bhwIEDsLe3R2hoKK5fvy6ViYyMxPHjx5GWloatW7diz549mDBhQtP3gm6p9omyvDaFiIhaEqOvQQkLC0NYWFiD84QQWLx4MWbNmoXHH38cAPDpp5/C3d0dmzZtwtixY5Gbm4vt27cjKysLffr0AQAsW7YMjz76KN599114enrexe4QERFRa2DSa1Dy8/NRXFyMkJAQaZparUZwcDAyMzMBAJmZmXBycpLCCQCEhITAwsICBw4caHC9lZWV0Ol0BgMRERG1XiYNKMXFxQAAd3d3g+nu7u7SvOLiYri5uRnMt7KygrOzs1SmrqSkJKjVamno0KGDKatNREREMtMi7uKJj49HeXm5NBQWFpq7SiRzCoXhQERELYtJA4pGowEAlJSUGEwvKSmR5mk0GpSWlhrMv3HjBi5duiSVqUupVEKlUhkMRERE1HqZNKD4+PhAo9EgPT1dmqbT6XDgwAFotVoAgFarRVlZGbKzs6Uyu3btQk1NDYKDg01ZHaqDd/MQEVFLYfRdPFeuXMGZM2ek8fz8fOTk5MDZ2RleXl6YOnUq3nrrLXTp0gU+Pj5444034OnpiVGjRgEAunXrhhEjRuD555/HqlWrUF1djdjYWIwdO5Z38BARERGAJgSUn376CQ8//LA0HhcXBwCIjo5GSkoKXnnlFVRUVGDChAkoKyvDQw89hO3bt6NNmzbSMmvWrEFsbCyGDRsGCwsLREREYOnSpSbYHSIiImoNFEK0vLeW6HQ6qNVqlJeX35PrURIViSZfp5zMuQ9P87S8o5yIqPUx5ve7RdzFQ0RERPcXBhQiIiKSHQYUIiIikh0GFCIiIpIdBhQiIiKSHQYUIiIikh2jn4NCLV/t02Tvp9uNG3ofD289JiKSL/agEB+BT0REssMeFLpv1e1VYY8KEZF8MKDcx9hrQkREcsVTPERERCQ77EEhMgIvtiUiah7sQSEiIiLZYUAhIiIi2WFAISIiItlhQCEiIiLZYUAhCR/YRkREcsGAQkRERLLD24yJ/ou3EBMRyQd7UKjReAqIiIiaCwMKERERyQ4DChEREckOr0Ehuo2GrkshIqJ7jz0oVA+vNSEiInNjDwrdEcPK7TWml4V3AxERGYcBhW6JwYSIiMyFAYWMVhtc5jDANFrdXhb2qBAR3R6vQSEiIiLZYQ8KmczNp4TYu0JERHeDPShEREQkO+xBoSbjRbRERHSvMKAQmcG9fDEhL8glotaAAYVIJpoSLPikWyJqrXgNChEREckOAwoRERHJDk/x0D1R92FufLib8e7ldSpERHLHgEL3FO/0MT8GHSJqiRhQiFoQXhRLRPcLBhRqVjzVI0/sZSEiuWFAIbNgUDEvufXE8NktRFQXAwqZ1a0upq3FAGM+jQkxTXlWC8MHETUGAwoRNRkfLkdE94pZn4OyfPlyeHt7o02bNggODsbBgwfNWR0yowQk3vaOnzvNb25yq49cKBT1B3Ouh4haLrP1oHzxxReIi4vDqlWrEBwcjMWLFyM0NBR5eXlwc3MzV7VIZuqGgMaEgjudFuL1Ly0TTxUR3V8UQpjnax4cHIy+ffvi3//+NwCgpqYGHTp0wOTJk/Hqq6/edlmdTge1Wo3y8nKoVCqT1y1RkWjydZL53ek6lzsFl4bCUd2yDD/Np6F/uZpy3Uxj7mBqapnGbN9UTHXNENG9ZMzvt1l6UKqqqpCdnY34+HhpmoWFBUJCQpCZmVmvfGVlJSorK6Xx8vJyAH/u6L1Qico7F6IW51X8ebzV/XTrTq8dr6uho6Ju2f+VafjYfBVJBuPz/rt83el1599uWTmorVtz1uluTh81VxlTLnez//4T2CzbbmhbanXTlmuMxqy7KdtqaL1NreO92lbd5UxVv8bUp7nap/Z3u1F9I8IMfvvtNwFA7Nu3z2D6jBkzRL9+/eqVT0hIEAA4cODAgQMHDq1gKCwsvGNWaBF38cTHxyMuLk4ar6mpwaVLl+Di4gKFia+e0+l06NChAwoLC+/J6aP7DdvTtNiepsX2NC22p2m1xvYUQuDy5cvw9PS8Y1mzBJS2bdvC0tISJSUlBtNLSkqg0WjqlVcqlVAqlQbTnJyc7mUVoVKpWs0BIQdsT9Nie5oW29O02J6m1draU93I83hmuc3YxsYGQUFBSE9Pl6bV1NQgPT0dWq3WHFUiIiIiGTHbKZ64uDhER0ejT58+6NevHxYvXoyKigo8++yz5qoSERERyYTZAsqTTz6JixcvYvbs2SguLkbPnj2xfft2uLu7m6tKAP48nZSQkFDvlBI1DdvTtNiepsX2NC22p2nd7+1ptuegEBEREd2KWR91T0RERNQQBhQiIiKSHQYUIiIikh0GFCIiIpIdBpSbLF++HN7e3mjTpg2Cg4Nx8OBBc1epRdizZw9GjhwJT09PKBQKbNq0yWC+EAKzZ8+Gh4cHbG1tERISgtOnT5unsi1AUlIS+vbtC0dHR7i5uWHUqFHIy8szKHP9+nXExMTAxcUFDg4OiIiIqPfgQ/rTypUrERgYKD3sSqvVYtu2bdJ8tuXdmTdvHhQKBaZOnSpNY5s2XmJiIhQKhcHg5+cnzb+f25IB5b+++OILxMXFISEhAYcOHUKPHj0QGhqK0tJSc1dN9ioqKtCjRw8sX768wfnz58/H0qVLsWrVKhw4cAD29vYIDQ3F9evXm7mmLUNGRgZiYmKwf/9+pKWlobq6GsOHD0dFRYVUZtq0adiyZQs2bNiAjIwMXLhwAaNHjzZjreWrffv2mDdvHrKzs/HTTz9h6NChePzxx3H8+HEAbMu7kZWVhffffx+BgYEG09mmxvnLX/6CoqIiafjxxx+lefd1W5rk7X+tQL9+/URMTIw0rtfrhaenp0hKSjJjrVoeACI1NVUar6mpERqNRixYsECaVlZWJpRKpVi3bp0ZatjylJaWCgAiIyNDCPFn+1lbW4sNGzZIZXJzcwUAkZmZaa5qtigPPPCA+Oijj9iWd+Hy5cuiS5cuIi0tTQwePFhMmTJFCMHj01gJCQmiR48eDc6739uSPSgAqqqqkJ2djZCQEGmahYUFQkJCkJmZacaatXz5+fkoLi42aFu1Wo3g4GC2bSOV//ed587OzgCA7OxsVFdXG7Spn58fvLy82KZ3oNfrsX79elRUVECr1bIt70JMTAzCw8MN2g7g8dkUp0+fhqenJzp16oTIyEgUFBQAYFu2iLcZ32v/+c9/oNfr6z3F1t3dHSdPnjRTrVqH4uJiAGiwbWvn0a3V1NRg6tSpGDhwILp37w7gzza1sbGp98JMtumtHT16FFqtFtevX4eDgwNSU1Ph7++PnJwctmUTrF+/HocOHUJWVla9eTw+jRMcHIyUlBT4+vqiqKgIc+bMwaBBg3Ds2LH7vi0ZUIhkLCYmBseOHTM4J03G8/X1RU5ODsrLy/HVV18hOjoaGRkZ5q5Wi1RYWIgpU6YgLS0Nbdq0MXd1WrywsDDp78DAQAQHB6Njx4748ssvYWtra8aamR9P8QBo27YtLC0t610ZXVJSAo1GY6ZatQ617ce2NV5sbCy2bt2KH374Ae3bt5emazQaVFVVoayszKA82/TWbGxs0LlzZwQFBSEpKQk9evTAkiVL2JZNkJ2djdLSUvTu3RtWVlawsrJCRkYGli5dCisrK7i7u7NN74KTkxO6du2KM2fO3PfHJwMK/vzHKygoCOnp6dK0mpoapKenQ6vVmrFmLZ+Pjw80Go1B2+p0Ohw4cIBtewtCCMTGxiI1NRW7du2Cj4+PwfygoCBYW1sbtGleXh4KCgrYpo1UU1ODyspKtmUTDBs2DEePHkVOTo409OnTB5GRkdLfbNOmu3LlCs6ePQsPDw8en+a+Slcu1q9fL5RKpUhJSREnTpwQEyZMEE5OTqK4uNjcVZO9y5cvi8OHD4vDhw8LAGLhwoXi8OHD4vz580IIIebNmyecnJzEN998I44cOSIef/xx4ePjI65du2bmmsvTxIkThVqtFrt37xZFRUXScPXqVanMiy++KLy8vMSuXbvETz/9JLRardBqtWastXy9+uqrIiMjQ+Tn54sjR46IV199VSgUCvH9998LIdiWpnDzXTxCsE2N8fLLL4vdu3eL/Px8sXfvXhESEiLatm0rSktLhRD3d1syoNxk2bJlwsvLS9jY2Ih+/fqJ/fv3m7tKLcIPP/wgANQboqOjhRB/3mr8xhtvCHd3d6FUKsWwYcNEXl6eeSstYw21JQCRnJwslbl27ZqYNGmSeOCBB4SdnZ3429/+JoqKisxXaRl77rnnRMeOHYWNjY1wdXUVw4YNk8KJEGxLU6gbUNimjffkk08KDw8PYWNjI9q1ayeefPJJcebMGWn+/dyWCiGEME/fDREREVHDeA0KERERyQ4DChEREckOAwoRERHJDgMKERERyQ4DChEREckOAwoRERHJDgMKERERyQ4DChEREckOAwqRiZw7dw4KhQI5OTnmropJpKSk1HvNu5wMGTIEU6dONXc1yISa8h3icdB6MaAQycju3buhUCjwwAMP4Pr16wbzsrKyoFAooFAomqUuTz75JE6dOtUs22qKjRs34s033zR3NYjoHmFAIdnS6/WoqakxdzXMwtHREampqQbTPv74Y3h5ed31uquqqhpVztbWFm5ubne9PVOrrb+zszMcHR3NXJvm19jPj6ilY0AhkxgyZAhiY2MRGxsLtVqNtm3b4o033sDNr3qqrKzE9OnT0a5dO9jb2yM4OBi7d++W5teeUti8eTP8/f2hVCpRUFBQb1t//PEHIiMj4erqCltbW3Tp0gXJycnS/JkzZ6Jr166ws7NDp06d8MYbb6C6ulqan5iYiJ49e+KTTz6Bl5cXHBwcMGnSJOj1esyfPx8ajQZubm7417/+ZbBdhUKBlStXIiwsDLa2tujUqRO++uqr27bLsWPHEBYWBgcHB7i7uyMqKgr/+c9/7tie0dHR+OSTT6Txa9euYf369YiOjjYo9/vvv+Opp55Cu3btYGdnh4CAAKxbt86gTO1nM3XqVLRt2xahoaEAgM2bN6NLly5o06YNHn74YaxevRoKhQJlZWUA6p/iqW23zz77DN7e3lCr1Rg7diwuX77c4D7odDrY2tpi27ZtBtNTU1Ph6OiIq1evAmj85/XRRx/Bx8cHbdq0kfbr5q79zz77DH369IGjoyM0Gg2efvpplJaWSvNre6fS09PRp08f2NnZYcCAAcjLyzOo35YtW9C3b1+0adMGbdu2xd/+9jdp3p2O4YYsXLgQAQEBsLe3R4cOHTBp0iRcuXLFoMzevXsxZMgQ2NnZ4YEHHkBoaCj++OMPaT8b+vwyMjLQr18/KJVKeHh44NVXX8WNGzekdX711VcICAiAra0tXFxcEBISgoqKCqkt+vXrB3t7ezg5OWHgwIE4f/58g/WvPe3y5ZdfYtCgQbC1tUXfvn1x6tQpZGVloU+fPnBwcEBYWBguXrwoLVdTU4O5c+eiffv2UCqV6NmzJ7Zv326w7oMHD6JXr15o06YN+vTpg8OHD9fbvrHfoRUrVkjHtbu7O8aMGXO7j4fkzMwvK6RWYvDgwcLBwUFMmTJFnDx5Unz++efCzs5OfPDBB1KZf/7zn2LAgAFiz5494syZM2LBggVCqVSKU6dOCSGESE5OFtbW1mLAgAFi79694uTJk6KioqLetmJiYkTPnj1FVlaWyM/PF2lpaWLz5s3S/DfffFPs3btX5Ofni82bNwt3d3fxzjvvSPMTEhKEg4ODGDNmjDh+/LjYvHmzsLGxEaGhoWLy5Mni5MmT4pNPPhEADN5oDUC4uLiIDz/8UOTl5YlZs2YJS0tLceLECSGEEPn5+QKAOHz4sBBCiD/++EO4urqK+Ph4kZubKw4dOiQeeeQR8fDDD9+yHWvfDJ2XlyeUSqU4f/68EEKIzz77TPTo0UOkpqaKm7+2v/76q1iwYIE4fPiwOHv2rFi6dKmwtLQUBw4cqPfZzJgxQ5w8eVKcPHlS/PLLL8La2lpMnz5dnDx5Uqxbt060a9dOABB//PGH9Hmo1ep67TZ69Ghx9OhRsWfPHqHRaMRrr712y/0ZM2aM+Mc//mEwLSIiwmBaYz4ve3t7MWLECHHo0CHx888/S/t18xt0P/74Y/Hdd9+Js2fPiszMTKHVakVYWFi9tg0ODha7d+8Wx48fF4MGDRIDBgyQymzdulVYWlqK2bNnixMnToicnBzx9ttvS/PvdAw3ZNGiRWLXrl0iPz9fpKenC19fXzFx4kRp/uHDh4VSqRQTJ04UOTk54tixY2LZsmXi4sWLt/z8fv31V2FnZycmTZokcnNzRWpqqmjbtq1ISEgQQghx4cIFYWVlJRYuXCjy8/PFkSNHxPLly8Xly5dFdXW1UKvVYvr06eLMmTPixIkTIiUlRTrW6qo9rv38/MT27dvFiRMnRP/+/UVQUJAYMmSI+PHHH8WhQ4dE586dxYsvvigtt3DhQqFSqcS6devEyZMnxSuvvCKsra2ltrp8+bJwdXUVTz/9tDh27JjYsmWL6NSpk9HfoZuPg6ysLGFpaSnWrl0rzp07Jw4dOiSWLFlyy8+G5I0BhUxi8ODBolu3bqKmpkaaNnPmTNGtWzchhBDnz58XlpaW4rfffjNYbtiwYSI+Pl4I8ecPIgCRk5Nz222NHDlSPPvss42u24IFC0RQUJA0npCQIOzs7IROp5OmhYaGCm9vb6HX66Vpvr6+IikpSRoHYPAPsBBCBAcHSz82dQPKm2++KYYPH25QvrCwUAogDan9Ef3jjz/EqFGjxJw5c4QQQjz88MNiyZIl9QJKQ8LDw8XLL78sjQ8ePFj06tXLoMzMmTNF9+7dDaa9/vrrdwwoddttxowZIjg4+JZ1SU1NFQ4ODlLQLC8vF23atBHbtm275TINfV7W1taitLTUoFzdgFJXVlaWACAuX74shPhf2+7cuVMq8+233woA4tq1a0IIIbRarYiMjGxwfY05hhtjw4YNwsXFRRp/6qmnxMCBA29ZvqHP77XXXhO+vr4G37fly5cLBwcHodfrRXZ2tgAgzp07V299v//+uwAgdu/e3aj61h7XH330kTRt3bp1AoBIT0+XpiUlJQlfX19p3NPTU/zrX/8yWFffvn3FpEmThBBCvP/++8LFxUVqeyGEWLlypdHfoZuPg6+//lqoVCqDY5RaLp7iIZPp37+/wQWcWq0Wp0+fhl6vx9GjR6HX69G1a1c4ODhIQ0ZGBs6ePSstY2Njg8DAwNtuZ+LEiVi/fj169uyJV155Bfv27TOY/8UXX2DgwIHQaDRwcHDArFmz6p0q8vb2Nrh+wd3dHf7+/rCwsDCYdvMpgtp9qjuem5vbYD1//vln/PDDDwb76+fnBwAG+3wrzz33HFJSUvDLL78gMzMTkZGR9cro9Xq8+eabCAgIgLOzMxwcHLBjx456+xsUFGQwnpeXh759+xpM69ev3x3rVLfdPDw86rXRzR599FFYW1tj8+bNAICvv/4aKpUKISEhUpnGfF4dO3aEq6vrbeuWnZ2NkSNHwsvLC46Ojhg8eDAA1FvXzceXh4cHAEj7kJOTg2HDhjW4/sYew3Xt3LkTw4YNQ7t27eDo6IioqCj8/vvv0imu222zVt3PLzc3F1qt1uD7NnDgQFy5cgW//vorevTogWHDhiEgIABPPPEEPvzwQ+mUkbOzM5555hmEhoZi5MiRWLJkCYqKim67fcCw3dzd3QEAAQEBBtNq21Gn0+HChQsYOHCgwToGDhwofV9yc3MRGBgonbID6n+/jP0OPfLII+jYsSM6deqEqKgorFmzRmpnankYUKhZXLlyBZaWlsjOzkZOTo405ObmYsmSJVI5W1vbO96lEhYWhvPnz2PatGm4cOEChg0bhunTpwOA9EP+6KOPYuvWrTh8+DBef/31ehcWWltbG4wrFIoGp93NRbpXrlzByJEjDfY3JycHp0+fxl//+tc7Lh8WFoZr165h/PjxGDlyJFxcXOqVWbBgAZYsWYKZM2fihx9+QE5ODkJDQ+vtr729fZP342bGtpGNjQ3GjBmDtWvXAgDWrl2LJ598ElZWVgAa/3ndqf4VFRUIDQ2FSqXCmjVrkJWVJV1kfLvPvvZYq90HW1vbW26jscfwzc6dO4fHHnsMgYGB+Prrr5GdnY3ly5cb1Ot226xl7OdnaWmJtLQ0bNu2Df7+/li2bBl8fX2Rn58PAEhOTkZmZiYGDBiAL774Al27dsX+/ftvu86G2q3uNFNf1G7sd8jR0RGHDh3CunXr4OHhgdmzZ6NHjx7SdVXUsjCgkMkcOHDAYHz//v3o0qULLC0t0atXL+j1epSWlqJz584Gg0ajMXpbrq6uiI6Oxueff47Fixfjgw8+AADs27cPHTt2xOuvv44+ffqgS5cut7z4rynq/iO+f/9+dOvWrcGyvXv3xvHjx+Ht7V1vnxvzg2NlZYVx48Zh9+7deO655xoss3fvXjz++OP4xz/+gR49eqBTp06NujXY19cXP/30k8G0rKysOy7XFJGRkdi+fTuOHz+OXbt2GfQEmerzOnnyJH7//XfMmzcPgwYNgp+f3217dm4lMDAQ6enpDc5ryjGcnZ2NmpoavPfee+jfvz+6du2KCxcuNHqbt9KtWzdkZmYaXIS+d+9eODo6on379gD+DAwDBw7EnDlzcPjwYdjY2BjcGdarVy/Ex8dj37596N69uxQiTUGlUsHT0xN79+41mL537174+/tL+3DkyBGD2+nrfr+a8h2ysrJCSEgI5s+fjyNHjuDcuXPYtWuXyfaNmg8DCplMQUEB4uLikJeXh3Xr1mHZsmWYMmUKAKBr166IjIzEuHHjsHHjRuTn5+PgwYNISkrCt99+a9R2Zs+ejW+++QZnzpzB8ePHsXXrVikkdOnSBQUFBVi/fj3Onj2LpUuX1rtd925s2LABn3zyCU6dOoWEhAQcPHgQsbGxDZaNiYnBpUuX8NRTTyErKwtnz57Fjh078Oyzz0Kv1zdqe2+++SYuXrwo3blRV5cuXZCWloZ9+/YhNzcXL7zwAkpKSu643hdeeAEnT57EzJkzcerUKXz55ZdISUkBAJM/Z+Wvf/0rNBoNIiMj4ePjg+DgYIP6m+Lz8vLygo2NDZYtW4ZffvkFmzdvbtIzUhISErBu3TokJCQgNzcXR48exTvvvAOgacdw586dUV1dLdXrs88+w6pVqwzKxMfHIysrC5MmTcKRI0dw8uRJrFy58rZ3qkyaNAmFhYWYPHkyTp48iW+++QYJCQmIi4uDhYUFDhw4gLfffhs//fQTCgoKsHHjRly8eBHdunVDfn4+4uPjkZmZifPnz+P777/H6dOnbxm0m2rGjBl455138MUXXyAvLw+vvvoqcnJypH8Tnn76aSgUCjz//PM4ceIEvvvuO7z77rsG6zD2O7R161YsXboUOTk5OH/+PD799FPU1NTA19fXpPtGzYMBhUxm3LhxuHbtGvr164eYmBhMmTIFEyZMkOYnJydj3LhxePnll+Hr64tRo0YhKyvL6Gd72NjYID4+HoGBgfjrX/8KS0tLrF+/HgDwf//3f5g2bRpiY2PRs2dP7Nu3D2+88YbJ9nHOnDlYv349AgMD8emnn2LdunXS/xHWVft/kHq9HsOHD0dAQACmTp0KJycng2td7rSvbdu2vWVomDVrFnr37o3Q0FAMGTIEGo0Go0aNuuN6fXx88NVXX2Hjxo0IDAzEypUr8frrrwMAlEplo+rWWAqFAk899RR+/vnnetfRmOrzcnV1RUpKCjZs2AB/f3/Mmzev3o9dYwwZMgQbNmzA5s2b0bNnTwwdOhQHDx6U5ht7DPfo0QMLFy7EO++8g+7du2PNmjVISkoyKNO1a1d8//33+Pnnn9GvXz9otVp888030mmwhrRr1w7fffcdDh48iB49euDFF1/E+PHjMWvWLAB/9mDs2bMHjz76KLp27YpZs2bhvffeQ1hYGOzs7HDy5ElERESga9eumDBhAmJiYvDCCy8Y3V6389JLLyEuLg4vv/wyAgICsH37dunWdgBwcHDAli1bcPToUfTq1Quvv/66FAZrGfsdcnJywsaNGzF06FB069YNq1atwrp16/CXv/zFpPtGzUMhbu4jJGqiIUOGoGfPnli8eLG5q3LPKBQKpKamNioAtET/+te/sGrVKhQWFpq7KkREuHVEJ6JWbcWKFejbty9cXFywd+9eLFiw4Janq4iImhsDCtF96vTp03jrrbdw6dIleHl54eWXX0Z8fLy5q0VEBICneIiIiEiGeJEsERERyQ4DChEREckOAwoRERHJDgMKERERyQ4DChEREckOAwoRERHJDgMKERERyQ4DChEREcnO/weO91vGXJBf6QAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = []\n",
    "for i in range(5):\n",
    "\n",
    "    out = full_iteration_ffcv(train_dataset=hyperparameters['train_dataset'], test_dataset=hyperparameters['test_dataset'], batch_size=hyperparameters['batch_size'], num_workers=hyperparameters['num_workers'], \n",
    "                                seed=i, alpha=hyperparameters['alpha'], num_classes=hyperparameters['num_classes'], lr=hyperparameters['lr'], epochs=hyperparameters['epochs'], \n",
    "                                momentum=hyperparameters['momentum'], weight_decay=hyperparameters['weight_decay'], label_smoothing=hyperparameters['label_smoothing'], \n",
    "                                length=hyperparameters['length'], get_val_samples=hyperparameters['get_val_samples'], no_transform=hyperparameters['no_transform'],  \n",
    "                                return_sequential=hyperparameters['return_sequential'], test_batch_size=hyperparameters['test_batch_size'], optimizer=hyperparameters['optimizer'], gamma=hyperparameters['gamma'], \n",
    "                                step_size=hyperparameters['step_size'], lr_scheduler=hyperparameters['lr_scheduler'], lr_tta=hyperparameters['lr_tta']) \n",
    "    x.append(out[1])\n",
    "\n",
    "x2 = []\n",
    "for i in range(5):\n",
    "\n",
    "    out = full_iteration_ffcv(train_dataset=hyperparameters['train_dataset'], test_dataset=hyperparameters['test_dataset'], batch_size=hyperparameters['batch_size'], num_workers=hyperparameters['num_workers'], \n",
    "                                seed=0, alpha=hyperparameters['alpha'], num_classes=hyperparameters['num_classes'], lr=hyperparameters['lr'], epochs=hyperparameters['epochs'], \n",
    "                                momentum=hyperparameters['momentum'], weight_decay=hyperparameters['weight_decay'], label_smoothing=hyperparameters['label_smoothing'], \n",
    "                                length=hyperparameters['length'], get_val_samples=hyperparameters['get_val_samples'], no_transform=hyperparameters['no_transform'],  \n",
    "                                return_sequential=hyperparameters['return_sequential'], test_batch_size=hyperparameters['test_batch_size'], optimizer=hyperparameters['optimizer'], gamma=hyperparameters['gamma'], \n",
    "                                step_size=hyperparameters['step_size'], lr_scheduler=hyperparameters['lr_scheduler'], lr_tta=hyperparameters['lr_tta']) \n",
    "    x2.append(out[1])\n",
    "\n",
    "plt.hist([np.var([x[i][s] for i in range(len(x))]) for s in range(10000)], bins = 100, color='blue', label = 'multi train seeds')\n",
    "plt.hist([np.var([x2[i][s] for i in range(len(x2))]) for s in range(10000)], alpha = 0.5, bins = 100, color = 'red', label = 'random init')\n",
    "plt.xlabel('per sample Margin variance across models')\n",
    "plt.legend()\n",
    "None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### - Tune your hyperparameters, visualize using wandb\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with wandb\n",
    "def full_iteration_ffcv(train_dataset: str = None, test_dataset: str = None, batch_size: int = None, num_workers: int = None, \n",
    "                        seed: int = None, alpha: float = None, num_classes: int = None, lr: float = None, epochs: int = None, \n",
    "                        momentum: float = None, weight_decay: float = None, label_smoothing: float = None, \n",
    "                        length: int = None, get_val_samples: bool = None, no_transform: bool = None, set_seed: bool = None, \n",
    "                        return_sequential: bool = None, test_batch_size: int = None, optimizer: str = None, gamma: float = 0.99, step_size: float = None, \n",
    "                        lr_scheduler: str = None, p_dropout: float = 0) -> tuple:\n",
    "    \n",
    "    loaders, train_indices = make_dataloaders(\n",
    "        train_dataset=train_dataset, test_dataset=test_dataset, batch_size=batch_size,\n",
    "        num_workers=num_workers, seed=seed, alpha=alpha, length=length, test_batch_size=test_batch_size, get_val_samples=get_val_samples, no_transform=no_transform, return_sequential=return_sequential,\n",
    "    ) # the val dataloader should return a larger batch\n",
    "    \n",
    "    print(train_indices[:50])\n",
    "\n",
    "    if set_seed:\n",
    "        torch.manual_seed(42)          \n",
    "        torch.cuda.manual_seed(42)\n",
    "         \n",
    "        \n",
    "    model = construct_rn9(num_classes=num_classes).to(memory_format=torch.channels_last).cuda()\n",
    "    print(\"Initial model weights:\")\n",
    "    for name, param in model.named_parameters():\n",
    "        print(f\"{name}: {param.data}\")\n",
    "    \n",
    "    train(\n",
    "        model, loaders, lr=lr, epochs=epochs, momentum=momentum,\n",
    "        weight_decay=weight_decay, label_smoothing=label_smoothing,\n",
    "        optimizer=optimizer, gamma=gamma, step_size=step_size, lr_scheduler=lr_scheduler\n",
    "    )\n",
    "\n",
    "\n",
    "def eval_test(model, loader, loss_fn):\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        temp_loss = 0\n",
    "        temp_acc = 0\n",
    "        N = 0\n",
    "        for it, (ims, labs) in enumerate(loader['test']):\n",
    "            with autocast('cuda'):\n",
    "                out = model(ims)\n",
    "                loss = loss_fn(out, labs)\n",
    "                pred = out.argmax(1).eq(labs)\n",
    "                total_correct = pred.sum().cpu().item()\n",
    "                temp_loss+=loss.item()\n",
    "                temp_acc+=total_correct/ims.shape[0]\n",
    "                N+=1\n",
    "    average_loss, average_acc = temp_loss/N, temp_acc/N\n",
    "    model.train()\n",
    "    return average_loss, average_acc\n",
    "\n",
    "\n",
    "def train(model: torch.nn.Module, loader: dict = None, lr: float = None, epochs: int = None, momentum: float = None,\n",
    "          weight_decay: float = None, label_smoothing: float = None, optimizer: str = None, gamma: float = 0.99, step_size: float = None, lr_scheduler: str = None) -> dict:\n",
    "    \n",
    "    if optimizer=='SGD':\n",
    "        opt = SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=weight_decay)\n",
    "    elif optimizer=='adam':\n",
    "        opt = Adam(model.parameters(), lr=lr)\n",
    "\n",
    "    if lr_scheduler=='exponential':\n",
    "        scheduler = ExponentialLR(opt, gamma=gamma)\n",
    "    elif lr_scheduler=='step':\n",
    "        scheduler = StepLR(opt, step_size=step_size, gamma=gamma)\n",
    "\n",
    "\n",
    "    #iters_per_epoch = len(loader['train'])\n",
    "    #lr_peak_epoch = 62\n",
    "    # Cyclic LR with single triangle\n",
    "    #lr_schedule = np.interp(np.arange((epochs+1) * iters_per_epoch),\n",
    "    #                        [0, lr_peak_epoch * iters_per_epoch, epochs * iters_per_epoch],\n",
    "    #                        [0, 1, 0])\n",
    "    #scheduler = lr_scheduler.LambdaLR(opt, lr_schedule.__getitem__)\n",
    "\n",
    "    scaler = GradScaler('cuda')\n",
    "    \n",
    "    loss_fn = CrossEntropyLoss(label_smoothing=label_smoothing)\n",
    "\n",
    "    model.train()\n",
    "\n",
    "    for ep in tqdm(range(epochs)):\n",
    "        \n",
    "        for it, (ims, labs) in enumerate(loader['train']):\n",
    "            opt.zero_grad(set_to_none=True)\n",
    "\n",
    "            with autocast('cuda'):\n",
    "                out = model(ims)\n",
    "                loss = loss_fn(out, labs)\n",
    "                pred = out.argmax(1).eq(labs)\n",
    "                total_correct = pred.sum().cpu().item()\n",
    "                train_loss, train_accuracy = loss.item(), total_correct/ims.shape[0]\n",
    "            \n",
    "            if (it % epochs)==0:\n",
    "                test_loss, test_accuracy = eval_test(model, loader, loss_fn)\n",
    "            else:\n",
    "                test_loss, test_accuracy = None, None\n",
    "\n",
    "            wandb.log({\n",
    "                \"epoch\": ep,\n",
    "                \"train_loss\": train_loss,\n",
    "                \"train_accuracy\": train_accuracy,\n",
    "                \"test_loss\": test_loss,\n",
    "                \"test_accuracy\": test_accuracy,\n",
    "                \"event\": 'model_update'\n",
    "            })\n",
    "            \n",
    "            scaler.scale(loss).backward()\n",
    "            scaler.step(opt)\n",
    "            scaler.update()\n",
    "\n",
    "        scheduler.step()\n",
    "   \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_experiment(name, ID, values, run_name):\n",
    "    hyperparameters = {\n",
    "        \"train_dataset\": '/home/gridsan/djuna/TsaiMadry_shared/datamodels_clustering/CIFAR10/cifar10_train_subset_binaryLabels.beton',\n",
    "        \"test_dataset\": '/home/gridsan/djuna/TsaiMadry_shared/datamodels_clustering/CIFAR10/cifar10_val_subset_binaryLabels.beton',\n",
    "        \"batch_size\": 32,\n",
    "        \"num_workers\": 1, \n",
    "        \"seed\": 5,\n",
    "        \"alpha\": 0.1,\n",
    "        \"num_classes\": 2,\n",
    "        \"lr\": 0.45,\n",
    "        \"epochs\": 200,#300,\n",
    "        \"momentum\": 0.9,\n",
    "        \"weight_decay\": 5e-4,\n",
    "        \"label_smoothing\": 0.0,\n",
    "        \"set_seed\": False,\n",
    "        \"test_batch_size\": 2000,\n",
    "        \"optimizer\": 'SGD',\n",
    "        \"lr_scheduler\": 'step',\n",
    "        \"step_size\": 60,\n",
    "        \"gamma\": 0.25, #0.25,\n",
    "        \"p_dropout\": 0\n",
    "    }\n",
    "\n",
    "    outdir = f\"/home/gridsan/djuna/TsaiMadry_shared/datamodels_clustering/wandb_logs/Experiment_{name}\" \n",
    "    if not os.path.exists(outdir):\n",
    "        os.makedirs(outdir)\n",
    "\n",
    "    hyperparameters[name] = values[ID]\n",
    "\n",
    "    # Initialize wandb with the hyperparameters\n",
    "    wandb.init(\n",
    "        project=\"cifar10_binary\",\n",
    "        name=f\"{name}_{values[ID]}\",\n",
    "        group=f\"Experiment_{run_name}\",\n",
    "        config=hyperparameters,\n",
    "        mode=\"offline\",  # Ensure offline mode is enabled\n",
    "        dir=outdir # Optional: specify log directory\n",
    "    )\n",
    "\n",
    "    config = wandb.config\n",
    "    full_iteration_ffcv(config.train_dataset, config.test_dataset, batch_size=config.batch_size, \n",
    "                        seed=config.seed, alpha=config.alpha, num_classes=config.num_classes, \n",
    "                        lr=config.lr, epochs=config.epochs, momentum=config.momentum, \n",
    "                        weight_decay=config.weight_decay, label_smoothing=config.label_smoothing, \n",
    "                        test_batch_size=config.test_batch_size, optimizer=config.optimizer,\n",
    "                        set_seed = config.set_seed, gamma = config.gamma, \n",
    "                        lr_scheduler = config.lr_scheduler, step_size = config.step_size, \n",
    "                        length=25000, get_val_samples=False, no_transform=False, \n",
    "                        num_workers=1, return_sequential=False, p_dropout = config.p_dropout) \n",
    "\n",
    "    try:\n",
    "        wandb.finish()\n",
    "    except KeyboardInterrupt:\n",
    "        wandb.finish()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.18.4"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "W&B syncing is set to <code>`offline`<code> in this directory.  <br/>Run <code>`wandb online`<code> or set <code>WANDB_MODE=online<code> to enable cloud syncing."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  5   9  26  27  38  67  71  73  77  78  87  90  91  94  95  97 103 107\n",
      " 116 122 138 145 156 163 171 177 178 187 194 198 206 218 220 239 241 247\n",
      " 252 275 278 279 282 297 308 330 349 350 357 368 382 393]\n",
      "Initial model weights:\n",
      "0.0.weight: tensor([[[[-0.0972,  0.0900, -0.1326],\n",
      "          [-0.0078, -0.1856, -0.0831],\n",
      "          [-0.0999, -0.1356,  0.0071]],\n",
      "\n",
      "         [[ 0.0200,  0.0469,  0.0839],\n",
      "          [ 0.1775, -0.0488,  0.0517],\n",
      "          [-0.1458,  0.1854, -0.0844]],\n",
      "\n",
      "         [[ 0.1036, -0.0884,  0.1442],\n",
      "          [-0.0890,  0.1085, -0.0634],\n",
      "          [-0.1787, -0.0818, -0.1276]]],\n",
      "\n",
      "\n",
      "        [[[-0.0505, -0.1637, -0.0558],\n",
      "          [ 0.1473,  0.1602,  0.1162],\n",
      "          [ 0.0128,  0.0547, -0.0157]],\n",
      "\n",
      "         [[ 0.0924,  0.1915,  0.0686],\n",
      "          [ 0.0435,  0.1749, -0.0623],\n",
      "          [ 0.1249,  0.1459, -0.1371]],\n",
      "\n",
      "         [[ 0.1189, -0.1699,  0.1446],\n",
      "          [ 0.1534,  0.1593, -0.0559],\n",
      "          [ 0.1227, -0.1736, -0.1084]]],\n",
      "\n",
      "\n",
      "        [[[ 0.0468, -0.0588, -0.0043],\n",
      "          [ 0.1889, -0.0063,  0.0584],\n",
      "          [ 0.1521,  0.1555,  0.0218]],\n",
      "\n",
      "         [[-0.1725, -0.0953,  0.1508],\n",
      "          [-0.0198, -0.0387,  0.0138],\n",
      "          [ 0.0153, -0.0330, -0.1572]],\n",
      "\n",
      "         [[-0.0778, -0.1303, -0.0574],\n",
      "          [ 0.1732, -0.1035,  0.1658],\n",
      "          [ 0.1135,  0.1026,  0.0847]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[ 0.1152, -0.0898, -0.1548],\n",
      "          [ 0.0613, -0.1054,  0.0993],\n",
      "          [ 0.0830, -0.1760,  0.1715]],\n",
      "\n",
      "         [[ 0.1347, -0.0233,  0.1722],\n",
      "          [ 0.1083, -0.1789, -0.0031],\n",
      "          [-0.0349,  0.0759, -0.0371]],\n",
      "\n",
      "         [[-0.0258, -0.0801, -0.1508],\n",
      "          [-0.1899,  0.0032, -0.1873],\n",
      "          [-0.1402, -0.1273, -0.1767]]],\n",
      "\n",
      "\n",
      "        [[[-0.0382,  0.0975,  0.1652],\n",
      "          [-0.1615, -0.0721, -0.0020],\n",
      "          [-0.1318, -0.1478,  0.1313]],\n",
      "\n",
      "         [[ 0.0664, -0.1448, -0.0390],\n",
      "          [-0.1420, -0.0298, -0.0372],\n",
      "          [ 0.1676, -0.0967,  0.1739]],\n",
      "\n",
      "         [[-0.0578,  0.0283,  0.0241],\n",
      "          [ 0.0550,  0.1401,  0.0010],\n",
      "          [ 0.0070, -0.1180, -0.1795]]],\n",
      "\n",
      "\n",
      "        [[[ 0.0485, -0.0215, -0.0321],\n",
      "          [ 0.1311,  0.1866,  0.0019],\n",
      "          [-0.1446, -0.0483, -0.1081]],\n",
      "\n",
      "         [[-0.0946, -0.1775,  0.0980],\n",
      "          [ 0.1734,  0.1364,  0.0637],\n",
      "          [ 0.1877,  0.0224, -0.0197]],\n",
      "\n",
      "         [[ 0.1372,  0.0015,  0.1854],\n",
      "          [ 0.1474,  0.1537,  0.0113],\n",
      "          [-0.1564, -0.1751,  0.0057]]]], device='cuda:0')\n",
      "0.1.weight: tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1.], device='cuda:0')\n",
      "0.1.bias: tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "       device='cuda:0')\n",
      "1.0.weight: tensor([[[[ 0.0052, -0.0033,  0.0071,  0.0046,  0.0183],\n",
      "          [-0.0101, -0.0017, -0.0065, -0.0229,  0.0078],\n",
      "          [ 0.0203,  0.0009, -0.0108, -0.0196, -0.0156],\n",
      "          [-0.0050,  0.0065, -0.0039,  0.0098, -0.0235],\n",
      "          [-0.0204,  0.0225, -0.0048, -0.0071, -0.0019]],\n",
      "\n",
      "         [[-0.0042, -0.0154, -0.0041, -0.0029,  0.0169],\n",
      "          [-0.0029,  0.0009,  0.0108, -0.0080,  0.0210],\n",
      "          [ 0.0181, -0.0175, -0.0024,  0.0225, -0.0182],\n",
      "          [-0.0102, -0.0003, -0.0207, -0.0023,  0.0054],\n",
      "          [-0.0131, -0.0240, -0.0250,  0.0025, -0.0204]],\n",
      "\n",
      "         [[ 0.0178, -0.0110,  0.0038,  0.0205,  0.0049],\n",
      "          [ 0.0124,  0.0063,  0.0036,  0.0156, -0.0216],\n",
      "          [ 0.0203,  0.0026,  0.0163, -0.0067,  0.0002],\n",
      "          [ 0.0202,  0.0008,  0.0197,  0.0204,  0.0023],\n",
      "          [ 0.0127,  0.0190, -0.0045, -0.0212, -0.0173]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-0.0243,  0.0077,  0.0095, -0.0068, -0.0041],\n",
      "          [ 0.0151, -0.0208, -0.0135, -0.0236, -0.0059],\n",
      "          [-0.0061, -0.0114,  0.0129,  0.0095,  0.0217],\n",
      "          [ 0.0115,  0.0182,  0.0217, -0.0011,  0.0248],\n",
      "          [ 0.0050, -0.0101,  0.0153,  0.0178, -0.0132]],\n",
      "\n",
      "         [[ 0.0035,  0.0065,  0.0001,  0.0150, -0.0118],\n",
      "          [ 0.0075, -0.0247,  0.0029,  0.0208, -0.0179],\n",
      "          [-0.0183,  0.0057, -0.0136, -0.0002,  0.0177],\n",
      "          [ 0.0055, -0.0168, -0.0037,  0.0072, -0.0049],\n",
      "          [ 0.0186, -0.0115,  0.0221, -0.0068, -0.0149]],\n",
      "\n",
      "         [[ 0.0142, -0.0215, -0.0165, -0.0091,  0.0066],\n",
      "          [ 0.0145,  0.0158, -0.0075,  0.0100,  0.0078],\n",
      "          [-0.0129,  0.0118, -0.0161,  0.0161,  0.0069],\n",
      "          [-0.0016, -0.0224,  0.0194, -0.0017,  0.0068],\n",
      "          [-0.0083, -0.0190,  0.0003,  0.0124, -0.0058]]],\n",
      "\n",
      "\n",
      "        [[[ 0.0142, -0.0015, -0.0153, -0.0199,  0.0117],\n",
      "          [-0.0030, -0.0075, -0.0019,  0.0193,  0.0249],\n",
      "          [ 0.0176,  0.0164, -0.0196, -0.0088,  0.0047],\n",
      "          [ 0.0186,  0.0029, -0.0034,  0.0039, -0.0221],\n",
      "          [-0.0126,  0.0187,  0.0086, -0.0024,  0.0131]],\n",
      "\n",
      "         [[-0.0181,  0.0119,  0.0018,  0.0096, -0.0066],\n",
      "          [ 0.0103, -0.0194,  0.0122, -0.0211,  0.0104],\n",
      "          [ 0.0181,  0.0205, -0.0205, -0.0144, -0.0005],\n",
      "          [ 0.0070,  0.0068,  0.0250,  0.0011, -0.0234],\n",
      "          [-0.0029, -0.0081, -0.0014,  0.0078,  0.0206]],\n",
      "\n",
      "         [[ 0.0239,  0.0106, -0.0054, -0.0041, -0.0246],\n",
      "          [ 0.0067, -0.0192,  0.0038, -0.0174, -0.0088],\n",
      "          [ 0.0124, -0.0165, -0.0228,  0.0006,  0.0100],\n",
      "          [-0.0004,  0.0193,  0.0058, -0.0025, -0.0023],\n",
      "          [ 0.0126, -0.0239, -0.0185,  0.0148,  0.0083]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 0.0025,  0.0207, -0.0050, -0.0118, -0.0101],\n",
      "          [ 0.0020, -0.0064,  0.0169,  0.0240,  0.0123],\n",
      "          [ 0.0032, -0.0035,  0.0080,  0.0023,  0.0202],\n",
      "          [-0.0084, -0.0012,  0.0110, -0.0130, -0.0127],\n",
      "          [-0.0173,  0.0009,  0.0054,  0.0194, -0.0223]],\n",
      "\n",
      "         [[ 0.0057, -0.0137,  0.0166, -0.0124,  0.0242],\n",
      "          [-0.0022,  0.0052,  0.0028, -0.0202,  0.0135],\n",
      "          [-0.0092,  0.0093,  0.0105, -0.0003, -0.0109],\n",
      "          [ 0.0219,  0.0230, -0.0183,  0.0238,  0.0189],\n",
      "          [ 0.0131, -0.0073, -0.0168, -0.0173, -0.0204]],\n",
      "\n",
      "         [[ 0.0247, -0.0176, -0.0022,  0.0102, -0.0201],\n",
      "          [-0.0127, -0.0155, -0.0212,  0.0200, -0.0140],\n",
      "          [-0.0229, -0.0060, -0.0077, -0.0166, -0.0021],\n",
      "          [ 0.0114, -0.0123, -0.0068, -0.0118, -0.0227],\n",
      "          [-0.0122,  0.0164, -0.0250,  0.0156, -0.0168]]],\n",
      "\n",
      "\n",
      "        [[[-0.0164,  0.0036, -0.0134, -0.0194,  0.0080],\n",
      "          [-0.0242, -0.0233, -0.0096,  0.0225, -0.0153],\n",
      "          [ 0.0208,  0.0208, -0.0074,  0.0137,  0.0178],\n",
      "          [-0.0019, -0.0189,  0.0184, -0.0189,  0.0166],\n",
      "          [ 0.0154,  0.0030, -0.0083,  0.0245, -0.0003]],\n",
      "\n",
      "         [[-0.0236, -0.0218,  0.0131, -0.0178, -0.0089],\n",
      "          [ 0.0068, -0.0111, -0.0092,  0.0101,  0.0053],\n",
      "          [-0.0165, -0.0092, -0.0198,  0.0040, -0.0050],\n",
      "          [-0.0122, -0.0130,  0.0010,  0.0001, -0.0250],\n",
      "          [-0.0230, -0.0156,  0.0188,  0.0074, -0.0041]],\n",
      "\n",
      "         [[ 0.0061,  0.0198, -0.0202, -0.0211,  0.0083],\n",
      "          [-0.0025, -0.0174, -0.0168, -0.0069, -0.0048],\n",
      "          [ 0.0116,  0.0068, -0.0127, -0.0199, -0.0235],\n",
      "          [-0.0074,  0.0200, -0.0010,  0.0207, -0.0236],\n",
      "          [-0.0195,  0.0247,  0.0046, -0.0185,  0.0053]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-0.0246, -0.0217,  0.0223,  0.0076, -0.0224],\n",
      "          [-0.0101, -0.0135, -0.0093, -0.0233, -0.0195],\n",
      "          [-0.0201,  0.0158,  0.0087, -0.0168, -0.0167],\n",
      "          [ 0.0012,  0.0186,  0.0092,  0.0104, -0.0158],\n",
      "          [-0.0233, -0.0037, -0.0115, -0.0024,  0.0051]],\n",
      "\n",
      "         [[ 0.0201,  0.0219, -0.0230, -0.0115,  0.0207],\n",
      "          [-0.0228,  0.0229, -0.0244, -0.0231,  0.0214],\n",
      "          [ 0.0126, -0.0237, -0.0035, -0.0207, -0.0041],\n",
      "          [-0.0143, -0.0050, -0.0159, -0.0077,  0.0152],\n",
      "          [-0.0055, -0.0168, -0.0187,  0.0187,  0.0031]],\n",
      "\n",
      "         [[-0.0115,  0.0234, -0.0007,  0.0158,  0.0008],\n",
      "          [-0.0074, -0.0169, -0.0206,  0.0201, -0.0142],\n",
      "          [-0.0024,  0.0095, -0.0184, -0.0084,  0.0135],\n",
      "          [-0.0048, -0.0013, -0.0040, -0.0087,  0.0248],\n",
      "          [ 0.0023,  0.0238, -0.0109,  0.0159, -0.0132]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[ 0.0223, -0.0237,  0.0025, -0.0101,  0.0199],\n",
      "          [ 0.0037,  0.0040, -0.0025, -0.0012, -0.0129],\n",
      "          [ 0.0003, -0.0110, -0.0033,  0.0022,  0.0197],\n",
      "          [ 0.0160,  0.0130, -0.0057, -0.0236, -0.0136],\n",
      "          [ 0.0099, -0.0095,  0.0186, -0.0123, -0.0079]],\n",
      "\n",
      "         [[-0.0112, -0.0080,  0.0203,  0.0235,  0.0150],\n",
      "          [-0.0239,  0.0008, -0.0211, -0.0174, -0.0226],\n",
      "          [ 0.0022,  0.0106,  0.0195,  0.0186, -0.0066],\n",
      "          [ 0.0028,  0.0143,  0.0173, -0.0145, -0.0055],\n",
      "          [ 0.0163, -0.0062,  0.0050, -0.0023,  0.0096]],\n",
      "\n",
      "         [[-0.0192, -0.0119,  0.0247,  0.0229, -0.0009],\n",
      "          [ 0.0038,  0.0165, -0.0011, -0.0112,  0.0090],\n",
      "          [-0.0065,  0.0169,  0.0017,  0.0209, -0.0046],\n",
      "          [-0.0051, -0.0096,  0.0104, -0.0040,  0.0210],\n",
      "          [-0.0231,  0.0036, -0.0171,  0.0236, -0.0112]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-0.0102,  0.0071,  0.0088, -0.0141,  0.0086],\n",
      "          [ 0.0043, -0.0075, -0.0223, -0.0068, -0.0203],\n",
      "          [-0.0118, -0.0069,  0.0112,  0.0197,  0.0083],\n",
      "          [ 0.0007,  0.0145,  0.0203, -0.0224, -0.0064],\n",
      "          [-0.0083, -0.0151,  0.0026, -0.0127,  0.0181]],\n",
      "\n",
      "         [[-0.0084, -0.0066, -0.0140, -0.0238,  0.0085],\n",
      "          [ 0.0158,  0.0178,  0.0160, -0.0225, -0.0033],\n",
      "          [-0.0167,  0.0104,  0.0231,  0.0149, -0.0214],\n",
      "          [ 0.0150, -0.0169,  0.0243, -0.0244,  0.0111],\n",
      "          [-0.0039, -0.0239,  0.0087, -0.0107, -0.0003]],\n",
      "\n",
      "         [[ 0.0079, -0.0024, -0.0174, -0.0029,  0.0160],\n",
      "          [-0.0177,  0.0223, -0.0104, -0.0048, -0.0136],\n",
      "          [-0.0080,  0.0169,  0.0111, -0.0083,  0.0190],\n",
      "          [ 0.0118,  0.0138,  0.0182,  0.0056,  0.0082],\n",
      "          [ 0.0039,  0.0112, -0.0163,  0.0051,  0.0053]]],\n",
      "\n",
      "\n",
      "        [[[-0.0111, -0.0208,  0.0226, -0.0068, -0.0133],\n",
      "          [ 0.0056,  0.0102, -0.0240,  0.0107, -0.0009],\n",
      "          [ 0.0019,  0.0206,  0.0054,  0.0153, -0.0199],\n",
      "          [ 0.0103,  0.0212,  0.0189,  0.0187,  0.0185],\n",
      "          [ 0.0243, -0.0128, -0.0220, -0.0031, -0.0148]],\n",
      "\n",
      "         [[ 0.0123,  0.0102, -0.0035,  0.0014, -0.0242],\n",
      "          [-0.0008,  0.0146, -0.0222, -0.0226,  0.0081],\n",
      "          [-0.0134,  0.0173, -0.0097,  0.0113, -0.0226],\n",
      "          [-0.0222, -0.0065, -0.0045, -0.0034, -0.0029],\n",
      "          [-0.0245, -0.0177,  0.0194,  0.0218, -0.0117]],\n",
      "\n",
      "         [[-0.0227, -0.0235, -0.0128, -0.0068, -0.0197],\n",
      "          [ 0.0081,  0.0103,  0.0154,  0.0025,  0.0149],\n",
      "          [ 0.0085, -0.0008,  0.0169, -0.0237, -0.0165],\n",
      "          [-0.0049, -0.0112,  0.0077, -0.0195, -0.0230],\n",
      "          [-0.0101,  0.0059, -0.0102, -0.0103,  0.0120]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-0.0118,  0.0095, -0.0042, -0.0087, -0.0117],\n",
      "          [ 0.0070,  0.0115, -0.0181,  0.0104, -0.0065],\n",
      "          [ 0.0240,  0.0034,  0.0242, -0.0196,  0.0048],\n",
      "          [-0.0044, -0.0217,  0.0198, -0.0118, -0.0049],\n",
      "          [-0.0010,  0.0142,  0.0013,  0.0211,  0.0140]],\n",
      "\n",
      "         [[ 0.0242,  0.0039, -0.0019,  0.0074,  0.0066],\n",
      "          [-0.0243, -0.0064,  0.0010, -0.0021,  0.0013],\n",
      "          [ 0.0164, -0.0063,  0.0193,  0.0170,  0.0201],\n",
      "          [-0.0195,  0.0181,  0.0147, -0.0121, -0.0206],\n",
      "          [-0.0005, -0.0193,  0.0164, -0.0219, -0.0192]],\n",
      "\n",
      "         [[ 0.0042,  0.0203,  0.0075, -0.0169, -0.0121],\n",
      "          [ 0.0117,  0.0160, -0.0039,  0.0183,  0.0034],\n",
      "          [-0.0207, -0.0218,  0.0134,  0.0164,  0.0183],\n",
      "          [ 0.0034,  0.0051,  0.0108,  0.0119,  0.0103],\n",
      "          [ 0.0190,  0.0225,  0.0212, -0.0110, -0.0203]]],\n",
      "\n",
      "\n",
      "        [[[ 0.0038, -0.0247, -0.0049,  0.0053, -0.0024],\n",
      "          [ 0.0055,  0.0076,  0.0187, -0.0092,  0.0217],\n",
      "          [-0.0005,  0.0034,  0.0193, -0.0112, -0.0198],\n",
      "          [ 0.0230, -0.0027, -0.0176,  0.0019, -0.0029],\n",
      "          [-0.0102, -0.0039, -0.0041,  0.0043, -0.0092]],\n",
      "\n",
      "         [[ 0.0055,  0.0007,  0.0176,  0.0054, -0.0040],\n",
      "          [ 0.0218,  0.0248, -0.0104,  0.0154, -0.0066],\n",
      "          [-0.0151, -0.0143, -0.0248,  0.0020, -0.0219],\n",
      "          [-0.0048, -0.0086, -0.0147,  0.0007,  0.0061],\n",
      "          [ 0.0248,  0.0108, -0.0133,  0.0237,  0.0095]],\n",
      "\n",
      "         [[-0.0169, -0.0222, -0.0085,  0.0092,  0.0180],\n",
      "          [-0.0174,  0.0133,  0.0165, -0.0160,  0.0183],\n",
      "          [ 0.0190, -0.0104, -0.0106,  0.0195, -0.0205],\n",
      "          [-0.0091,  0.0093, -0.0006,  0.0219, -0.0216],\n",
      "          [ 0.0074,  0.0005, -0.0148, -0.0139, -0.0023]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 0.0056,  0.0205, -0.0232, -0.0235, -0.0081],\n",
      "          [-0.0240, -0.0065, -0.0239, -0.0165,  0.0161],\n",
      "          [ 0.0085,  0.0139,  0.0043,  0.0035,  0.0041],\n",
      "          [-0.0118, -0.0207,  0.0154,  0.0118, -0.0244],\n",
      "          [-0.0176,  0.0246,  0.0079, -0.0035, -0.0143]],\n",
      "\n",
      "         [[ 0.0250, -0.0228, -0.0227,  0.0220, -0.0109],\n",
      "          [ 0.0028,  0.0099,  0.0208,  0.0227, -0.0115],\n",
      "          [ 0.0074, -0.0035,  0.0005, -0.0096,  0.0129],\n",
      "          [-0.0195, -0.0082,  0.0234, -0.0154, -0.0107],\n",
      "          [-0.0071, -0.0063, -0.0139,  0.0075, -0.0231]],\n",
      "\n",
      "         [[ 0.0132,  0.0073,  0.0050,  0.0158,  0.0003],\n",
      "          [-0.0106, -0.0197,  0.0019,  0.0245, -0.0182],\n",
      "          [-0.0201,  0.0151, -0.0130,  0.0219, -0.0208],\n",
      "          [ 0.0199,  0.0015,  0.0111,  0.0148,  0.0230],\n",
      "          [ 0.0240, -0.0168, -0.0209, -0.0117,  0.0123]]]], device='cuda:0')\n",
      "1.1.weight: tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1.], device='cuda:0')\n",
      "1.1.bias: tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
      "2.module.0.0.weight: tensor([[[[-0.0067,  0.0008,  0.0128],\n",
      "          [ 0.0177,  0.0044,  0.0058],\n",
      "          [ 0.0206,  0.0185,  0.0067]],\n",
      "\n",
      "         [[-0.0012, -0.0255,  0.0161],\n",
      "          [-0.0235, -0.0263,  0.0064],\n",
      "          [ 0.0253, -0.0288,  0.0202]],\n",
      "\n",
      "         [[ 0.0019, -0.0048, -0.0186],\n",
      "          [ 0.0129, -0.0124,  0.0289],\n",
      "          [-0.0090,  0.0165,  0.0278]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 0.0261,  0.0169, -0.0177],\n",
      "          [ 0.0238,  0.0201,  0.0292],\n",
      "          [-0.0024, -0.0163,  0.0162]],\n",
      "\n",
      "         [[ 0.0008, -0.0209, -0.0176],\n",
      "          [-0.0134,  0.0107,  0.0156],\n",
      "          [-0.0216, -0.0254,  0.0145]],\n",
      "\n",
      "         [[-0.0008, -0.0064,  0.0246],\n",
      "          [ 0.0094, -0.0275,  0.0150],\n",
      "          [ 0.0041, -0.0104, -0.0284]]],\n",
      "\n",
      "\n",
      "        [[[-0.0282, -0.0168,  0.0233],\n",
      "          [ 0.0182, -0.0059,  0.0237],\n",
      "          [-0.0189,  0.0215,  0.0199]],\n",
      "\n",
      "         [[-0.0167,  0.0236,  0.0102],\n",
      "          [ 0.0171, -0.0199,  0.0208],\n",
      "          [ 0.0210, -0.0128,  0.0010]],\n",
      "\n",
      "         [[-0.0045, -0.0252, -0.0276],\n",
      "          [ 0.0158, -0.0162, -0.0153],\n",
      "          [-0.0073, -0.0031,  0.0222]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 0.0236, -0.0083,  0.0127],\n",
      "          [-0.0225,  0.0138, -0.0290],\n",
      "          [ 0.0247,  0.0070, -0.0148]],\n",
      "\n",
      "         [[-0.0209, -0.0047,  0.0247],\n",
      "          [ 0.0121,  0.0291, -0.0068],\n",
      "          [-0.0039, -0.0179,  0.0218]],\n",
      "\n",
      "         [[-0.0070, -0.0261, -0.0124],\n",
      "          [ 0.0244,  0.0266,  0.0217],\n",
      "          [-0.0212, -0.0232,  0.0148]]],\n",
      "\n",
      "\n",
      "        [[[ 0.0013,  0.0270, -0.0020],\n",
      "          [ 0.0224,  0.0162, -0.0264],\n",
      "          [ 0.0139,  0.0202,  0.0244]],\n",
      "\n",
      "         [[ 0.0209,  0.0215, -0.0188],\n",
      "          [ 0.0170,  0.0276, -0.0150],\n",
      "          [ 0.0202, -0.0129,  0.0093]],\n",
      "\n",
      "         [[-0.0219, -0.0140, -0.0204],\n",
      "          [-0.0012,  0.0122,  0.0248],\n",
      "          [ 0.0194, -0.0247,  0.0068]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-0.0119, -0.0019,  0.0021],\n",
      "          [-0.0001,  0.0249,  0.0283],\n",
      "          [-0.0205, -0.0145, -0.0268]],\n",
      "\n",
      "         [[-0.0116, -0.0191, -0.0135],\n",
      "          [-0.0168,  0.0117,  0.0075],\n",
      "          [ 0.0178,  0.0019,  0.0231]],\n",
      "\n",
      "         [[-0.0038,  0.0159,  0.0152],\n",
      "          [ 0.0186,  0.0236,  0.0134],\n",
      "          [ 0.0202, -0.0110, -0.0024]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[-0.0249, -0.0155, -0.0041],\n",
      "          [ 0.0174,  0.0024,  0.0242],\n",
      "          [ 0.0136, -0.0100,  0.0126]],\n",
      "\n",
      "         [[ 0.0092, -0.0174,  0.0053],\n",
      "          [ 0.0234,  0.0051, -0.0091],\n",
      "          [ 0.0054,  0.0006,  0.0121]],\n",
      "\n",
      "         [[ 0.0094,  0.0017,  0.0027],\n",
      "          [-0.0249, -0.0231, -0.0054],\n",
      "          [-0.0273,  0.0255, -0.0280]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-0.0290,  0.0130,  0.0197],\n",
      "          [-0.0228,  0.0127,  0.0053],\n",
      "          [-0.0286, -0.0262,  0.0036]],\n",
      "\n",
      "         [[ 0.0193,  0.0158, -0.0007],\n",
      "          [-0.0019,  0.0244,  0.0256],\n",
      "          [-0.0048,  0.0193, -0.0251]],\n",
      "\n",
      "         [[-0.0292,  0.0147, -0.0081],\n",
      "          [ 0.0176, -0.0246,  0.0238],\n",
      "          [-0.0142,  0.0066, -0.0071]]],\n",
      "\n",
      "\n",
      "        [[[-0.0031,  0.0156, -0.0057],\n",
      "          [-0.0186,  0.0248, -0.0177],\n",
      "          [ 0.0246, -0.0283,  0.0056]],\n",
      "\n",
      "         [[-0.0094, -0.0107,  0.0257],\n",
      "          [ 0.0153, -0.0163,  0.0036],\n",
      "          [-0.0203,  0.0229,  0.0031]],\n",
      "\n",
      "         [[-0.0212,  0.0039,  0.0155],\n",
      "          [-0.0117, -0.0026,  0.0149],\n",
      "          [-0.0097, -0.0257, -0.0076]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-0.0092, -0.0182,  0.0220],\n",
      "          [ 0.0018,  0.0144, -0.0026],\n",
      "          [-0.0189,  0.0200,  0.0288]],\n",
      "\n",
      "         [[-0.0121, -0.0132,  0.0037],\n",
      "          [ 0.0191,  0.0131,  0.0256],\n",
      "          [ 0.0077,  0.0108,  0.0190]],\n",
      "\n",
      "         [[-0.0112,  0.0237,  0.0016],\n",
      "          [-0.0262,  0.0012, -0.0073],\n",
      "          [ 0.0132, -0.0184,  0.0095]]],\n",
      "\n",
      "\n",
      "        [[[-0.0294,  0.0151, -0.0184],\n",
      "          [-0.0254, -0.0164,  0.0219],\n",
      "          [ 0.0243, -0.0118, -0.0177]],\n",
      "\n",
      "         [[ 0.0186,  0.0155, -0.0273],\n",
      "          [ 0.0244,  0.0216,  0.0166],\n",
      "          [ 0.0089, -0.0153,  0.0106]],\n",
      "\n",
      "         [[-0.0122, -0.0015,  0.0172],\n",
      "          [-0.0066,  0.0227, -0.0273],\n",
      "          [-0.0072,  0.0243, -0.0172]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 0.0133, -0.0180,  0.0275],\n",
      "          [ 0.0211,  0.0100, -0.0137],\n",
      "          [ 0.0195, -0.0098,  0.0216]],\n",
      "\n",
      "         [[-0.0005,  0.0248,  0.0178],\n",
      "          [ 0.0143,  0.0293, -0.0141],\n",
      "          [ 0.0229, -0.0239, -0.0167]],\n",
      "\n",
      "         [[ 0.0055,  0.0226, -0.0140],\n",
      "          [ 0.0134,  0.0058, -0.0056],\n",
      "          [ 0.0055, -0.0262,  0.0076]]]], device='cuda:0')\n",
      "2.module.0.1.weight: tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1.], device='cuda:0')\n",
      "2.module.0.1.bias: tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
      "3.0.weight: tensor([[[[-1.0797e-05, -7.4226e-03, -2.0054e-02],\n",
      "          [-1.2167e-02,  5.2503e-03,  9.1568e-03],\n",
      "          [-2.0379e-02, -2.7549e-02, -2.5680e-02]],\n",
      "\n",
      "         [[ 2.7077e-02,  6.7590e-03, -2.0077e-02],\n",
      "          [-1.4154e-02, -1.7101e-02,  1.1846e-02],\n",
      "          [ 7.0859e-03,  1.3140e-02,  3.7591e-03]],\n",
      "\n",
      "         [[-1.8836e-02, -2.8675e-02, -4.6019e-04],\n",
      "          [-7.5794e-03,  9.3397e-03,  8.3769e-03],\n",
      "          [-5.6262e-04, -2.7953e-03,  8.7676e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-1.4073e-02,  1.0599e-02, -2.6882e-02],\n",
      "          [-2.1887e-02, -1.6660e-02,  1.2277e-02],\n",
      "          [ 2.8651e-02, -2.1008e-02,  1.8685e-02]],\n",
      "\n",
      "         [[-1.0843e-02, -1.5685e-02,  8.5275e-03],\n",
      "          [ 2.4588e-02, -4.4191e-03,  3.5011e-03],\n",
      "          [-1.8071e-02,  1.3007e-03,  1.1004e-02]],\n",
      "\n",
      "         [[-2.2366e-02, -8.5486e-03,  1.0880e-02],\n",
      "          [ 2.0122e-02, -9.7229e-03, -6.9414e-03],\n",
      "          [ 7.0568e-05, -6.1989e-03, -1.4462e-02]]],\n",
      "\n",
      "\n",
      "        [[[-9.0624e-03,  1.2073e-02,  2.1209e-02],\n",
      "          [-7.4237e-04, -8.3454e-03, -1.1416e-02],\n",
      "          [-2.2670e-02, -2.4490e-03,  2.9290e-02]],\n",
      "\n",
      "         [[-3.8853e-03,  2.2542e-02, -1.0273e-02],\n",
      "          [ 2.8173e-03, -1.7846e-02,  3.0453e-03],\n",
      "          [ 3.9035e-03, -1.8322e-02, -8.0229e-03]],\n",
      "\n",
      "         [[-2.4693e-02,  1.7519e-02,  3.1484e-03],\n",
      "          [ 1.5085e-02,  2.7748e-02,  9.1849e-04],\n",
      "          [ 1.4274e-02,  7.2665e-03,  2.4006e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 2.5540e-02,  2.2546e-02, -1.0307e-02],\n",
      "          [-1.2311e-02, -1.9956e-03, -1.7751e-02],\n",
      "          [ 2.1144e-02, -3.5991e-03, -8.8850e-03]],\n",
      "\n",
      "         [[ 1.1653e-04, -7.7781e-03,  1.0465e-02],\n",
      "          [ 2.7305e-02,  2.0042e-02, -7.4320e-03],\n",
      "          [-2.3589e-02,  1.6795e-02,  1.8981e-02]],\n",
      "\n",
      "         [[ 1.7927e-02,  2.0575e-02, -2.4861e-02],\n",
      "          [ 2.3559e-02,  4.9559e-03, -2.6674e-02],\n",
      "          [-1.5991e-02, -1.8352e-02,  4.7768e-03]]],\n",
      "\n",
      "\n",
      "        [[[ 7.4986e-03,  2.3008e-02, -2.8440e-02],\n",
      "          [-1.6226e-02,  1.9795e-02,  2.1841e-02],\n",
      "          [ 2.9255e-02, -1.6836e-02, -2.3376e-02]],\n",
      "\n",
      "         [[-3.6296e-03, -1.8926e-02,  1.5056e-02],\n",
      "          [-1.6350e-02,  8.8835e-03, -2.1744e-02],\n",
      "          [-2.1931e-02, -8.1262e-04, -1.1861e-02]],\n",
      "\n",
      "         [[-1.3392e-02, -5.0936e-03,  6.0791e-03],\n",
      "          [ 1.6002e-02, -2.0220e-02,  2.2720e-02],\n",
      "          [ 1.8798e-02,  3.7909e-03, -6.1076e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-1.7534e-02,  1.4556e-02, -2.1887e-02],\n",
      "          [ 2.2657e-02,  2.7258e-02, -1.4888e-02],\n",
      "          [-2.6362e-03,  2.7879e-02, -2.5093e-02]],\n",
      "\n",
      "         [[ 1.7928e-02,  2.8534e-02,  1.6342e-02],\n",
      "          [ 1.6620e-02,  1.8104e-02,  2.0382e-02],\n",
      "          [-2.2393e-02,  3.3950e-03,  2.4767e-02]],\n",
      "\n",
      "         [[-9.2456e-03,  1.1169e-02, -2.8744e-02],\n",
      "          [-1.3735e-02,  8.0631e-03,  5.2540e-03],\n",
      "          [ 4.4599e-03,  1.5182e-02, -1.7586e-02]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[ 1.7081e-02, -2.3019e-03, -2.4646e-02],\n",
      "          [ 4.7232e-03,  1.6714e-02,  3.1796e-03],\n",
      "          [ 2.8519e-02, -6.3955e-03, -8.3449e-03]],\n",
      "\n",
      "         [[-2.6101e-03, -1.8663e-02,  1.7104e-02],\n",
      "          [ 2.8603e-02, -1.1592e-02, -2.3028e-02],\n",
      "          [-7.3859e-03,  9.7140e-03,  2.3351e-02]],\n",
      "\n",
      "         [[-6.0045e-03,  1.3398e-02, -2.6847e-03],\n",
      "          [ 2.5968e-02,  1.8496e-02, -2.9409e-03],\n",
      "          [ 1.1117e-02,  4.2411e-03,  7.9361e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-8.1799e-03, -1.0447e-02, -1.1475e-02],\n",
      "          [ 8.2294e-03,  2.8527e-02,  2.8151e-02],\n",
      "          [-1.6773e-02, -2.4352e-02, -1.2328e-02]],\n",
      "\n",
      "         [[ 2.9249e-02,  1.6633e-02,  4.0932e-03],\n",
      "          [ 1.3732e-02, -2.8740e-02,  2.0805e-02],\n",
      "          [ 2.7081e-02,  2.6029e-02, -2.5647e-02]],\n",
      "\n",
      "         [[ 8.5805e-04, -1.9532e-02, -1.9637e-02],\n",
      "          [ 1.5447e-02,  2.5469e-02,  1.4289e-02],\n",
      "          [-1.4419e-02, -8.1068e-03, -2.4631e-02]]],\n",
      "\n",
      "\n",
      "        [[[-7.0560e-03, -2.0301e-02, -4.6708e-03],\n",
      "          [-1.1283e-02, -2.5589e-02,  5.8249e-03],\n",
      "          [-1.1912e-02,  8.4811e-03,  1.6328e-02]],\n",
      "\n",
      "         [[-1.7415e-02, -1.7974e-02, -1.4835e-02],\n",
      "          [-1.5250e-02, -2.2944e-02,  2.3615e-02],\n",
      "          [ 1.8249e-02,  1.4963e-02, -7.3644e-03]],\n",
      "\n",
      "         [[-4.0844e-03, -2.2048e-02, -1.0527e-02],\n",
      "          [-4.3295e-03,  1.1095e-02, -3.3296e-04],\n",
      "          [ 1.7299e-02,  2.2875e-02, -1.8788e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 1.4864e-02, -1.6154e-02,  2.8469e-02],\n",
      "          [-4.6128e-03,  1.3643e-02,  2.1211e-02],\n",
      "          [-5.8237e-03, -2.5828e-02, -9.2249e-03]],\n",
      "\n",
      "         [[-4.8623e-03, -6.3018e-04, -1.0830e-03],\n",
      "          [-7.2107e-03,  2.2323e-02,  2.7643e-02],\n",
      "          [ 1.6807e-02, -7.7183e-04, -7.8587e-03]],\n",
      "\n",
      "         [[-1.3342e-03,  2.5417e-02,  8.2622e-03],\n",
      "          [-1.2638e-02, -7.5175e-03,  2.4697e-02],\n",
      "          [-8.2523e-03, -2.4813e-02,  1.6587e-03]]],\n",
      "\n",
      "\n",
      "        [[[-3.4231e-03, -2.6962e-02, -2.4685e-02],\n",
      "          [ 1.4728e-02, -4.4906e-03,  2.7251e-02],\n",
      "          [-2.6067e-02,  5.7675e-03, -2.0680e-02]],\n",
      "\n",
      "         [[-2.7727e-02,  2.1431e-02, -5.1248e-03],\n",
      "          [ 2.1464e-03,  9.4623e-04, -4.2377e-03],\n",
      "          [ 7.9861e-03, -1.6972e-02,  1.6116e-02]],\n",
      "\n",
      "         [[-2.2055e-02,  4.7399e-04,  6.6301e-04],\n",
      "          [ 2.7125e-02, -1.9486e-02,  2.0891e-02],\n",
      "          [-2.7380e-02, -2.7401e-02, -2.1615e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 2.2700e-02, -1.7637e-02, -2.7814e-02],\n",
      "          [ 4.2842e-03,  1.5314e-02, -1.0683e-02],\n",
      "          [ 3.2520e-03, -2.2945e-02, -2.0840e-02]],\n",
      "\n",
      "         [[ 1.7394e-02,  4.7906e-03,  9.0051e-03],\n",
      "          [ 2.4398e-02,  9.0371e-03,  1.2995e-02],\n",
      "          [-1.4946e-02,  1.1301e-02,  2.6968e-02]],\n",
      "\n",
      "         [[-1.1363e-02,  2.4262e-02, -1.8591e-02],\n",
      "          [ 2.0195e-02, -7.9942e-03,  1.3455e-02],\n",
      "          [-2.0297e-02, -2.4850e-02,  5.3879e-03]]]], device='cuda:0')\n",
      "3.1.weight: tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1.], device='cuda:0')\n",
      "3.1.bias: tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "       device='cuda:0')\n",
      "5.module.0.0.weight: tensor([[[[-4.6903e-04, -8.2567e-03,  1.3665e-02],\n",
      "          [ 1.9909e-02,  1.1860e-02, -1.0688e-02],\n",
      "          [ 7.9871e-03, -1.5701e-02, -1.7714e-02]],\n",
      "\n",
      "         [[ 1.7009e-02,  4.9644e-03, -3.8686e-03],\n",
      "          [-1.6706e-02, -1.5059e-02,  1.3062e-02],\n",
      "          [ 5.8451e-04,  1.3761e-02, -3.7122e-03]],\n",
      "\n",
      "         [[ 1.4013e-02, -1.4867e-02,  2.0101e-02],\n",
      "          [ 1.8739e-02,  8.3544e-03,  1.9036e-02],\n",
      "          [ 6.6533e-03, -9.2543e-03, -6.9564e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 7.0537e-03,  2.1238e-03, -5.7874e-03],\n",
      "          [ 5.6132e-03,  1.0009e-02, -1.0755e-02],\n",
      "          [ 3.3407e-03,  1.3833e-02, -1.3061e-03]],\n",
      "\n",
      "         [[ 1.4828e-02,  1.2099e-02,  5.2575e-03],\n",
      "          [ 1.3731e-02, -7.2881e-03, -2.6877e-03],\n",
      "          [-5.2776e-03, -6.2626e-03,  1.1177e-02]],\n",
      "\n",
      "         [[-1.3796e-02, -1.5475e-02,  1.0067e-02],\n",
      "          [ 1.9400e-02,  1.5853e-02,  2.2462e-03],\n",
      "          [-1.5942e-02,  6.6543e-03,  1.8672e-02]]],\n",
      "\n",
      "\n",
      "        [[[ 1.5447e-02, -1.7216e-02, -3.5626e-03],\n",
      "          [ 3.2212e-03, -2.1578e-03, -3.3689e-03],\n",
      "          [ 3.7473e-03, -1.0641e-02,  1.6656e-02]],\n",
      "\n",
      "         [[-1.3373e-02, -1.7738e-02, -1.6222e-02],\n",
      "          [ 7.9867e-03,  1.4788e-02, -6.7872e-03],\n",
      "          [ 7.6740e-03, -1.0422e-02,  4.7031e-03]],\n",
      "\n",
      "         [[ 1.0102e-02, -2.0089e-02,  1.4180e-02],\n",
      "          [-1.6236e-03, -9.5150e-03, -1.3779e-04],\n",
      "          [-3.5227e-03, -2.4222e-03,  8.5941e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 9.1298e-03,  5.7427e-04, -1.7715e-02],\n",
      "          [ 7.0815e-03,  8.5427e-03, -2.7039e-03],\n",
      "          [ 1.8868e-02,  1.2716e-02,  1.3836e-02]],\n",
      "\n",
      "         [[-1.2938e-02, -1.6699e-03, -5.1918e-03],\n",
      "          [-6.1770e-03,  5.6579e-03, -1.0177e-02],\n",
      "          [-1.7601e-02, -2.7733e-03, -6.7528e-03]],\n",
      "\n",
      "         [[ 1.9703e-02,  1.7968e-02, -3.6467e-03],\n",
      "          [ 1.4283e-02,  1.4070e-02, -7.7603e-04],\n",
      "          [ 1.4383e-02,  1.3832e-02, -1.8589e-02]]],\n",
      "\n",
      "\n",
      "        [[[ 1.4996e-02, -1.8248e-02,  1.8751e-02],\n",
      "          [-1.8578e-02, -1.0682e-02,  8.5787e-03],\n",
      "          [-1.3688e-02, -1.6211e-02, -5.9330e-03]],\n",
      "\n",
      "         [[-5.8646e-03,  4.5982e-03,  2.0324e-02],\n",
      "          [ 5.2911e-03,  1.8259e-02, -1.5098e-02],\n",
      "          [-2.3342e-03,  1.1198e-02,  5.9473e-03]],\n",
      "\n",
      "         [[-1.8119e-02,  1.1087e-02, -1.5952e-02],\n",
      "          [ 1.2361e-02,  1.9357e-02, -1.6900e-02],\n",
      "          [-8.5926e-03, -1.2268e-02, -1.3798e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-1.7498e-02,  1.4882e-02, -1.5722e-02],\n",
      "          [-8.2274e-04, -3.1281e-03, -4.0400e-03],\n",
      "          [-1.9034e-02, -9.6740e-03,  2.2602e-03]],\n",
      "\n",
      "         [[-5.6520e-03,  4.2788e-03,  2.0977e-03],\n",
      "          [ 7.2527e-03, -8.1238e-04, -6.4633e-03],\n",
      "          [-1.1636e-02,  4.8724e-03, -7.7402e-03]],\n",
      "\n",
      "         [[-1.3189e-02, -1.6094e-02, -8.0082e-03],\n",
      "          [ 2.5157e-03,  1.4537e-02,  8.9526e-05],\n",
      "          [ 1.5163e-02,  1.1677e-02,  8.8817e-03]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[-4.6261e-03, -1.3875e-02,  1.4785e-02],\n",
      "          [-1.1524e-02, -5.1939e-03, -1.5511e-02],\n",
      "          [-1.2510e-02, -7.1584e-03, -1.8407e-02]],\n",
      "\n",
      "         [[ 1.1810e-02,  1.8037e-02,  1.8165e-02],\n",
      "          [-1.2937e-03,  8.7544e-03, -5.9754e-03],\n",
      "          [-1.3446e-02,  4.5220e-03,  2.8354e-03]],\n",
      "\n",
      "         [[ 6.8246e-03, -1.8023e-02, -6.6865e-03],\n",
      "          [ 1.3841e-02, -5.9699e-03, -2.1984e-03],\n",
      "          [-7.6934e-03, -1.4050e-02,  1.9270e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 1.4477e-02,  1.6066e-02, -4.9641e-03],\n",
      "          [ 1.6316e-03, -9.5570e-03, -1.2236e-02],\n",
      "          [-1.5932e-02,  1.1314e-02, -1.7404e-02]],\n",
      "\n",
      "         [[-5.6732e-03,  1.8757e-02,  1.9398e-02],\n",
      "          [ 1.9532e-02, -1.8982e-02, -1.9379e-02],\n",
      "          [ 2.7639e-03, -1.2578e-02,  1.1437e-03]],\n",
      "\n",
      "         [[-7.0998e-03,  1.8252e-02, -1.4493e-02],\n",
      "          [ 1.4962e-02, -2.1796e-03, -1.6783e-02],\n",
      "          [-7.0484e-03, -4.0994e-03,  2.1549e-03]]],\n",
      "\n",
      "\n",
      "        [[[-1.9564e-02, -6.6734e-03,  2.2538e-04],\n",
      "          [-6.0093e-04,  1.7629e-02,  9.4264e-03],\n",
      "          [-1.2762e-02, -4.0763e-03,  1.1613e-02]],\n",
      "\n",
      "         [[-1.5593e-02, -1.4961e-02,  1.6679e-02],\n",
      "          [ 2.0454e-02, -3.5065e-03,  9.0220e-03],\n",
      "          [-1.2654e-02, -5.2785e-03,  6.3458e-03]],\n",
      "\n",
      "         [[-1.3200e-02, -8.2133e-03,  1.5975e-02],\n",
      "          [ 5.8840e-03,  5.3331e-03, -1.6235e-03],\n",
      "          [-1.0301e-02, -5.0802e-03, -1.0856e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 1.2714e-02,  1.9883e-02,  7.3503e-03],\n",
      "          [ 8.0705e-03, -1.5080e-02, -9.3002e-03],\n",
      "          [-9.5330e-03, -7.0326e-03, -1.8826e-02]],\n",
      "\n",
      "         [[-1.0735e-02,  5.3018e-03,  1.3964e-02],\n",
      "          [ 7.8367e-03,  1.8964e-02, -1.7658e-02],\n",
      "          [-7.0348e-03,  1.9759e-02,  3.0093e-03]],\n",
      "\n",
      "         [[-1.4257e-02, -1.1532e-02, -6.6912e-03],\n",
      "          [-1.7528e-02, -1.3710e-03,  1.9589e-02],\n",
      "          [-1.1124e-02, -1.6499e-02,  8.0827e-03]]],\n",
      "\n",
      "\n",
      "        [[[ 7.5542e-04,  1.8260e-02, -1.2272e-02],\n",
      "          [-1.8086e-03, -1.8552e-02, -6.1968e-03],\n",
      "          [ 4.5601e-03, -1.2451e-02, -1.6823e-02]],\n",
      "\n",
      "         [[-3.4269e-03, -5.3874e-03, -7.0932e-03],\n",
      "          [ 1.5304e-02,  9.8887e-03,  5.1764e-03],\n",
      "          [-7.6547e-03, -2.0573e-02, -2.0052e-02]],\n",
      "\n",
      "         [[-1.3890e-02,  1.6347e-02,  1.9042e-02],\n",
      "          [ 8.3977e-03,  1.2177e-02, -1.7410e-02],\n",
      "          [ 1.1690e-02, -9.7205e-04, -8.4071e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-8.4708e-03, -5.8725e-03,  1.4099e-02],\n",
      "          [-1.0949e-02, -1.6755e-02,  1.5524e-02],\n",
      "          [ 1.1147e-02,  5.4935e-04,  1.2316e-02]],\n",
      "\n",
      "         [[-5.9555e-03,  1.1468e-02, -1.8618e-03],\n",
      "          [ 2.0820e-02,  1.2727e-03,  6.2564e-03],\n",
      "          [ 1.8240e-02, -1.5289e-02, -8.4150e-03]],\n",
      "\n",
      "         [[-1.3110e-02,  1.5565e-02,  1.5553e-02],\n",
      "          [-1.9276e-02,  1.4857e-02,  1.1423e-03],\n",
      "          [ 5.8722e-03, -1.7924e-02, -1.1655e-02]]]], device='cuda:0')\n",
      "5.module.0.1.weight: tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1.], device='cuda:0')\n",
      "5.module.0.1.bias: tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "       device='cuda:0')\n",
      "6.0.weight: tensor([[[[-1.0047e-02,  1.2228e-02, -1.3401e-03],\n",
      "          [-2.0499e-02, -2.9666e-03, -3.6614e-03],\n",
      "          [ 9.7812e-03, -7.0322e-03,  9.0287e-03]],\n",
      "\n",
      "         [[ 7.9668e-03, -4.9193e-03, -7.5725e-03],\n",
      "          [-4.5136e-03, -1.7098e-02, -1.0155e-02],\n",
      "          [ 8.7280e-03, -1.7876e-02,  1.1886e-02]],\n",
      "\n",
      "         [[ 1.7867e-02,  1.3551e-02,  1.1897e-02],\n",
      "          [-1.4141e-02, -1.3041e-02,  1.7042e-02],\n",
      "          [ 1.2788e-02,  1.1259e-02, -1.6261e-04]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-9.4437e-03, -2.8418e-03, -7.7057e-03],\n",
      "          [-1.5704e-02, -1.5590e-02,  1.8455e-02],\n",
      "          [ 1.2589e-02, -1.6974e-03, -1.8368e-02]],\n",
      "\n",
      "         [[ 2.0622e-02, -7.6590e-03,  4.7307e-03],\n",
      "          [ 4.6183e-03, -4.0654e-03, -1.0795e-02],\n",
      "          [ 4.3997e-03, -1.9197e-02,  6.3018e-03]],\n",
      "\n",
      "         [[ 1.4793e-03,  4.7714e-03,  1.5816e-02],\n",
      "          [-6.2649e-03, -9.8083e-03,  3.8002e-03],\n",
      "          [-9.4633e-03, -1.4299e-02,  1.0463e-02]]],\n",
      "\n",
      "\n",
      "        [[[ 4.1895e-03,  6.8924e-04, -1.8520e-03],\n",
      "          [-9.8599e-03, -1.0930e-02, -1.1787e-02],\n",
      "          [-6.9337e-03,  7.4076e-03,  1.2627e-02]],\n",
      "\n",
      "         [[ 6.9599e-03, -1.3037e-02,  6.9433e-03],\n",
      "          [-1.8032e-02,  8.7660e-03, -1.9035e-02],\n",
      "          [ 5.8142e-03,  2.0271e-03, -1.3650e-03]],\n",
      "\n",
      "         [[ 1.2100e-02, -1.5581e-02,  8.9158e-04],\n",
      "          [-1.0311e-03,  7.6497e-03, -1.5839e-03],\n",
      "          [ 9.2580e-03, -1.6875e-02,  9.0082e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-4.9103e-03, -1.7625e-02, -6.6143e-03],\n",
      "          [-1.5459e-02,  1.0240e-02,  1.4516e-02],\n",
      "          [-8.2231e-03,  1.7171e-02, -1.3347e-02]],\n",
      "\n",
      "         [[-2.5700e-04, -1.6255e-02, -1.8827e-02],\n",
      "          [ 1.9894e-02, -4.0470e-03, -8.1292e-04],\n",
      "          [ 1.6972e-02, -1.6927e-02,  1.3218e-02]],\n",
      "\n",
      "         [[ 1.8167e-02,  2.0803e-02, -3.4352e-03],\n",
      "          [ 1.9535e-02, -5.8348e-03,  1.5030e-02],\n",
      "          [ 1.9461e-02, -5.2085e-03, -2.5773e-03]]],\n",
      "\n",
      "\n",
      "        [[[ 1.8523e-02, -2.0225e-03, -3.6530e-03],\n",
      "          [-1.5266e-02, -1.6298e-04, -4.5599e-03],\n",
      "          [ 1.5696e-04,  1.4999e-02, -1.8776e-02]],\n",
      "\n",
      "         [[ 1.4874e-02,  1.6757e-02,  1.3847e-02],\n",
      "          [ 5.9783e-03,  3.9499e-03, -2.0745e-02],\n",
      "          [ 1.7663e-02, -8.2206e-03, -1.5668e-02]],\n",
      "\n",
      "         [[ 1.8349e-02,  5.2687e-03,  6.2379e-03],\n",
      "          [-1.8132e-02, -1.1459e-02,  1.2140e-02],\n",
      "          [ 5.5746e-04, -7.1912e-03,  9.4814e-05]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 1.7373e-03, -1.3648e-02, -3.2191e-03],\n",
      "          [-1.1779e-03,  1.1515e-02, -9.9889e-03],\n",
      "          [ 4.1310e-03, -3.9326e-03, -2.8398e-03]],\n",
      "\n",
      "         [[-3.8769e-04, -3.3592e-03,  1.0520e-02],\n",
      "          [-5.6845e-03, -1.9909e-02, -6.1542e-03],\n",
      "          [ 6.7243e-03,  8.5014e-03, -8.7509e-03]],\n",
      "\n",
      "         [[ 1.2395e-02,  1.3333e-03,  9.4238e-03],\n",
      "          [-7.4159e-03, -1.8876e-03,  1.0147e-02],\n",
      "          [-2.0068e-02,  1.5250e-02, -6.0396e-03]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[-1.7790e-02,  1.6193e-02,  1.2643e-02],\n",
      "          [-1.8627e-03,  1.0624e-02,  1.0891e-02],\n",
      "          [-8.6188e-03,  1.5822e-03,  1.4698e-03]],\n",
      "\n",
      "         [[ 1.8218e-02, -1.9175e-03, -1.0213e-02],\n",
      "          [-2.0832e-02, -3.6198e-03,  1.4241e-02],\n",
      "          [ 8.8010e-03, -1.5754e-02,  4.3081e-03]],\n",
      "\n",
      "         [[ 4.7186e-03,  1.0364e-02,  1.9521e-02],\n",
      "          [ 1.1998e-03, -1.9957e-02, -3.7630e-03],\n",
      "          [-1.3875e-02,  8.2847e-03,  1.8666e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-8.7047e-03, -1.1739e-02, -1.8790e-02],\n",
      "          [-1.4883e-02,  2.4388e-03,  2.5637e-03],\n",
      "          [ 2.0460e-02, -2.7495e-03,  8.9057e-03]],\n",
      "\n",
      "         [[ 7.5140e-03,  6.7462e-03, -8.7686e-04],\n",
      "          [ 1.7837e-02, -8.2641e-03,  1.5648e-02],\n",
      "          [-9.1266e-03, -7.8543e-04, -3.0705e-03]],\n",
      "\n",
      "         [[-1.1997e-02, -1.2812e-02, -4.4824e-04],\n",
      "          [-6.6235e-03,  7.6706e-04, -1.0225e-02],\n",
      "          [-1.7419e-02,  2.0605e-02,  2.7727e-03]]],\n",
      "\n",
      "\n",
      "        [[[ 6.6257e-03,  1.3749e-02, -6.4766e-04],\n",
      "          [ 1.4224e-02,  1.0133e-02, -1.4120e-02],\n",
      "          [ 1.2090e-02, -1.0207e-02,  5.7440e-03]],\n",
      "\n",
      "         [[-1.4423e-03,  1.3313e-02,  6.2760e-03],\n",
      "          [ 3.4538e-03,  8.6603e-03,  1.2482e-02],\n",
      "          [ 1.3124e-02, -1.7596e-02, -1.6524e-02]],\n",
      "\n",
      "         [[ 3.3306e-03, -1.1061e-02, -2.3007e-04],\n",
      "          [ 6.8359e-03, -3.0783e-03, -1.4596e-02],\n",
      "          [ 6.9196e-03, -1.0880e-02,  1.1677e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-1.0518e-02, -8.4454e-03,  1.0809e-04],\n",
      "          [ 1.9847e-02, -1.4071e-02, -1.8752e-03],\n",
      "          [ 2.0707e-02,  1.0497e-02,  9.7460e-03]],\n",
      "\n",
      "         [[ 1.5981e-02,  4.6925e-03, -2.0080e-02],\n",
      "          [ 9.9810e-03,  6.1561e-03, -6.3786e-04],\n",
      "          [ 6.3403e-04,  8.6620e-04, -1.3215e-02]],\n",
      "\n",
      "         [[ 1.1061e-02, -5.8779e-03,  7.9803e-04],\n",
      "          [-5.4869e-03, -1.8142e-02,  3.5266e-03],\n",
      "          [ 1.8078e-02,  1.3707e-02,  7.6611e-04]]],\n",
      "\n",
      "\n",
      "        [[[ 1.8991e-03,  1.9555e-02,  1.4838e-03],\n",
      "          [ 2.0035e-02, -8.1587e-03, -1.2160e-02],\n",
      "          [ 4.1767e-03, -5.4460e-04, -2.0820e-02]],\n",
      "\n",
      "         [[-1.4390e-02,  1.1585e-02,  6.8705e-03],\n",
      "          [-1.2291e-02, -1.7332e-02, -4.1408e-03],\n",
      "          [-1.7580e-03, -1.1749e-02,  1.6680e-02]],\n",
      "\n",
      "         [[ 1.3999e-03,  6.5773e-03, -1.8787e-02],\n",
      "          [ 1.6550e-02, -1.6890e-02, -1.3572e-03],\n",
      "          [-1.6859e-02,  8.6403e-03, -1.9099e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 2.0280e-02, -1.6368e-02,  2.9318e-03],\n",
      "          [ 7.4248e-03,  1.6847e-02,  1.0718e-02],\n",
      "          [-4.3484e-03, -5.9367e-03,  1.8648e-02]],\n",
      "\n",
      "         [[ 1.1864e-03, -4.0052e-03, -1.8662e-02],\n",
      "          [ 5.9987e-03,  1.8790e-02,  9.7597e-04],\n",
      "          [ 2.0160e-02, -1.9009e-02,  1.6274e-02]],\n",
      "\n",
      "         [[ 4.7151e-03, -2.0028e-02, -1.3031e-02],\n",
      "          [ 3.5248e-03, -1.0642e-02, -9.1761e-04],\n",
      "          [-1.2461e-02,  8.3885e-03, -5.1131e-04]]]], device='cuda:0')\n",
      "6.1.weight: tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1.], device='cuda:0')\n",
      "6.1.bias: tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
      "9.weight: tensor([[ 0.0607, -0.0400, -0.0111, -0.0828,  0.0350,  0.0520,  0.0580, -0.0254,\n",
      "         -0.0799, -0.0220, -0.0772, -0.0207,  0.0493, -0.0729, -0.0248, -0.0563,\n",
      "          0.0143, -0.0073,  0.0146,  0.0760,  0.0288, -0.0839,  0.0627, -0.0195,\n",
      "         -0.0323, -0.0785, -0.0387, -0.0343, -0.0500, -0.0681,  0.0441, -0.0128,\n",
      "          0.0301, -0.0410, -0.0585,  0.0709,  0.0333, -0.0755,  0.0771,  0.0589,\n",
      "         -0.0277, -0.0444,  0.0253, -0.0047,  0.0173,  0.0297,  0.0232, -0.0542,\n",
      "         -0.0293,  0.0057,  0.0328,  0.0266, -0.0678, -0.0620, -0.0659, -0.0336,\n",
      "          0.0455, -0.0450, -0.0764,  0.0509,  0.0559,  0.0882,  0.0084,  0.0618,\n",
      "         -0.0241,  0.0361, -0.0041,  0.0615, -0.0183,  0.0840, -0.0770,  0.0496,\n",
      "          0.0388, -0.0231, -0.0264,  0.0245, -0.0176,  0.0516, -0.0188,  0.0149,\n",
      "         -0.0249,  0.0101,  0.0572, -0.0393,  0.0154,  0.0483, -0.0811,  0.0633,\n",
      "         -0.0726, -0.0185, -0.0297,  0.0763,  0.0646, -0.0850,  0.0543,  0.0227,\n",
      "         -0.0220, -0.0807,  0.0285, -0.0265,  0.0596,  0.0055,  0.0563,  0.0057,\n",
      "         -0.0843, -0.0709, -0.0093,  0.0782, -0.0278, -0.0838,  0.0832,  0.0387,\n",
      "          0.0413,  0.0576, -0.0795, -0.0261,  0.0140, -0.0009, -0.0321,  0.0414,\n",
      "          0.0010,  0.0808, -0.0480,  0.0312,  0.0145, -0.0454,  0.0279, -0.0491],\n",
      "        [ 0.0093, -0.0857, -0.0371, -0.0415, -0.0540,  0.0660,  0.0446, -0.0711,\n",
      "          0.0019,  0.0412,  0.0513, -0.0474, -0.0543,  0.0412, -0.0283,  0.0492,\n",
      "         -0.0481, -0.0347,  0.0827, -0.0466, -0.0495, -0.0357, -0.0129,  0.0732,\n",
      "          0.0032,  0.0511,  0.0438,  0.0757,  0.0366,  0.0319,  0.0738, -0.0805,\n",
      "          0.0867, -0.0845,  0.0871,  0.0695,  0.0257,  0.0416, -0.0609,  0.0172,\n",
      "          0.0535, -0.0622, -0.0597,  0.0849,  0.0475, -0.0643, -0.0113,  0.0756,\n",
      "          0.0641,  0.0674, -0.0767,  0.0570,  0.0595, -0.0559, -0.0681,  0.0145,\n",
      "          0.0415, -0.0859, -0.0071, -0.0501,  0.0255,  0.0283, -0.0777, -0.0811,\n",
      "         -0.0167, -0.0669, -0.0583, -0.0565,  0.0082, -0.0305,  0.0546,  0.0827,\n",
      "          0.0673,  0.0191, -0.0203, -0.0496,  0.0384, -0.0449,  0.0556,  0.0436,\n",
      "         -0.0850,  0.0878, -0.0426, -0.0669, -0.0124, -0.0331,  0.0024,  0.0073,\n",
      "          0.0821, -0.0274,  0.0599,  0.0043,  0.0553, -0.0710,  0.0419,  0.0840,\n",
      "         -0.0217, -0.0056,  0.0767,  0.0665, -0.0803, -0.0023, -0.0188, -0.0276,\n",
      "          0.0829, -0.0255,  0.0612, -0.0866,  0.0207, -0.0675, -0.0525,  0.0242,\n",
      "         -0.0201,  0.0874, -0.0838,  0.0834, -0.0395, -0.0613, -0.0692, -0.0651,\n",
      "         -0.0069, -0.0861, -0.0637,  0.0857, -0.0508, -0.0179, -0.0169,  0.0825]],\n",
      "       device='cuda:0')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [01:41<00:00,  1.97it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>▁▁▁▁▁▂▂▂▂▂▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇▇███</td></tr><tr><td>test_accuracy</td><td>▅▅▅▅▅▆▁▆▃▆▇▆▇▇▇▆▇█▇▇▇▇██████▇███▇██▇████</td></tr><tr><td>test_loss</td><td>█▄▃▃▄▃▃▃▂▆▂▃▃▂▂▁▁▂▂▂▃▁▃▁▁▁▁▂▁▂▂▂▂▂▂▂▁▁▁▂</td></tr><tr><td>train_accuracy</td><td>▃▄▃▅▁█▄▄▃▃▅█▇▅▇▆▅▄▇▆▆▇▄▆██▇█▇██████▇████</td></tr><tr><td>train_loss</td><td>▄▆█▅▅▇▅▇▄▆▆▅▃▇▅▄▃▄▆▅▄▅▅▃▄▆▃▂▂▁▂▂▅▁▂▃▂▁▃▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>199</td></tr><tr><td>event</td><td>model_update</td></tr><tr><td>train_accuracy</td><td>0.96875</td></tr><tr><td>train_loss</td><td>0.03103</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "You can sync this run to the cloud by running:<br/><code>wandb sync /home/gridsan/djuna/TsaiMadry_shared/datamodels_clustering/wandb_logs/Experiment_seed/wandb/offline-run-20241017_181236-pc2g0lo7<code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>/home/gridsan/djuna/TsaiMadry_shared/datamodels_clustering/wandb_logs/Experiment_seed/wandb/offline-run-20241017_181236-pc2g0lo7/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.18.4"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "W&B syncing is set to <code>`offline`<code> in this directory.  <br/>Run <code>`wandb online`<code> or set <code>WANDB_MODE=online<code> to enable cloud syncing."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  5   9  26  27  38  67  71  73  77  78  87  90  91  94  95  97 103 107\n",
      " 116 122 138 145 156 163 171 177 178 187 194 198 206 218 220 239 241 247\n",
      " 252 275 278 279 282 297 308 330 349 350 357 368 382 393]\n",
      "Initial model weights:\n",
      "0.0.weight: tensor([[[[ 0.0309, -0.1148, -0.1857],\n",
      "          [-0.0242,  0.0949,  0.1518],\n",
      "          [-0.1508, -0.1182,  0.1801]],\n",
      "\n",
      "         [[ 0.1701, -0.1021,  0.1673],\n",
      "          [ 0.0647, -0.0378,  0.1059],\n",
      "          [-0.0526, -0.0031, -0.1764]],\n",
      "\n",
      "         [[ 0.0319, -0.1768,  0.0601],\n",
      "          [ 0.0126,  0.0306,  0.1360],\n",
      "          [-0.1224,  0.0473, -0.1851]]],\n",
      "\n",
      "\n",
      "        [[[-0.0987,  0.1626, -0.0029],\n",
      "          [-0.1364, -0.1714,  0.0228],\n",
      "          [ 0.0165, -0.0104, -0.0809]],\n",
      "\n",
      "         [[-0.1820, -0.1516,  0.1171],\n",
      "          [-0.0065,  0.1243, -0.0491],\n",
      "          [-0.1492,  0.0758,  0.1216]],\n",
      "\n",
      "         [[ 0.0919, -0.1885, -0.0146],\n",
      "          [-0.0023,  0.0722,  0.1059],\n",
      "          [-0.1713, -0.0600, -0.1658]]],\n",
      "\n",
      "\n",
      "        [[[ 0.1831,  0.1503,  0.1573],\n",
      "          [ 0.1145,  0.1114, -0.0643],\n",
      "          [ 0.0498, -0.0239, -0.1496]],\n",
      "\n",
      "         [[-0.0884, -0.1555,  0.0918],\n",
      "          [ 0.0532, -0.1010, -0.1284],\n",
      "          [ 0.0101, -0.1123, -0.0130]],\n",
      "\n",
      "         [[-0.1276,  0.0217, -0.1483],\n",
      "          [-0.1172, -0.1654,  0.0854],\n",
      "          [ 0.0655, -0.0546,  0.0162]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[ 0.1208, -0.0295,  0.1911],\n",
      "          [-0.1803, -0.1636,  0.0984],\n",
      "          [-0.1287, -0.1867, -0.1808]],\n",
      "\n",
      "         [[ 0.0061,  0.1849, -0.1446],\n",
      "          [-0.1082, -0.1709,  0.0703],\n",
      "          [ 0.0975, -0.0777, -0.1750]],\n",
      "\n",
      "         [[-0.1016, -0.0570, -0.0613],\n",
      "          [-0.1867,  0.1775,  0.1134],\n",
      "          [-0.1723,  0.1094,  0.1071]]],\n",
      "\n",
      "\n",
      "        [[[-0.0770,  0.1676,  0.1524],\n",
      "          [ 0.0983, -0.0213,  0.1396],\n",
      "          [-0.0133,  0.0942, -0.1532]],\n",
      "\n",
      "         [[-0.1529,  0.1195,  0.1151],\n",
      "          [ 0.1562, -0.1092, -0.0679],\n",
      "          [-0.1526,  0.0285, -0.0483]],\n",
      "\n",
      "         [[ 0.1042,  0.1822, -0.0414],\n",
      "          [-0.0602, -0.0961, -0.0249],\n",
      "          [-0.0916,  0.1728,  0.1469]]],\n",
      "\n",
      "\n",
      "        [[[ 0.1007, -0.0088,  0.1908],\n",
      "          [ 0.1213, -0.1149, -0.1555],\n",
      "          [-0.1730, -0.0813,  0.1444]],\n",
      "\n",
      "         [[-0.1479, -0.1193,  0.1378],\n",
      "          [-0.0231, -0.1331,  0.1637],\n",
      "          [-0.1769,  0.1075,  0.1227]],\n",
      "\n",
      "         [[ 0.0833,  0.1129, -0.1391],\n",
      "          [-0.1463, -0.0324,  0.0291],\n",
      "          [ 0.0079,  0.0738, -0.0484]]]], device='cuda:0')\n",
      "0.1.weight: tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1.], device='cuda:0')\n",
      "0.1.bias: tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "       device='cuda:0')\n",
      "1.0.weight: tensor([[[[ 7.9878e-03,  2.1543e-02,  2.3997e-02, -9.6783e-03,  2.6883e-03],\n",
      "          [-1.7576e-02,  1.1917e-02, -5.7715e-03,  1.5277e-02, -2.3138e-02],\n",
      "          [ 1.9880e-02,  2.1874e-02, -1.3788e-02, -1.6147e-02, -9.0453e-03],\n",
      "          [ 9.8151e-03, -5.8033e-03, -3.3377e-03,  1.4927e-02,  1.1057e-02],\n",
      "          [-7.8216e-03,  3.9286e-04,  2.3237e-03, -1.3291e-02, -1.6507e-02]],\n",
      "\n",
      "         [[ 1.9925e-02, -1.7170e-03, -1.2872e-02, -2.2660e-02,  1.1678e-02],\n",
      "          [ 2.0963e-02, -1.6119e-02,  1.5335e-02,  1.5390e-02, -1.2033e-02],\n",
      "          [ 1.8388e-02, -1.6774e-02, -1.9426e-03,  1.1899e-02,  2.2628e-02],\n",
      "          [ 2.1081e-02, -3.6048e-03, -1.9584e-02,  2.1109e-02, -1.4575e-02],\n",
      "          [-2.1993e-02, -1.6933e-04, -1.5649e-02, -9.3233e-03,  3.8302e-03]],\n",
      "\n",
      "         [[ 1.8399e-02, -9.8102e-03, -1.4323e-02, -1.2266e-02, -3.5446e-03],\n",
      "          [ 7.1657e-03, -2.4304e-02, -1.1976e-02, -6.8972e-03, -1.7164e-02],\n",
      "          [-5.2625e-03,  1.0806e-02, -7.3292e-03, -1.9185e-02,  5.4492e-03],\n",
      "          [ 1.3075e-02, -1.2383e-02, -9.0436e-03,  1.7233e-02,  1.9698e-02],\n",
      "          [ 2.7315e-03, -1.6815e-02,  8.9032e-03, -5.6038e-03, -1.5647e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-2.2767e-02,  2.3716e-03, -1.5153e-02,  2.5799e-03,  1.0345e-03],\n",
      "          [ 1.6791e-02, -2.4330e-02,  8.3934e-03,  3.4334e-04, -1.7799e-02],\n",
      "          [-2.1168e-02,  1.1531e-02, -6.3418e-04, -1.5474e-02,  1.5466e-03],\n",
      "          [-1.8188e-03, -4.0875e-03,  1.9275e-03,  1.1578e-02, -1.3308e-02],\n",
      "          [-7.1363e-03, -1.5432e-02, -1.6060e-02,  1.2635e-02, -1.0153e-02]],\n",
      "\n",
      "         [[ 1.6809e-02,  2.1652e-02, -2.1481e-02, -1.6417e-02,  1.9392e-02],\n",
      "          [-1.9866e-02, -2.1631e-02,  8.1759e-03,  7.6703e-03,  9.3504e-03],\n",
      "          [ 2.3124e-02, -1.6202e-02,  6.9235e-03,  1.4269e-03, -1.8759e-02],\n",
      "          [-9.4216e-03, -3.0625e-03,  8.1416e-03,  7.0385e-03, -2.3045e-02],\n",
      "          [-7.6228e-03, -1.7199e-02,  2.4319e-03,  2.0839e-02,  6.2781e-03]],\n",
      "\n",
      "         [[ 2.3736e-02,  2.2615e-02, -1.7048e-03, -1.2433e-02, -5.8501e-03],\n",
      "          [ 1.5590e-02,  2.1491e-02, -1.3132e-02,  8.2116e-03, -1.0612e-02],\n",
      "          [ 1.9941e-03,  2.2996e-02, -2.3495e-02,  6.7637e-04, -1.5966e-02],\n",
      "          [-1.6453e-02,  1.1837e-02, -1.4077e-02,  2.2843e-02,  8.7609e-03],\n",
      "          [ 2.4136e-02,  2.3509e-02, -9.9036e-03, -1.5017e-02, -1.7244e-03]]],\n",
      "\n",
      "\n",
      "        [[[-2.0111e-02,  4.6744e-03,  1.0170e-02,  1.8922e-02,  1.3318e-02],\n",
      "          [ 2.3856e-02,  5.2422e-03, -1.5199e-02, -1.6378e-02, -1.6028e-02],\n",
      "          [-2.2542e-02, -1.0242e-02, -1.3598e-02,  4.4202e-03, -1.8609e-02],\n",
      "          [ 2.8067e-03,  6.7461e-03, -1.5270e-02, -6.8101e-03,  5.8562e-03],\n",
      "          [-5.6671e-04, -1.0800e-02, -2.2034e-02,  4.0978e-04, -1.0172e-02]],\n",
      "\n",
      "         [[-1.7184e-02, -1.0092e-02,  1.9476e-02, -1.1496e-02,  2.1341e-02],\n",
      "          [ 2.4215e-02,  6.9885e-04,  1.0691e-02, -6.1691e-03,  1.6692e-03],\n",
      "          [-2.1766e-02,  1.0383e-02,  1.2651e-02, -1.8834e-02,  6.3701e-03],\n",
      "          [-2.1735e-02,  8.1350e-03, -1.9372e-02, -2.5073e-03, -2.3779e-02],\n",
      "          [ 1.3073e-02, -1.5827e-02,  4.6801e-03,  2.8320e-03, -1.3355e-02]],\n",
      "\n",
      "         [[-4.9575e-03, -2.4600e-02, -1.3113e-02, -1.6640e-02, -1.8397e-02],\n",
      "          [-1.5636e-02, -1.4601e-02,  2.4233e-02, -2.1790e-02,  8.9759e-03],\n",
      "          [-8.3059e-03, -4.1916e-03,  1.9945e-02, -2.0565e-02, -1.0645e-02],\n",
      "          [ 7.2920e-03,  2.3146e-02, -1.9395e-03,  4.6374e-03, -5.8273e-03],\n",
      "          [ 1.2292e-02, -5.1360e-03, -2.2427e-02, -1.0307e-02, -2.5451e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-4.1538e-04,  8.3461e-03, -8.1447e-03, -1.9885e-03, -6.7439e-03],\n",
      "          [ 2.5719e-03, -2.2915e-02, -1.4857e-02,  1.2644e-02, -8.1428e-03],\n",
      "          [-9.4498e-03, -1.8164e-02, -8.5181e-03, -2.1200e-02,  3.3745e-03],\n",
      "          [-2.0751e-02,  1.7815e-03, -1.3069e-02, -4.1069e-03,  4.1842e-03],\n",
      "          [ 1.6190e-02,  1.2894e-02,  4.2483e-04,  3.4855e-04, -4.5629e-03]],\n",
      "\n",
      "         [[-1.8096e-02, -2.4998e-03,  7.9548e-03,  3.0046e-03,  2.3668e-02],\n",
      "          [ 8.7814e-04,  6.6078e-03,  1.4739e-02, -1.8626e-02,  1.8065e-02],\n",
      "          [ 3.0151e-03, -2.2925e-02, -7.6555e-03,  2.3372e-02, -9.2764e-03],\n",
      "          [-2.9380e-03,  1.3537e-03, -1.1204e-02,  2.2668e-02, -3.4196e-03],\n",
      "          [-8.0708e-03,  8.2696e-03, -8.8912e-03, -1.9266e-02, -7.1472e-03]],\n",
      "\n",
      "         [[-1.1360e-02, -4.9271e-03, -1.8479e-02, -1.7274e-02, -9.2342e-03],\n",
      "          [-1.1026e-03,  4.5520e-03, -9.1086e-03, -1.5565e-03,  1.4106e-03],\n",
      "          [ 1.4095e-02, -2.4593e-02, -1.2623e-02, -8.5876e-03,  1.8026e-02],\n",
      "          [ 1.0391e-02, -9.5261e-03, -6.9458e-03,  2.2281e-02, -2.4301e-02],\n",
      "          [-8.9517e-03, -2.3120e-02, -5.7925e-03,  1.8558e-02, -5.9494e-03]]],\n",
      "\n",
      "\n",
      "        [[[-3.1947e-03, -7.1255e-04,  1.6370e-02,  1.8616e-02,  5.7643e-03],\n",
      "          [ 1.5877e-02,  1.9610e-02, -8.7375e-03, -1.8856e-02, -1.8628e-02],\n",
      "          [-7.1769e-03, -1.3699e-02,  5.8585e-03,  1.1452e-02,  4.4395e-03],\n",
      "          [-4.2934e-03, -2.8767e-03, -2.3404e-02, -6.4015e-03,  1.5064e-02],\n",
      "          [ 2.2780e-02,  2.2388e-02,  3.1213e-03, -1.3679e-02, -2.2900e-02]],\n",
      "\n",
      "         [[-1.2665e-02, -1.5631e-02, -4.2995e-03, -2.4814e-02, -1.1458e-02],\n",
      "          [-3.2103e-04, -3.6044e-03, -1.6003e-02,  6.9336e-03,  2.4311e-02],\n",
      "          [ 5.9224e-03,  2.0891e-02, -3.8423e-03, -8.1985e-03, -1.9321e-02],\n",
      "          [-1.0873e-02, -1.0886e-02, -7.2701e-03, -3.6069e-03, -2.2924e-02],\n",
      "          [ 1.7431e-02, -2.3843e-03, -1.8832e-02, -1.6213e-03, -2.2486e-02]],\n",
      "\n",
      "         [[ 5.6681e-03,  2.2189e-02,  6.9708e-03, -2.6344e-03,  1.4642e-02],\n",
      "          [ 1.4257e-02, -2.2244e-02, -2.4216e-02, -6.6047e-03, -2.0444e-02],\n",
      "          [-1.9685e-03, -7.5916e-03, -1.3210e-02, -1.3331e-02, -4.2263e-03],\n",
      "          [ 2.1619e-02,  2.4467e-02, -1.8995e-02,  9.4811e-04, -9.8851e-03],\n",
      "          [ 2.4705e-02, -2.3945e-02, -2.0426e-02,  2.9886e-03,  1.5170e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-1.1168e-02,  1.3650e-02, -2.0483e-02,  8.9795e-03,  1.0019e-02],\n",
      "          [-1.4597e-02, -1.0894e-02, -7.4044e-03, -1.1971e-02,  8.4514e-03],\n",
      "          [ 8.7033e-03,  1.3975e-02, -1.0647e-02, -3.2360e-03, -1.0097e-03],\n",
      "          [-2.2136e-02,  2.3016e-02,  8.7813e-04, -9.2304e-04,  6.3708e-03],\n",
      "          [ 1.4078e-02, -2.8786e-03,  5.8662e-03, -2.0634e-02, -4.7634e-03]],\n",
      "\n",
      "         [[-3.3536e-03, -1.1883e-03,  2.7198e-03, -3.4144e-03, -8.0986e-03],\n",
      "          [-1.7314e-02,  2.1832e-02, -1.1331e-02, -1.8793e-02, -1.7927e-02],\n",
      "          [-2.4796e-02, -9.7927e-04, -1.4647e-02,  1.9924e-02, -2.2049e-02],\n",
      "          [-1.6024e-02, -1.5870e-02,  2.0746e-02, -7.1521e-03, -1.8675e-02],\n",
      "          [ 1.5489e-02, -1.9362e-03, -4.2185e-03,  1.3706e-03,  2.4960e-02]],\n",
      "\n",
      "         [[ 5.0777e-03, -2.2129e-02,  1.2899e-02, -9.1484e-04, -2.4286e-02],\n",
      "          [ 9.1445e-03,  1.2828e-02,  1.6614e-02, -2.1252e-02,  8.5077e-03],\n",
      "          [-4.6689e-03, -2.3644e-02, -2.1426e-02,  8.8714e-03, -1.7909e-02],\n",
      "          [-2.0775e-02, -2.3794e-02,  1.1892e-02, -2.0703e-02,  1.4034e-02],\n",
      "          [-4.1171e-03, -1.7322e-02,  1.9152e-02, -9.5134e-03, -2.1976e-02]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[ 2.3672e-02, -1.4225e-02,  1.3590e-02, -5.0670e-03, -1.9906e-02],\n",
      "          [-7.2608e-03, -1.9766e-02,  2.3912e-02, -1.3312e-02, -1.4421e-02],\n",
      "          [ 2.3838e-02,  8.5525e-03, -8.4121e-03, -5.1426e-04,  1.8979e-02],\n",
      "          [-5.2297e-03,  4.4700e-03,  5.2344e-03,  9.7512e-03, -1.9238e-03],\n",
      "          [ 2.0013e-02, -2.1410e-02,  1.6026e-02, -4.4290e-03, -1.2660e-02]],\n",
      "\n",
      "         [[ 8.5051e-03,  6.5245e-03,  1.9162e-02,  6.0518e-03, -4.2854e-03],\n",
      "          [ 7.2351e-03, -7.5607e-03, -2.1540e-02, -5.9908e-03, -1.0730e-02],\n",
      "          [ 1.4189e-02,  3.5766e-03, -6.2854e-03, -5.4345e-03,  1.9228e-02],\n",
      "          [-1.4014e-02,  1.0448e-03,  1.4991e-02,  1.0898e-02,  1.9429e-02],\n",
      "          [ 5.0480e-03,  1.9142e-02, -6.4258e-03,  8.8901e-03,  9.8451e-03]],\n",
      "\n",
      "         [[-2.3783e-02, -8.4901e-03,  6.8476e-03,  2.1439e-02, -4.0157e-03],\n",
      "          [-9.5845e-04, -2.4662e-02,  2.4243e-02,  1.8600e-02,  9.9198e-03],\n",
      "          [-1.1142e-02, -2.2836e-02, -1.8375e-02,  1.8641e-02, -8.8829e-03],\n",
      "          [ 2.1826e-03, -1.7861e-03, -2.1751e-02,  1.3998e-02, -1.4434e-02],\n",
      "          [ 2.0368e-02,  2.8432e-03, -1.2208e-02,  1.4178e-02, -9.9720e-04]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-3.4721e-03,  1.3719e-02,  1.4861e-02,  7.8947e-03,  1.7049e-02],\n",
      "          [-9.8839e-03,  4.2866e-04, -1.4686e-02,  1.4502e-02, -2.0734e-02],\n",
      "          [-1.7715e-02,  2.3094e-02, -2.3689e-02,  1.2780e-02,  1.2050e-03],\n",
      "          [-4.5456e-03,  1.2426e-02,  1.1822e-02, -3.9961e-03,  2.4927e-02],\n",
      "          [-2.2428e-02, -9.1467e-03,  7.7605e-03,  2.0979e-03, -2.2044e-02]],\n",
      "\n",
      "         [[-9.7721e-03,  1.0644e-02, -1.0999e-02,  4.3635e-03, -1.0015e-02],\n",
      "          [ 3.8781e-03,  1.6408e-02, -7.0417e-03, -1.9356e-02, -1.9602e-03],\n",
      "          [-2.1121e-02, -2.0325e-02, -1.3661e-02, -4.1296e-03,  6.9937e-03],\n",
      "          [-1.5239e-03,  8.7882e-03, -1.9090e-02,  1.0605e-02, -3.1410e-03],\n",
      "          [ 1.5205e-02, -1.5478e-02,  1.5283e-02,  5.8942e-03,  1.9147e-02]],\n",
      "\n",
      "         [[ 3.2951e-03,  9.2042e-03, -8.4778e-03, -1.2138e-03, -2.5449e-03],\n",
      "          [ 5.7562e-03, -6.7316e-03,  1.7461e-03, -4.1773e-03, -1.0238e-02],\n",
      "          [ 1.6043e-03,  8.0059e-03,  2.3592e-03,  2.1627e-02,  8.9671e-03],\n",
      "          [-1.0457e-02, -8.7621e-03,  1.2111e-02,  1.1568e-02,  1.8186e-03],\n",
      "          [-2.3376e-02, -9.6991e-03,  1.4644e-02, -2.3783e-02,  3.6708e-03]]],\n",
      "\n",
      "\n",
      "        [[[ 1.2784e-02,  1.4700e-02, -4.6309e-03,  1.8576e-02, -7.3200e-03],\n",
      "          [ 1.7119e-02,  9.4797e-03, -2.1462e-02,  1.5736e-03,  1.2183e-02],\n",
      "          [-2.2648e-02,  1.7825e-02,  1.1310e-02,  3.3230e-03,  8.7168e-03],\n",
      "          [-1.6150e-02, -1.0508e-02,  1.9555e-02, -1.5799e-02,  2.2249e-03],\n",
      "          [ 7.9795e-03, -1.6160e-02,  4.7444e-03,  7.2371e-04, -2.7842e-03]],\n",
      "\n",
      "         [[ 1.4871e-02, -2.3806e-02, -1.4636e-02,  1.2852e-02, -4.8055e-03],\n",
      "          [-5.2640e-03,  9.0016e-03,  2.3765e-03, -1.2112e-03,  8.5443e-03],\n",
      "          [ 1.3943e-02,  1.9614e-02,  2.4733e-02, -2.3418e-02,  2.2111e-02],\n",
      "          [-2.3489e-02, -9.0592e-04,  1.9701e-02, -4.2865e-03, -2.1142e-02],\n",
      "          [ 3.9236e-03, -4.4173e-05, -2.1129e-02,  8.3869e-03, -2.6963e-03]],\n",
      "\n",
      "         [[-3.7644e-03,  9.9451e-03, -9.9651e-03, -3.3674e-03, -2.4438e-02],\n",
      "          [-1.1169e-02, -3.2898e-03,  1.5521e-02,  5.7556e-03,  2.2920e-02],\n",
      "          [-2.4571e-02,  1.5400e-02,  1.7512e-02,  6.4618e-03,  2.3449e-02],\n",
      "          [-1.1875e-02, -1.0859e-02, -3.4210e-05,  7.6355e-03, -1.1341e-02],\n",
      "          [-1.0616e-02,  1.0410e-02,  2.0341e-03, -1.7469e-02,  6.3365e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 2.0950e-02,  4.7020e-03,  7.4086e-03,  4.0325e-03, -9.0473e-03],\n",
      "          [-1.6001e-02, -1.5603e-02,  3.4256e-03,  1.7957e-02, -1.7041e-02],\n",
      "          [ 2.1732e-02, -2.0536e-02, -1.2758e-02, -3.2634e-03, -8.0273e-03],\n",
      "          [-2.4793e-02, -1.6286e-03, -1.4239e-02, -3.0053e-03,  1.2749e-02],\n",
      "          [-1.1691e-02,  6.2040e-03, -9.2621e-04, -2.6830e-03, -1.5003e-02]],\n",
      "\n",
      "         [[ 1.5859e-02,  1.4093e-02,  1.3753e-02, -1.3515e-02, -3.8635e-03],\n",
      "          [ 1.6985e-02,  3.7936e-03,  4.1839e-03,  1.0689e-02, -5.1781e-03],\n",
      "          [ 1.8676e-02, -1.8653e-02,  2.3027e-02,  1.9319e-04, -2.3580e-02],\n",
      "          [ 1.9085e-02,  1.3742e-02,  1.3540e-02, -1.0732e-02, -1.4518e-02],\n",
      "          [-2.4980e-02, -7.4644e-03, -1.7785e-02, -1.2412e-02,  2.4214e-02]],\n",
      "\n",
      "         [[ 2.0141e-02, -6.8968e-03, -8.7526e-03,  5.3274e-03, -1.6935e-02],\n",
      "          [-2.2340e-02,  4.8780e-03, -2.7306e-03,  1.6871e-02, -2.8387e-04],\n",
      "          [ 8.6681e-03,  1.9684e-02, -1.6812e-02,  6.2876e-03,  1.5355e-02],\n",
      "          [ 1.9920e-02, -3.2616e-03, -6.1831e-03,  1.4997e-02,  1.3013e-02],\n",
      "          [-1.4923e-02,  3.7266e-04, -1.7675e-02,  1.6550e-02, -1.5549e-02]]],\n",
      "\n",
      "\n",
      "        [[[-1.0182e-02,  1.6762e-03,  2.3011e-02,  1.7551e-03,  1.0799e-02],\n",
      "          [-6.5532e-04,  9.1139e-03, -1.3783e-02, -2.4165e-02,  1.8096e-02],\n",
      "          [ 1.7410e-02,  5.7799e-03, -1.9836e-02,  9.2421e-03, -1.0382e-02],\n",
      "          [ 2.0727e-02,  8.6312e-03, -2.4115e-02, -2.0860e-02, -7.5667e-03],\n",
      "          [ 2.5664e-03,  3.5986e-03,  2.5933e-03,  1.3538e-02, -3.9057e-03]],\n",
      "\n",
      "         [[-2.1106e-02,  8.6195e-03, -1.5563e-02,  1.8848e-02,  2.1215e-02],\n",
      "          [ 2.1813e-02, -1.1733e-02, -1.1745e-03,  4.1773e-03, -1.6169e-03],\n",
      "          [ 3.7216e-03, -2.4663e-02, -2.0221e-02,  9.1876e-03, -1.2019e-02],\n",
      "          [-1.6608e-03,  1.7884e-02,  1.9594e-02,  1.6274e-02,  6.3561e-03],\n",
      "          [ 6.2045e-03, -3.8620e-03,  2.4416e-03, -1.9243e-02,  1.7709e-02]],\n",
      "\n",
      "         [[-1.0463e-02,  8.7348e-03,  9.9575e-03,  5.3140e-03,  1.2042e-02],\n",
      "          [-7.3901e-03,  1.4012e-02,  3.3248e-03,  2.3899e-02, -7.1300e-03],\n",
      "          [-3.3382e-03, -1.4013e-02, -1.7197e-02, -2.4328e-02, -1.3990e-02],\n",
      "          [ 1.2396e-03,  1.6680e-03, -1.3100e-03, -1.3355e-02,  1.1375e-02],\n",
      "          [ 1.0713e-02,  9.6220e-03,  2.2735e-02, -9.3683e-03,  1.3541e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-1.4579e-02, -2.9202e-03,  3.4786e-03,  1.7688e-02,  1.1365e-02],\n",
      "          [-4.4977e-03, -1.5013e-02, -6.1397e-03,  1.6757e-02, -2.9546e-03],\n",
      "          [-1.3638e-03, -2.4746e-02, -1.0646e-02, -2.3116e-02,  1.8211e-02],\n",
      "          [ 9.8686e-03, -2.3182e-02, -1.2213e-03,  1.7841e-02, -2.1523e-02],\n",
      "          [-1.3195e-02,  1.6530e-02,  1.0290e-02, -2.2562e-03,  9.8711e-03]],\n",
      "\n",
      "         [[-1.5851e-02,  1.8190e-02, -1.7594e-02,  1.7729e-02,  1.9789e-02],\n",
      "          [-6.8363e-03,  1.6080e-02,  1.0888e-02,  1.9841e-02,  1.9858e-02],\n",
      "          [-2.3775e-02,  1.2639e-02,  1.0101e-02, -2.1604e-02, -3.6497e-03],\n",
      "          [-1.8967e-02,  1.7778e-02, -2.0321e-02, -2.0776e-02, -6.2853e-03],\n",
      "          [ 1.3224e-02,  1.1013e-02, -1.1132e-02,  4.8140e-03,  1.7739e-02]],\n",
      "\n",
      "         [[-1.8962e-02,  1.4944e-02,  9.4849e-03,  1.8522e-02, -2.2759e-02],\n",
      "          [-2.1735e-02,  2.0539e-02, -6.0928e-03, -9.4674e-03, -1.9291e-02],\n",
      "          [-1.5289e-02, -7.0921e-03,  6.0719e-03,  8.9465e-03,  2.2775e-02],\n",
      "          [ 1.5016e-02,  2.6241e-03, -1.0698e-02, -3.2970e-03, -1.2496e-02],\n",
      "          [-1.7759e-02, -2.2018e-02,  3.6199e-03, -5.6703e-03,  1.6936e-02]]]],\n",
      "       device='cuda:0')\n",
      "1.1.weight: tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1.], device='cuda:0')\n",
      "1.1.bias: tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
      "2.module.0.0.weight: tensor([[[[-0.0170,  0.0283, -0.0126],\n",
      "          [ 0.0229,  0.0103,  0.0191],\n",
      "          [ 0.0128,  0.0250,  0.0196]],\n",
      "\n",
      "         [[-0.0203,  0.0182,  0.0293],\n",
      "          [-0.0267,  0.0244,  0.0170],\n",
      "          [ 0.0240,  0.0171,  0.0248]],\n",
      "\n",
      "         [[-0.0154, -0.0102, -0.0255],\n",
      "          [-0.0256, -0.0070,  0.0206],\n",
      "          [ 0.0182,  0.0154, -0.0068]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-0.0222,  0.0012, -0.0121],\n",
      "          [ 0.0066, -0.0092, -0.0189],\n",
      "          [-0.0111, -0.0085, -0.0014]],\n",
      "\n",
      "         [[-0.0276, -0.0242, -0.0217],\n",
      "          [-0.0029,  0.0193, -0.0216],\n",
      "          [-0.0258, -0.0040,  0.0039]],\n",
      "\n",
      "         [[-0.0081,  0.0087, -0.0015],\n",
      "          [ 0.0238, -0.0261, -0.0038],\n",
      "          [-0.0060,  0.0127,  0.0054]]],\n",
      "\n",
      "\n",
      "        [[[-0.0007,  0.0140,  0.0198],\n",
      "          [-0.0166,  0.0034, -0.0194],\n",
      "          [-0.0192, -0.0252, -0.0099]],\n",
      "\n",
      "         [[ 0.0195, -0.0016, -0.0092],\n",
      "          [ 0.0285, -0.0031, -0.0004],\n",
      "          [-0.0276,  0.0108, -0.0049]],\n",
      "\n",
      "         [[ 0.0212, -0.0189,  0.0185],\n",
      "          [-0.0122,  0.0142,  0.0073],\n",
      "          [-0.0109, -0.0098, -0.0139]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 0.0152,  0.0179, -0.0240],\n",
      "          [ 0.0069, -0.0122, -0.0019],\n",
      "          [ 0.0054, -0.0058, -0.0273]],\n",
      "\n",
      "         [[-0.0135, -0.0140,  0.0132],\n",
      "          [-0.0053,  0.0277, -0.0017],\n",
      "          [-0.0203, -0.0271, -0.0052]],\n",
      "\n",
      "         [[-0.0029,  0.0223, -0.0267],\n",
      "          [ 0.0059,  0.0235, -0.0066],\n",
      "          [-0.0178,  0.0241, -0.0140]]],\n",
      "\n",
      "\n",
      "        [[[-0.0100, -0.0010,  0.0132],\n",
      "          [ 0.0126,  0.0174,  0.0005],\n",
      "          [ 0.0036,  0.0131, -0.0065]],\n",
      "\n",
      "         [[-0.0068,  0.0194, -0.0146],\n",
      "          [ 0.0198,  0.0094,  0.0031],\n",
      "          [-0.0235,  0.0054,  0.0037]],\n",
      "\n",
      "         [[-0.0290, -0.0049, -0.0124],\n",
      "          [ 0.0232, -0.0028,  0.0176],\n",
      "          [-0.0280, -0.0281,  0.0007]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-0.0061, -0.0202, -0.0261],\n",
      "          [ 0.0205, -0.0010, -0.0083],\n",
      "          [-0.0257,  0.0185, -0.0029]],\n",
      "\n",
      "         [[-0.0232, -0.0055,  0.0103],\n",
      "          [ 0.0062, -0.0043,  0.0088],\n",
      "          [-0.0153, -0.0008,  0.0165]],\n",
      "\n",
      "         [[-0.0070, -0.0280, -0.0145],\n",
      "          [-0.0104, -0.0271,  0.0085],\n",
      "          [-0.0130,  0.0019,  0.0182]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[-0.0104,  0.0239, -0.0252],\n",
      "          [ 0.0201, -0.0273, -0.0100],\n",
      "          [-0.0047, -0.0143, -0.0053]],\n",
      "\n",
      "         [[-0.0210,  0.0199,  0.0148],\n",
      "          [ 0.0204,  0.0058, -0.0135],\n",
      "          [-0.0281, -0.0184,  0.0093]],\n",
      "\n",
      "         [[ 0.0022,  0.0201,  0.0057],\n",
      "          [-0.0031, -0.0241, -0.0022],\n",
      "          [ 0.0037, -0.0282,  0.0207]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-0.0030, -0.0064,  0.0097],\n",
      "          [ 0.0201, -0.0165,  0.0165],\n",
      "          [-0.0168, -0.0044, -0.0006]],\n",
      "\n",
      "         [[ 0.0098,  0.0293, -0.0176],\n",
      "          [-0.0187, -0.0021,  0.0011],\n",
      "          [ 0.0218, -0.0286,  0.0156]],\n",
      "\n",
      "         [[-0.0143,  0.0154, -0.0078],\n",
      "          [ 0.0058, -0.0016, -0.0273],\n",
      "          [-0.0252,  0.0196, -0.0267]]],\n",
      "\n",
      "\n",
      "        [[[-0.0116,  0.0013,  0.0065],\n",
      "          [-0.0119, -0.0102,  0.0022],\n",
      "          [ 0.0018, -0.0283,  0.0202]],\n",
      "\n",
      "         [[ 0.0201,  0.0024,  0.0206],\n",
      "          [ 0.0082, -0.0079, -0.0247],\n",
      "          [-0.0036, -0.0168, -0.0094]],\n",
      "\n",
      "         [[-0.0238, -0.0191, -0.0247],\n",
      "          [ 0.0043, -0.0118, -0.0196],\n",
      "          [-0.0011,  0.0014,  0.0282]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-0.0286, -0.0229,  0.0031],\n",
      "          [-0.0197,  0.0084, -0.0270],\n",
      "          [-0.0178, -0.0143,  0.0291]],\n",
      "\n",
      "         [[ 0.0093,  0.0144,  0.0295],\n",
      "          [ 0.0073, -0.0086, -0.0223],\n",
      "          [ 0.0003,  0.0008,  0.0044]],\n",
      "\n",
      "         [[-0.0159, -0.0043, -0.0059],\n",
      "          [-0.0223,  0.0200, -0.0157],\n",
      "          [-0.0110,  0.0042,  0.0128]]],\n",
      "\n",
      "\n",
      "        [[[ 0.0047,  0.0225, -0.0238],\n",
      "          [-0.0273,  0.0018, -0.0161],\n",
      "          [ 0.0247, -0.0131, -0.0183]],\n",
      "\n",
      "         [[-0.0004,  0.0141,  0.0136],\n",
      "          [ 0.0137, -0.0260,  0.0052],\n",
      "          [ 0.0023,  0.0291,  0.0150]],\n",
      "\n",
      "         [[-0.0039,  0.0281, -0.0181],\n",
      "          [ 0.0126,  0.0141,  0.0193],\n",
      "          [ 0.0280, -0.0259, -0.0052]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-0.0134, -0.0045, -0.0003],\n",
      "          [-0.0254, -0.0141,  0.0009],\n",
      "          [ 0.0067,  0.0281, -0.0235]],\n",
      "\n",
      "         [[ 0.0224, -0.0190,  0.0170],\n",
      "          [-0.0026,  0.0104, -0.0057],\n",
      "          [ 0.0160,  0.0148, -0.0055]],\n",
      "\n",
      "         [[-0.0222, -0.0286, -0.0022],\n",
      "          [ 0.0275, -0.0291, -0.0026],\n",
      "          [-0.0281, -0.0277, -0.0067]]]], device='cuda:0')\n",
      "2.module.0.1.weight: tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1.], device='cuda:0')\n",
      "2.module.0.1.bias: tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
      "3.0.weight: tensor([[[[-1.0624e-02,  2.7612e-02,  1.4109e-02],\n",
      "          [-2.4201e-02,  2.4634e-02,  6.4137e-03],\n",
      "          [ 6.6860e-03, -1.0510e-03, -1.7443e-02]],\n",
      "\n",
      "         [[ 6.4735e-03,  6.2729e-03, -1.7726e-02],\n",
      "          [-1.9135e-05,  9.1266e-03,  8.3640e-04],\n",
      "          [-1.3384e-02, -2.1154e-02, -1.0052e-03]],\n",
      "\n",
      "         [[-2.4717e-02,  2.0624e-02, -1.5934e-02],\n",
      "          [-2.3317e-02,  2.5816e-02,  1.2581e-02],\n",
      "          [ 2.4372e-02, -7.9604e-03,  2.5251e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-3.2318e-03,  3.7394e-03, -1.7844e-02],\n",
      "          [ 2.2151e-02, -1.9341e-02, -3.9379e-03],\n",
      "          [-2.7775e-02, -1.5179e-02,  6.6364e-03]],\n",
      "\n",
      "         [[-2.1874e-02, -9.7835e-03, -2.2936e-02],\n",
      "          [-2.6847e-02, -2.8990e-02,  1.2994e-03],\n",
      "          [-2.5022e-02, -5.4020e-03,  2.4039e-02]],\n",
      "\n",
      "         [[ 1.6089e-02,  2.0844e-02,  2.8305e-03],\n",
      "          [-1.2025e-03, -1.4897e-02,  7.2568e-03],\n",
      "          [-2.8760e-03,  2.8966e-02,  5.9447e-03]]],\n",
      "\n",
      "\n",
      "        [[[-2.4706e-02, -1.4559e-02,  3.0782e-03],\n",
      "          [ 2.5288e-02, -1.7147e-02, -1.3047e-02],\n",
      "          [-2.1808e-02,  4.8537e-03, -1.9986e-02]],\n",
      "\n",
      "         [[ 2.8209e-02,  3.8526e-03,  1.4471e-02],\n",
      "          [-1.4886e-02,  1.2285e-02,  1.0665e-02],\n",
      "          [-1.8899e-02,  2.4861e-03, -2.5685e-02]],\n",
      "\n",
      "         [[ 2.6162e-02,  2.2084e-02,  1.8720e-02],\n",
      "          [-2.7972e-02,  1.8377e-02,  1.4913e-02],\n",
      "          [ 6.6173e-03,  2.9062e-02, -2.8860e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 1.1980e-02,  2.9460e-02,  1.3139e-02],\n",
      "          [-2.7258e-02, -2.5982e-03,  2.1930e-02],\n",
      "          [ 1.7300e-02, -1.6359e-04, -8.0183e-03]],\n",
      "\n",
      "         [[ 1.8747e-02,  2.2504e-02,  2.3983e-02],\n",
      "          [ 7.3613e-03,  2.2661e-03,  2.9385e-02],\n",
      "          [-2.4361e-02, -5.4315e-03, -2.2929e-02]],\n",
      "\n",
      "         [[-1.4019e-02, -6.5315e-03, -1.2604e-02],\n",
      "          [-7.2476e-03,  1.9467e-02,  2.0145e-02],\n",
      "          [ 2.0151e-02, -1.5135e-02, -2.7291e-02]]],\n",
      "\n",
      "\n",
      "        [[[ 2.3406e-02,  6.5791e-04,  2.2839e-02],\n",
      "          [-1.9956e-02,  8.8866e-04, -1.3189e-02],\n",
      "          [-2.2412e-02, -1.2897e-02, -2.7416e-02]],\n",
      "\n",
      "         [[ 1.9030e-02, -1.0364e-02,  1.7906e-02],\n",
      "          [ 2.7823e-02,  1.0177e-04, -1.6489e-02],\n",
      "          [ 1.8798e-02, -3.0204e-03, -1.4690e-02]],\n",
      "\n",
      "         [[ 2.6200e-02, -1.4737e-03,  1.0937e-02],\n",
      "          [ 2.3301e-02,  5.1256e-03,  1.2886e-02],\n",
      "          [ 9.4189e-03,  1.4048e-02,  1.1892e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-2.7645e-03, -1.4852e-02,  2.2347e-02],\n",
      "          [-1.2486e-02,  2.5150e-02, -1.4150e-02],\n",
      "          [-1.4939e-02,  1.1450e-02,  1.8236e-02]],\n",
      "\n",
      "         [[-1.6521e-02,  5.2022e-03, -6.8156e-03],\n",
      "          [-1.4783e-03,  1.6134e-02, -2.1928e-02],\n",
      "          [ 5.6379e-03,  2.5216e-02, -1.4456e-02]],\n",
      "\n",
      "         [[ 2.5290e-02, -1.2401e-02, -7.2619e-03],\n",
      "          [-6.9488e-04,  1.2878e-02,  8.1937e-03],\n",
      "          [-1.8544e-03,  2.2709e-02, -1.6080e-02]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[-6.1740e-03,  1.3373e-02, -2.7857e-03],\n",
      "          [-1.1771e-02, -2.7109e-02,  1.1437e-02],\n",
      "          [ 1.5806e-03,  2.5413e-02, -3.0440e-03]],\n",
      "\n",
      "         [[-1.1266e-02,  6.2645e-03,  7.3403e-03],\n",
      "          [-9.3400e-03,  1.6196e-03,  6.6259e-03],\n",
      "          [ 2.6409e-02,  2.6343e-02, -5.1137e-03]],\n",
      "\n",
      "         [[ 1.1347e-02,  2.6543e-02,  5.5506e-03],\n",
      "          [-8.5848e-03,  1.5036e-02, -6.4364e-03],\n",
      "          [-9.7511e-03,  1.7051e-02, -3.3566e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 2.6394e-02,  2.0905e-02,  1.6753e-02],\n",
      "          [ 1.8089e-02, -2.3345e-02,  1.9725e-03],\n",
      "          [ 2.2693e-02,  2.8598e-02, -1.5842e-02]],\n",
      "\n",
      "         [[ 1.1262e-02, -1.6568e-02, -1.1285e-02],\n",
      "          [-1.7783e-04,  2.2589e-02, -2.3287e-02],\n",
      "          [-2.6138e-03,  2.3738e-02,  2.3361e-02]],\n",
      "\n",
      "         [[ 2.2048e-02, -5.8089e-03,  5.7787e-03],\n",
      "          [ 1.0737e-02,  1.1071e-02,  7.7790e-03],\n",
      "          [ 1.8919e-02,  1.4284e-02, -8.2951e-03]]],\n",
      "\n",
      "\n",
      "        [[[-2.0309e-02,  1.9665e-02, -7.4998e-03],\n",
      "          [ 7.8856e-03,  1.5912e-02, -2.2124e-02],\n",
      "          [-1.9643e-02,  2.8096e-02, -5.0275e-03]],\n",
      "\n",
      "         [[-2.7551e-02, -1.0791e-02, -4.2615e-03],\n",
      "          [ 7.2613e-03, -2.2874e-02,  2.6838e-02],\n",
      "          [ 1.2877e-02,  2.6233e-02,  1.8649e-02]],\n",
      "\n",
      "         [[ 1.5151e-02, -7.0108e-03, -2.8775e-02],\n",
      "          [-2.2225e-02, -2.6113e-02, -5.7556e-04],\n",
      "          [-4.3722e-03,  2.6127e-02, -1.4761e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-2.6256e-02,  5.6628e-03,  1.7472e-02],\n",
      "          [ 1.9540e-02,  2.7731e-02,  3.5690e-03],\n",
      "          [-2.7611e-02,  2.4415e-02, -1.2013e-02]],\n",
      "\n",
      "         [[ 1.5450e-02, -2.3950e-03, -1.4397e-02],\n",
      "          [ 1.1630e-02,  1.9252e-02, -7.3163e-03],\n",
      "          [ 7.5194e-03, -2.9184e-02, -3.1389e-03]],\n",
      "\n",
      "         [[ 8.8862e-03, -8.2895e-03, -2.5959e-02],\n",
      "          [ 1.1965e-02,  2.8727e-02,  1.3561e-02],\n",
      "          [-1.3019e-02, -1.1236e-02,  1.4957e-02]]],\n",
      "\n",
      "\n",
      "        [[[ 2.3088e-02,  1.3352e-02,  1.1709e-03],\n",
      "          [ 1.5390e-02, -2.2304e-02,  9.0211e-03],\n",
      "          [ 2.3101e-02, -2.6902e-02,  1.6346e-02]],\n",
      "\n",
      "         [[-1.1381e-02,  2.9137e-02,  4.6891e-03],\n",
      "          [ 5.0600e-03, -1.7931e-02, -5.2944e-03],\n",
      "          [-1.0544e-02,  2.5371e-03,  1.0384e-02]],\n",
      "\n",
      "         [[-1.0358e-02, -6.2431e-03,  2.0396e-02],\n",
      "          [ 1.9686e-02,  2.2074e-02, -2.3741e-02],\n",
      "          [ 2.6640e-03,  1.3975e-03, -2.0229e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-3.2029e-03,  5.1478e-03,  1.7186e-02],\n",
      "          [ 1.9545e-02,  2.1229e-02, -1.9699e-02],\n",
      "          [-2.7591e-02, -1.6191e-02,  2.7165e-02]],\n",
      "\n",
      "         [[ 2.1815e-02, -2.7934e-03, -2.9126e-02],\n",
      "          [ 1.1293e-02, -2.3369e-02, -2.6974e-02],\n",
      "          [ 2.2963e-02,  2.8478e-02,  1.8566e-02]],\n",
      "\n",
      "         [[ 1.1659e-02, -9.5647e-03,  1.9539e-02],\n",
      "          [ 8.0136e-03, -1.1808e-02, -2.4234e-03],\n",
      "          [-1.9925e-02,  7.5103e-03,  2.8792e-02]]]], device='cuda:0')\n",
      "3.1.weight: tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1.], device='cuda:0')\n",
      "3.1.bias: tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "       device='cuda:0')\n",
      "5.module.0.0.weight: tensor([[[[ 1.9423e-02,  8.5540e-03,  1.6592e-02],\n",
      "          [-1.4227e-02, -1.8311e-02,  1.2213e-02],\n",
      "          [-1.6905e-02, -1.8124e-02, -1.1255e-02]],\n",
      "\n",
      "         [[ 1.4199e-04,  5.3166e-03, -7.1849e-03],\n",
      "          [-1.2377e-02,  1.5512e-02,  2.0177e-02],\n",
      "          [ 9.8309e-03,  1.7438e-02, -1.9479e-02]],\n",
      "\n",
      "         [[-1.9227e-02, -7.2391e-03, -1.9130e-02],\n",
      "          [-3.9227e-03,  1.0475e-02,  1.3150e-02],\n",
      "          [-1.4750e-02,  2.0212e-02,  6.7700e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 6.7298e-03, -1.7752e-02,  2.0785e-02],\n",
      "          [ 7.1537e-03, -1.2541e-02, -2.0062e-02],\n",
      "          [-5.4040e-03,  8.3993e-06, -5.6242e-03]],\n",
      "\n",
      "         [[ 1.4614e-02, -1.6358e-02,  1.5913e-02],\n",
      "          [ 4.3139e-03, -1.6662e-02,  1.3337e-02],\n",
      "          [ 1.0661e-02,  1.0805e-02, -1.7856e-02]],\n",
      "\n",
      "         [[ 5.5028e-03,  2.6694e-03,  1.6618e-02],\n",
      "          [-6.2963e-03,  1.5961e-02, -2.0730e-02],\n",
      "          [ 1.4757e-02,  1.6756e-02, -2.3674e-04]]],\n",
      "\n",
      "\n",
      "        [[[ 2.0802e-02,  8.0940e-03, -1.1286e-02],\n",
      "          [-2.4036e-03,  4.7028e-04, -2.9262e-03],\n",
      "          [-1.8091e-02,  1.0111e-02, -7.2650e-04]],\n",
      "\n",
      "         [[ 9.5365e-03,  2.6410e-03,  3.5255e-03],\n",
      "          [ 2.0499e-02,  1.3412e-02,  1.4069e-02],\n",
      "          [ 4.2628e-04, -1.1966e-02, -8.2174e-03]],\n",
      "\n",
      "         [[-6.4785e-03, -3.0191e-03, -6.0051e-03],\n",
      "          [-3.2159e-03,  2.0254e-02, -6.0967e-03],\n",
      "          [-1.9079e-02,  4.1194e-03, -1.5673e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 1.8753e-02,  1.6590e-02, -4.2898e-04],\n",
      "          [-1.0537e-02,  8.5555e-03, -1.6023e-02],\n",
      "          [-1.1610e-02, -1.6804e-02, -1.0931e-02]],\n",
      "\n",
      "         [[-6.8706e-03,  1.6419e-02, -1.2098e-02],\n",
      "          [ 1.8449e-02, -9.6713e-03, -1.9403e-02],\n",
      "          [-3.7879e-03,  7.1073e-03, -1.5460e-02]],\n",
      "\n",
      "         [[-1.7104e-03,  1.8117e-02,  1.7837e-02],\n",
      "          [-1.7114e-02,  6.1288e-03, -1.6336e-03],\n",
      "          [-5.8872e-03,  1.7433e-02, -8.4452e-03]]],\n",
      "\n",
      "\n",
      "        [[[-8.5944e-03, -4.3927e-03,  1.0990e-02],\n",
      "          [-1.1069e-02, -1.2500e-02, -2.2408e-03],\n",
      "          [ 2.0467e-03, -2.1188e-03,  2.8384e-03]],\n",
      "\n",
      "         [[-2.0659e-03, -9.7358e-03, -1.0337e-02],\n",
      "          [ 1.4823e-02, -1.6491e-02, -3.5070e-03],\n",
      "          [-4.5096e-03, -1.0791e-02, -1.4849e-02]],\n",
      "\n",
      "         [[-1.3101e-02, -1.9136e-02, -7.0245e-03],\n",
      "          [ 1.5730e-02, -3.6019e-03, -5.3600e-03],\n",
      "          [ 6.4919e-03, -6.9580e-03,  1.2150e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-4.5086e-03, -1.5525e-02, -3.3948e-03],\n",
      "          [ 1.3387e-02, -1.5804e-02, -1.5796e-02],\n",
      "          [ 1.4731e-02, -2.0420e-02, -2.0645e-03]],\n",
      "\n",
      "         [[-1.3564e-02,  7.0940e-03,  1.0729e-03],\n",
      "          [ 6.2039e-03, -1.5052e-02,  9.0869e-03],\n",
      "          [-9.6057e-03,  1.0108e-02, -8.9391e-03]],\n",
      "\n",
      "         [[ 9.5870e-03,  1.8860e-02, -5.2607e-03],\n",
      "          [-1.7979e-02, -1.3030e-03,  8.9161e-03],\n",
      "          [ 1.5661e-02, -1.3070e-02,  4.3186e-03]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[-7.0372e-03,  1.9383e-02, -2.0298e-02],\n",
      "          [ 1.0341e-02, -6.7435e-03, -1.6563e-02],\n",
      "          [-1.6527e-02,  1.9879e-02,  8.0772e-03]],\n",
      "\n",
      "         [[ 1.5914e-02,  1.9364e-02,  1.4943e-02],\n",
      "          [ 1.5685e-02,  1.5467e-02,  1.9403e-02],\n",
      "          [ 1.1617e-02,  4.7042e-03, -9.0118e-03]],\n",
      "\n",
      "         [[-1.7041e-02,  1.3022e-02,  1.1991e-02],\n",
      "          [ 1.1062e-03, -1.1021e-03, -9.5097e-03],\n",
      "          [ 4.7635e-03,  1.6500e-02, -9.5574e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-4.9952e-03,  1.8209e-02,  1.9983e-02],\n",
      "          [-4.2509e-03,  1.3230e-02, -1.2163e-02],\n",
      "          [ 1.7518e-02, -1.9156e-02, -6.4768e-03]],\n",
      "\n",
      "         [[-6.7144e-04,  3.4281e-03, -5.7246e-03],\n",
      "          [ 1.9987e-02, -7.6123e-05, -1.4921e-02],\n",
      "          [-1.7340e-02,  1.2524e-02, -4.5731e-03]],\n",
      "\n",
      "         [[-1.6107e-02, -1.4895e-02, -9.0840e-03],\n",
      "          [ 1.7950e-02,  1.8148e-02,  1.0657e-02],\n",
      "          [ 1.8309e-02,  1.7344e-02,  7.7631e-03]]],\n",
      "\n",
      "\n",
      "        [[[-1.2230e-02,  1.9124e-02, -1.8638e-02],\n",
      "          [-5.8910e-03,  3.4083e-03, -1.9612e-02],\n",
      "          [ 5.9315e-03,  9.8514e-03, -1.3928e-02]],\n",
      "\n",
      "         [[ 1.7383e-02, -1.1888e-02,  1.2734e-03],\n",
      "          [-8.1189e-03,  1.2277e-02, -1.3293e-02],\n",
      "          [-8.9053e-03, -5.5382e-03,  1.6734e-02]],\n",
      "\n",
      "         [[ 3.6930e-03, -1.0042e-02, -7.4888e-03],\n",
      "          [-1.4570e-02,  2.8232e-03, -3.6147e-03],\n",
      "          [-1.7171e-02, -1.2160e-02,  3.8204e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-1.3627e-02,  5.2071e-03, -1.2369e-02],\n",
      "          [ 1.4588e-02,  9.8464e-04,  1.9703e-02],\n",
      "          [ 1.4407e-02,  4.4671e-03, -1.0236e-02]],\n",
      "\n",
      "         [[ 4.7193e-03,  1.7589e-02,  1.0596e-02],\n",
      "          [-5.6906e-03, -1.2341e-02,  1.4351e-02],\n",
      "          [ 1.5154e-02,  1.8930e-02,  1.9626e-02]],\n",
      "\n",
      "         [[ 1.3738e-02, -1.0672e-02, -7.8613e-03],\n",
      "          [ 9.1872e-03, -1.2989e-03, -5.0893e-03],\n",
      "          [ 1.4474e-02,  6.5577e-03,  1.8606e-02]]],\n",
      "\n",
      "\n",
      "        [[[ 8.4860e-03, -3.3258e-03, -2.0280e-02],\n",
      "          [-8.2073e-03, -9.5739e-03, -2.0368e-02],\n",
      "          [-1.2244e-02,  1.1803e-02,  1.1456e-02]],\n",
      "\n",
      "         [[ 1.5826e-03, -1.9392e-02,  7.1250e-03],\n",
      "          [-9.9939e-03, -2.4656e-03,  1.1767e-02],\n",
      "          [ 1.3257e-02,  6.2396e-03, -1.0644e-02]],\n",
      "\n",
      "         [[-1.1166e-02,  1.3383e-02,  1.4393e-02],\n",
      "          [-1.6927e-02,  1.4394e-02,  3.2752e-03],\n",
      "          [ 6.7643e-03, -1.2409e-02, -1.7018e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-1.5627e-02, -1.5727e-02,  7.5061e-03],\n",
      "          [ 4.6503e-03,  1.2383e-02, -1.4976e-02],\n",
      "          [-1.9732e-02,  5.5128e-03,  4.1398e-05]],\n",
      "\n",
      "         [[-1.7281e-02,  1.8933e-03,  1.7812e-02],\n",
      "          [ 1.2012e-02, -2.0248e-02,  2.1978e-03],\n",
      "          [ 4.5396e-03,  1.2390e-02, -6.6603e-03]],\n",
      "\n",
      "         [[-1.3302e-02,  1.6077e-03, -1.6158e-02],\n",
      "          [ 1.3724e-02, -1.8950e-02, -3.2397e-03],\n",
      "          [-1.3008e-03, -7.2950e-03,  2.0301e-02]]]], device='cuda:0')\n",
      "5.module.0.1.weight: tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1.], device='cuda:0')\n",
      "5.module.0.1.bias: tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "       device='cuda:0')\n",
      "6.0.weight: tensor([[[[ 9.7884e-03,  2.5159e-04, -1.6614e-02],\n",
      "          [ 1.0723e-02, -2.8439e-03,  4.0819e-03],\n",
      "          [ 7.3135e-03,  3.2466e-03,  6.2114e-03]],\n",
      "\n",
      "         [[ 1.7807e-02, -1.8764e-02,  8.3971e-03],\n",
      "          [-1.4569e-02, -2.5896e-03,  1.7085e-02],\n",
      "          [-5.0443e-03, -2.0261e-02,  1.6651e-02]],\n",
      "\n",
      "         [[-6.2534e-03, -8.7822e-04,  7.4419e-03],\n",
      "          [ 1.5512e-02, -2.2172e-03, -1.9071e-02],\n",
      "          [ 1.9696e-02, -1.3937e-02, -2.0019e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 1.5032e-02, -1.9163e-02,  8.4393e-03],\n",
      "          [ 6.7364e-03, -1.1247e-02,  1.7373e-02],\n",
      "          [ 1.2201e-02, -1.4524e-02,  1.7252e-02]],\n",
      "\n",
      "         [[-1.6180e-02, -5.1760e-03,  1.0391e-02],\n",
      "          [ 3.1997e-03,  5.0202e-03,  1.5711e-03],\n",
      "          [-8.5274e-03,  5.4054e-03, -4.3110e-04]],\n",
      "\n",
      "         [[-9.9470e-03, -1.4382e-02, -1.7040e-02],\n",
      "          [ 1.7293e-02,  1.9548e-02, -1.7441e-02],\n",
      "          [-1.1806e-02,  1.1101e-02,  1.2445e-02]]],\n",
      "\n",
      "\n",
      "        [[[ 1.0040e-02,  2.6080e-03, -2.0421e-03],\n",
      "          [ 6.3551e-03, -1.3132e-02,  4.3201e-03],\n",
      "          [-8.6904e-03, -7.0975e-03,  1.0538e-02]],\n",
      "\n",
      "         [[ 5.6148e-03, -1.6551e-02,  5.5455e-03],\n",
      "          [-1.3417e-02, -1.9898e-02,  1.2915e-02],\n",
      "          [ 1.5920e-02, -2.8748e-03,  8.8845e-03]],\n",
      "\n",
      "         [[ 8.0066e-03, -1.9370e-02,  1.0746e-02],\n",
      "          [ 2.4538e-03, -1.7295e-02,  1.7075e-02],\n",
      "          [ 2.0621e-02,  1.4943e-02, -1.5867e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-1.9933e-02, -1.2612e-02, -1.6848e-03],\n",
      "          [-6.4529e-03, -1.5544e-02, -1.0668e-02],\n",
      "          [ 8.9834e-03,  1.2962e-02,  1.8056e-02]],\n",
      "\n",
      "         [[ 1.2351e-02,  8.3506e-03, -6.7224e-04],\n",
      "          [-9.0866e-03,  4.3532e-03,  2.3493e-03],\n",
      "          [ 4.3908e-03, -1.8066e-02,  2.3157e-03]],\n",
      "\n",
      "         [[-6.8582e-03,  1.0950e-02, -3.5628e-03],\n",
      "          [ 1.0773e-02, -2.5440e-03,  1.7624e-02],\n",
      "          [-2.0219e-02, -8.8786e-03, -1.5969e-02]]],\n",
      "\n",
      "\n",
      "        [[[ 1.3618e-02,  2.2981e-03, -1.1776e-02],\n",
      "          [ 2.0185e-02,  6.7004e-03, -1.7063e-02],\n",
      "          [-5.0696e-03, -1.6710e-03, -8.6590e-03]],\n",
      "\n",
      "         [[ 1.2203e-02,  1.1511e-02,  1.0480e-02],\n",
      "          [ 1.3867e-03, -3.0763e-03, -1.0616e-02],\n",
      "          [ 1.4231e-02,  1.3651e-02,  9.0518e-03]],\n",
      "\n",
      "         [[-4.4667e-04,  8.6301e-03, -3.3350e-03],\n",
      "          [-1.5312e-02, -2.8406e-03, -1.5360e-02],\n",
      "          [-9.5350e-04,  1.1295e-02,  8.4511e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 1.1845e-02,  1.4802e-02,  2.0727e-02],\n",
      "          [ 1.5682e-02, -1.8993e-02,  1.5598e-02],\n",
      "          [-5.6501e-03, -2.7531e-03, -7.1239e-03]],\n",
      "\n",
      "         [[ 1.4382e-02, -1.9107e-02,  1.2247e-02],\n",
      "          [-1.5146e-03, -1.1573e-02, -1.0631e-02],\n",
      "          [-8.9520e-03, -1.7086e-02,  8.4015e-03]],\n",
      "\n",
      "         [[ 1.3979e-02, -1.7823e-02, -8.9913e-03],\n",
      "          [ 1.3364e-03,  7.9606e-03, -3.3529e-03],\n",
      "          [-1.3698e-03,  1.1483e-02, -7.9687e-03]]],\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "        [[[ 1.3973e-02, -1.2492e-03,  9.6805e-03],\n",
      "          [-2.9954e-03,  1.6186e-02,  1.1738e-02],\n",
      "          [ 1.8425e-02,  7.5424e-03, -1.7116e-02]],\n",
      "\n",
      "         [[ 2.3474e-05,  1.2254e-02,  5.9180e-03],\n",
      "          [ 2.0649e-02, -2.0489e-02,  4.7561e-03],\n",
      "          [ 1.0265e-02, -7.1008e-03,  9.2214e-03]],\n",
      "\n",
      "         [[ 2.7609e-03,  7.5520e-03,  8.6495e-03],\n",
      "          [ 3.9271e-03, -1.2134e-02, -2.0795e-02],\n",
      "          [ 6.4416e-04, -2.3946e-04,  3.5628e-03]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 7.9545e-03, -1.6322e-02, -1.1550e-02],\n",
      "          [-1.3551e-02,  1.9242e-02, -1.8530e-02],\n",
      "          [-2.0641e-02, -4.2925e-03,  6.3211e-03]],\n",
      "\n",
      "         [[ 1.3506e-02, -1.0824e-02,  1.9367e-02],\n",
      "          [-1.5706e-02, -1.5700e-02, -4.1949e-03],\n",
      "          [ 1.1083e-02, -7.3434e-03, -7.4855e-03]],\n",
      "\n",
      "         [[-2.0005e-02, -1.4688e-02,  3.8613e-03],\n",
      "          [ 1.9537e-02,  1.6569e-02,  1.7392e-02],\n",
      "          [ 1.7008e-02, -9.2811e-03, -1.3858e-02]]],\n",
      "\n",
      "\n",
      "        [[[ 2.5634e-03, -9.7260e-03, -1.2291e-02],\n",
      "          [-6.4794e-03, -1.9199e-02,  3.4114e-03],\n",
      "          [-4.2565e-03,  1.1513e-03, -1.2985e-02]],\n",
      "\n",
      "         [[-6.5440e-03,  9.2687e-03,  3.5182e-03],\n",
      "          [-9.9313e-04,  1.8663e-02, -9.1014e-03],\n",
      "          [-4.9423e-03,  3.4946e-03, -6.6136e-03]],\n",
      "\n",
      "         [[ 7.2742e-03,  1.6614e-03, -7.6773e-03],\n",
      "          [-1.8181e-02,  8.7284e-03, -1.9467e-02],\n",
      "          [-1.5322e-02,  1.5744e-02,  1.2211e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[-1.7294e-02, -8.1436e-03, -6.4583e-03],\n",
      "          [-2.9656e-03,  1.1282e-02,  1.5347e-02],\n",
      "          [-1.8420e-02, -1.3404e-03, -2.0242e-02]],\n",
      "\n",
      "         [[-1.4215e-03, -1.3744e-02,  1.2369e-02],\n",
      "          [ 6.1523e-03, -3.0749e-03,  8.6975e-03],\n",
      "          [-1.8163e-02, -2.0243e-02, -1.2251e-02]],\n",
      "\n",
      "         [[ 2.2796e-03,  8.0563e-04,  1.1500e-02],\n",
      "          [-9.6723e-03,  4.5736e-03,  1.6446e-02],\n",
      "          [ 6.7479e-03,  6.0662e-03,  3.5772e-03]]],\n",
      "\n",
      "\n",
      "        [[[ 8.3362e-03,  6.3776e-03,  1.6245e-03],\n",
      "          [-5.9079e-03,  1.3857e-02,  3.2961e-04],\n",
      "          [-2.6438e-03,  4.8083e-03, -1.1564e-02]],\n",
      "\n",
      "         [[-7.6665e-03,  1.1964e-02, -1.0612e-02],\n",
      "          [ 1.7230e-02,  1.2315e-02, -1.8976e-02],\n",
      "          [-2.1372e-03, -1.0696e-02, -3.7575e-03]],\n",
      "\n",
      "         [[ 1.8711e-02, -1.6360e-02, -2.7067e-03],\n",
      "          [ 1.5787e-02,  1.2450e-02,  1.3403e-02],\n",
      "          [-6.8180e-03, -8.0877e-03, -1.1953e-02]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 5.6354e-03,  5.8496e-03, -9.2387e-04],\n",
      "          [ 6.3903e-03,  2.0156e-02, -3.7782e-03],\n",
      "          [ 1.2903e-02, -9.9684e-04,  1.8400e-02]],\n",
      "\n",
      "         [[-7.4491e-03, -2.5903e-03, -1.7878e-02],\n",
      "          [ 1.6782e-02, -1.3165e-02, -1.4989e-02],\n",
      "          [ 4.6110e-03,  6.6622e-03, -2.7627e-04]],\n",
      "\n",
      "         [[ 3.3136e-03,  5.9994e-04,  1.5020e-02],\n",
      "          [-3.2841e-03,  2.0234e-02, -5.2880e-03],\n",
      "          [-1.7000e-02,  1.3800e-02,  1.5010e-03]]]], device='cuda:0')\n",
      "6.1.weight: tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1.], device='cuda:0')\n",
      "6.1.bias: tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
      "9.weight: tensor([[ 0.0094,  0.0420,  0.0054,  0.0611,  0.0655,  0.0619, -0.0652,  0.0461,\n",
      "         -0.0817,  0.0694,  0.0543,  0.0728, -0.0856,  0.0417,  0.0138,  0.0256,\n",
      "          0.0657,  0.0690,  0.0313,  0.0629, -0.0821, -0.0779, -0.0611, -0.0345,\n",
      "         -0.0348,  0.0599, -0.0134, -0.0291,  0.0720, -0.0665,  0.0414, -0.0706,\n",
      "         -0.0450, -0.0554, -0.0777, -0.0264,  0.0034,  0.0074,  0.0632,  0.0094,\n",
      "          0.0754,  0.0248,  0.0311, -0.0512, -0.0045, -0.0202, -0.0544,  0.0429,\n",
      "         -0.0866, -0.0162,  0.0008, -0.0636,  0.0625,  0.0044, -0.0381,  0.0490,\n",
      "         -0.0439, -0.0702, -0.0438,  0.0095, -0.0282,  0.0684, -0.0053,  0.0697,\n",
      "          0.0250,  0.0411, -0.0243, -0.0458,  0.0687,  0.0412, -0.0275,  0.0557,\n",
      "         -0.0590,  0.0562, -0.0319,  0.0113, -0.0577,  0.0102,  0.0483, -0.0470,\n",
      "         -0.0722, -0.0069,  0.0454, -0.0295,  0.0418, -0.0239, -0.0668,  0.0336,\n",
      "          0.0069,  0.0798,  0.0019,  0.0041,  0.0375,  0.0381, -0.0139, -0.0558,\n",
      "         -0.0636,  0.0649,  0.0089, -0.0097, -0.0377, -0.0043, -0.0618, -0.0087,\n",
      "         -0.0493, -0.0077,  0.0250, -0.0247,  0.0362,  0.0171, -0.0804,  0.0316,\n",
      "         -0.0550,  0.0331,  0.0050,  0.0013,  0.0027, -0.0492,  0.0814,  0.0831,\n",
      "          0.0551, -0.0736, -0.0252,  0.0499, -0.0568,  0.0877, -0.0863,  0.0535],\n",
      "        [-0.0461,  0.0148, -0.0584,  0.0812,  0.0852,  0.0871,  0.0361, -0.0781,\n",
      "         -0.0421,  0.0263,  0.0090,  0.0837, -0.0642, -0.0606, -0.0403, -0.0141,\n",
      "          0.0132, -0.0109,  0.0095,  0.0724,  0.0601, -0.0432,  0.0106, -0.0415,\n",
      "         -0.0076,  0.0650,  0.0052, -0.0566,  0.0756,  0.0178, -0.0755, -0.0339,\n",
      "          0.0389, -0.0731, -0.0188, -0.0561, -0.0670,  0.0016,  0.0230, -0.0803,\n",
      "         -0.0767, -0.0430,  0.0405, -0.0359,  0.0826, -0.0601,  0.0197,  0.0454,\n",
      "          0.0842, -0.0818,  0.0273, -0.0434,  0.0284, -0.0535, -0.0240, -0.0325,\n",
      "         -0.0515, -0.0098,  0.0148, -0.0670,  0.0435,  0.0350, -0.0411, -0.0032,\n",
      "         -0.0228,  0.0661, -0.0607, -0.0368,  0.0632, -0.0672, -0.0559, -0.0366,\n",
      "          0.0862,  0.0257, -0.0557, -0.0222,  0.0137,  0.0702,  0.0394,  0.0299,\n",
      "         -0.0721, -0.0564,  0.0063, -0.0398,  0.0432, -0.0532, -0.0068,  0.0070,\n",
      "         -0.0629, -0.0645, -0.0364,  0.0422,  0.0520, -0.0078, -0.0429,  0.0260,\n",
      "         -0.0753,  0.0147, -0.0270,  0.0006, -0.0758, -0.0240, -0.0728, -0.0517,\n",
      "          0.0687,  0.0378, -0.0036,  0.0085,  0.0056,  0.0756, -0.0627,  0.0469,\n",
      "         -0.0305,  0.0736,  0.0251,  0.0239, -0.0008, -0.0221, -0.0695,  0.0632,\n",
      "          0.0132, -0.0134,  0.0805,  0.0567, -0.0274,  0.0408,  0.0787, -0.0382]],\n",
      "       device='cuda:0')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|██████████████████████████████████████████████████                                                                                         | 72/200 [00:31<00:53,  2.39it/s]"
     ]
    },
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mFailed to connect to the remote Jupyter Server 'http://d-7-1-2:8888/'. Verify the server is running and reachable."
     ]
    }
   ],
   "source": [
    "for i in range(2):\n",
    "    run_experiment('seed', i, [0,0,0,0,0], 'best_diff_init_seeds shorter (same init)')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "test",
   "language": "python",
   "name": "test"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}