Model_roberta_base.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/snaiws/NLP_project/blob/RoBERTa%2Fbaseline/Model_roberta_base.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qhsY0hJvziNP",
        "outputId": "018a7223-e397-453c-8a95-7cf125625450"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.19.2-py3-none-any.whl (4.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.2 MB 9.5 MB/s \n",
            "\u001b[?25hCollecting datasets\n",
            "  Downloading datasets-2.2.2-py3-none-any.whl (346 kB)\n",
            "\u001b[K     |████████████████████████████████| 346 kB 72.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.3)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.7.0-py3-none-any.whl (86 kB)\n",
            "\u001b[K     |████████████████████████████████| 86 kB 6.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.7.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 61.6 MB/s \n",
            "\u001b[?25hCollecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 57.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.2.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n",
            "Collecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0.1)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[K     |████████████████████████████████| 212 kB 56.6 MB/s \n",
            "\u001b[?25hCollecting dill<0.3.5\n",
            "  Downloading dill-0.3.4-py2.py3-none-any.whl (86 kB)\n",
            "\u001b[K     |████████████████████████████████| 86 kB 5.9 MB/s \n",
            "\u001b[?25hCollecting fsspec[http]>=2021.05.0\n",
            "  Downloading fsspec-2022.5.0-py3-none-any.whl (140 kB)\n",
            "\u001b[K     |████████████████████████████████| 140 kB 37.2 MB/s \n",
            "\u001b[?25hCollecting aiohttp\n",
            "  Downloading aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 57.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.3.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.5.18.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1\n",
            "  Downloading urllib3-1.25.11-py2.py3-none-any.whl (127 kB)\n",
            "\u001b[K     |████████████████████████████████| 127 kB 58.4 MB/s \n",
            "\u001b[?25hCollecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.3.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (144 kB)\n",
            "\u001b[K     |████████████████████████████████| 144 kB 51.9 MB/s \n",
            "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n",
            "\u001b[K     |████████████████████████████████| 271 kB 62.4 MB/s \n",
            "\u001b[?25hCollecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-6.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (94 kB)\n",
            "\u001b[K     |████████████████████████████████| 94 kB 3.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.0.12)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (21.4.0)\n",
            "Collecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n",
            "Collecting asynctest==0.13.0\n",
            "  Downloading asynctest-0.13.0-py3-none-any.whl (26 kB)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.8.0)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2022.1)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: multidict, frozenlist, yarl, urllib3, asynctest, async-timeout, aiosignal, pyyaml, fsspec, dill, aiohttp, xxhash, tokenizers, responses, huggingface-hub, transformers, datasets\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "  Attempting uninstall: dill\n",
            "    Found existing installation: dill 0.3.5.1\n",
            "    Uninstalling dill-0.3.5.1:\n",
            "      Successfully uninstalled dill-0.3.5.1\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "Successfully installed aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.2 asynctest-0.13.0 datasets-2.2.2 dill-0.3.4 frozenlist-1.3.0 fsspec-2022.5.0 huggingface-hub-0.7.0 multidict-6.0.2 pyyaml-6.0 responses-0.18.0 tokenizers-0.12.1 transformers-4.19.2 urllib3-1.25.11 xxhash-3.0.0 yarl-1.7.2\n"
          ]
        }
      ],
      "source": [
        "# transformer, datset 설치\n",
        "!pip install transformers datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kHaTl0OG-bvy",
        "outputId": "af4e9f9f-d217-4c71-f211-0265cef2d58f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting jupyter-tensorboard\n",
            "  Downloading jupyter_tensorboard-0.2.0.tar.gz (15 kB)\n",
            "Requirement already satisfied: notebook>=5.0 in /usr/local/lib/python3.7/dist-packages (from jupyter-tensorboard) (5.3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from notebook>=5.0->jupyter-tensorboard) (2.11.3)\n",
            "Requirement already satisfied: nbformat in /usr/local/lib/python3.7/dist-packages (from notebook>=5.0->jupyter-tensorboard) (5.4.0)\n",
            "Requirement already satisfied: jupyter-core>=4.4.0 in /usr/local/lib/python3.7/dist-packages (from notebook>=5.0->jupyter-tensorboard) (4.10.0)\n",
            "Requirement already satisfied: nbconvert in /usr/local/lib/python3.7/dist-packages (from notebook>=5.0->jupyter-tensorboard) (5.6.1)\n",
            "Requirement already satisfied: jupyter-client>=5.2.0 in /usr/local/lib/python3.7/dist-packages (from notebook>=5.0->jupyter-tensorboard) (5.3.5)\n",
            "Requirement already satisfied: traitlets>=4.2.1 in /usr/local/lib/python3.7/dist-packages (from notebook>=5.0->jupyter-tensorboard) (5.1.1)\n",
            "Requirement already satisfied: tornado>=4 in /usr/local/lib/python3.7/dist-packages (from notebook>=5.0->jupyter-tensorboard) (5.1.1)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.7/dist-packages (from notebook>=5.0->jupyter-tensorboard) (0.2.0)\n",
            "Requirement already satisfied: Send2Trash in /usr/local/lib/python3.7/dist-packages (from notebook>=5.0->jupyter-tensorboard) (1.8.0)\n",
            "Requirement already satisfied: terminado>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from notebook>=5.0->jupyter-tensorboard) (0.13.3)\n",
            "Requirement already satisfied: ipykernel in /usr/local/lib/python3.7/dist-packages (from notebook>=5.0->jupyter-tensorboard) (4.10.1)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from jupyter-client>=5.2.0->notebook>=5.0->jupyter-tensorboard) (2.8.2)\n",
            "Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.7/dist-packages (from jupyter-client>=5.2.0->notebook>=5.0->jupyter-tensorboard) (23.0.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.1->jupyter-client>=5.2.0->notebook>=5.0->jupyter-tensorboard) (1.15.0)\n",
            "Requirement already satisfied: ptyprocess in /usr/local/lib/python3.7/dist-packages (from terminado>=0.8.1->notebook>=5.0->jupyter-tensorboard) (0.7.0)\n",
            "Requirement already satisfied: ipython>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from ipykernel->notebook>=5.0->jupyter-tensorboard) (5.5.0)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->notebook>=5.0->jupyter-tensorboard) (4.4.2)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->notebook>=5.0->jupyter-tensorboard) (0.8.1)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->notebook>=5.0->jupyter-tensorboard) (57.4.0)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->notebook>=5.0->jupyter-tensorboard) (2.6.1)\n",
            "Requirement already satisfied: pexpect in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->notebook>=5.0->jupyter-tensorboard) (4.8.0)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->notebook>=5.0->jupyter-tensorboard) (0.7.5)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->notebook>=5.0->jupyter-tensorboard) (1.0.18)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython>=4.0.0->ipykernel->notebook>=5.0->jupyter-tensorboard) (0.2.5)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->notebook>=5.0->jupyter-tensorboard) (2.0.1)\n",
            "Requirement already satisfied: entrypoints>=0.2.2 in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=5.0->jupyter-tensorboard) (0.4)\n",
            "Requirement already satisfied: testpath in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=5.0->jupyter-tensorboard) (0.6.0)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=5.0->jupyter-tensorboard) (0.7.1)\n",
            "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=5.0->jupyter-tensorboard) (0.8.4)\n",
            "Requirement already satisfied: bleach in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=5.0->jupyter-tensorboard) (5.0.0)\n",
            "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=5.0->jupyter-tensorboard) (1.5.0)\n",
            "Requirement already satisfied: jsonschema>=2.6 in /usr/local/lib/python3.7/dist-packages (from nbformat->notebook>=5.0->jupyter-tensorboard) (4.3.3)\n",
            "Requirement already satisfied: fastjsonschema in /usr/local/lib/python3.7/dist-packages (from nbformat->notebook>=5.0->jupyter-tensorboard) (2.15.3)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from jsonschema>=2.6->nbformat->notebook>=5.0->jupyter-tensorboard) (4.11.3)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema>=2.6->nbformat->notebook>=5.0->jupyter-tensorboard) (0.18.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from jsonschema>=2.6->nbformat->notebook>=5.0->jupyter-tensorboard) (4.2.0)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema>=2.6->nbformat->notebook>=5.0->jupyter-tensorboard) (21.4.0)\n",
            "Requirement already satisfied: importlib-resources>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema>=2.6->nbformat->notebook>=5.0->jupyter-tensorboard) (5.7.1)\n",
            "Requirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.7/dist-packages (from importlib-resources>=1.4.0->jsonschema>=2.6->nbformat->notebook>=5.0->jupyter-tensorboard) (3.8.0)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.7/dist-packages (from bleach->nbconvert->notebook>=5.0->jupyter-tensorboard) (0.5.1)\n",
            "Building wheels for collected packages: jupyter-tensorboard\n",
            "  Building wheel for jupyter-tensorboard (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for jupyter-tensorboard: filename=jupyter_tensorboard-0.2.0-py2.py3-none-any.whl size=15258 sha256=67d836603526b5be9b94f8e75b8cf62161bdd882365b98078da34be2815ebb8d\n",
            "  Stored in directory: /root/.cache/pip/wheels/59/14/ab/6d0bce449039ebdcbf45c1aff8c19153a68bf3f0492a32620e\n",
            "Successfully built jupyter-tensorboard\n",
            "Installing collected packages: jupyter-tensorboard\n",
            "Successfully installed jupyter-tensorboard-0.2.0\n"
          ]
        }
      ],
      "source": [
        "# tensorboard 패키지 설치\n",
        "!pip install jupyter-tensorboard"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UwDRmaJt-ZtL"
      },
      "outputs": [],
      "source": [
        "# tensorboard extension 설치\n",
        "%load_ext tensorboard"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install optuna"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jUboi_I4SEvx",
        "outputId": "9cfa458d-59a4-407d-b29e-1bd493b91ff9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting optuna\n",
            "  Downloading optuna-2.10.0-py3-none-any.whl (308 kB)\n",
            "\u001b[K     |████████████████████████████████| 308 kB 9.7 MB/s \n",
            "\u001b[?25hCollecting colorlog\n",
            "  Downloading colorlog-6.6.0-py2.py3-none-any.whl (11 kB)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from optuna) (21.3)\n",
            "Requirement already satisfied: scipy!=1.4.0 in /usr/local/lib/python3.7/dist-packages (from optuna) (1.4.1)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from optuna) (6.0)\n",
            "Collecting cliff\n",
            "  Downloading cliff-3.10.1-py3-none-any.whl (81 kB)\n",
            "\u001b[K     |████████████████████████████████| 81 kB 10.2 MB/s \n",
            "\u001b[?25hCollecting alembic\n",
            "  Downloading alembic-1.8.0-py3-none-any.whl (209 kB)\n",
            "\u001b[K     |████████████████████████████████| 209 kB 68.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from optuna) (1.21.6)\n",
            "Collecting cmaes>=0.8.2\n",
            "  Downloading cmaes-0.8.2-py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: sqlalchemy>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from optuna) (1.4.36)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from optuna) (4.64.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->optuna) (3.0.9)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.7/dist-packages (from sqlalchemy>=1.1.0->optuna) (1.1.2)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from sqlalchemy>=1.1.0->optuna) (4.11.3)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from alembic->optuna) (5.7.1)\n",
            "Collecting Mako\n",
            "  Downloading Mako-1.2.0-py3-none-any.whl (78 kB)\n",
            "\u001b[K     |████████████████████████████████| 78 kB 7.6 MB/s \n",
            "\u001b[?25hCollecting autopage>=0.4.0\n",
            "  Downloading autopage-0.5.1-py3-none-any.whl (29 kB)\n",
            "Collecting cmd2>=1.0.0\n",
            "  Downloading cmd2-2.4.1-py3-none-any.whl (146 kB)\n",
            "\u001b[K     |████████████████████████████████| 146 kB 55.3 MB/s \n",
            "\u001b[?25hCollecting stevedore>=2.0.1\n",
            "  Downloading stevedore-3.5.0-py3-none-any.whl (49 kB)\n",
            "\u001b[K     |████████████████████████████████| 49 kB 6.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: PrettyTable>=0.7.2 in /usr/local/lib/python3.7/dist-packages (from cliff->optuna) (3.3.0)\n",
            "Collecting pbr!=2.1.0,>=2.0.0\n",
            "  Downloading pbr-5.9.0-py2.py3-none-any.whl (112 kB)\n",
            "\u001b[K     |████████████████████████████████| 112 kB 61.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: attrs>=16.3.0 in /usr/local/lib/python3.7/dist-packages (from cmd2>=1.0.0->cliff->optuna) (21.4.0)\n",
            "Collecting pyperclip>=1.6\n",
            "  Downloading pyperclip-1.8.2.tar.gz (20 kB)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from cmd2>=1.0.0->cliff->optuna) (4.2.0)\n",
            "Requirement already satisfied: wcwidth>=0.1.7 in /usr/local/lib/python3.7/dist-packages (from cmd2>=1.0.0->cliff->optuna) (0.2.5)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->sqlalchemy>=1.1.0->optuna) (3.8.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.9.2 in /usr/local/lib/python3.7/dist-packages (from Mako->alembic->optuna) (2.0.1)\n",
            "Building wheels for collected packages: pyperclip\n",
            "  Building wheel for pyperclip (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyperclip: filename=pyperclip-1.8.2-py3-none-any.whl size=11137 sha256=60e2bc1f7ce68d6e3c7055efe3b3a4d2508f61e794ab0a93c3eb2b939dbae7bc\n",
            "  Stored in directory: /root/.cache/pip/wheels/9f/18/84/8f69f8b08169c7bae2dde6bd7daf0c19fca8c8e500ee620a28\n",
            "Successfully built pyperclip\n",
            "Installing collected packages: pyperclip, pbr, stevedore, Mako, cmd2, autopage, colorlog, cmaes, cliff, alembic, optuna\n",
            "Successfully installed Mako-1.2.0 alembic-1.8.0 autopage-0.5.1 cliff-3.10.1 cmaes-0.8.2 cmd2-2.4.1 colorlog-6.6.0 optuna-2.10.0 pbr-5.9.0 pyperclip-1.8.2 stevedore-3.5.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XOT2hjP30nYZ",
        "outputId": "0a48e3e2-cbea-4ff1-a536-365206f7eab9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Za-SXc_G0uc3"
      },
      "outputs": [],
      "source": [
        "import transformers\n",
        "from transformers import EarlyStoppingCallback\n",
        "from datasets import load_dataset\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import Dataset, DataLoader,  RandomSampler, SequentialSampler, random_split\n",
        "from torch.nn.utils import clip_grad_norm_\n",
        "\n",
        "from transformers import AutoTokenizer\n",
        "from transformers import AutoModelForSequenceClassification, TrainingArguments, Trainer\n",
        "\n",
        "from datasets import load_metric\n",
        "from torch.utils.tensorboard import SummaryWriter\n",
        "import os\n",
        "import re\n",
        "import pandas as pd\n",
        "from torch.optim import AdamW\n",
        "from transformers import get_linear_schedule_with_warmup\n",
        "import matplotlib.pyplot as plt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FSdHKK8h1ouK"
      },
      "outputs": [],
      "source": [
        "#reset gpu cache\n",
        "import gc\n",
        "gc.collect()\n",
        "torch.cuda.empty_cache()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IJKSbUoR1qEo",
        "outputId": "4c9a78ac-c795-48b0-840d-8c75d1911a39"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "# available GPUs : 1\n",
            "GPU name : Tesla P100-PCIE-16GB\n",
            "cuda\n"
          ]
        }
      ],
      "source": [
        "# seed\n",
        "seed = 42\n",
        "torch.manual_seed(seed)\n",
        "torch.cuda.manual_seed_all(seed)\n",
        "\n",
        "# device type\n",
        "if torch.cuda.is_available():\n",
        "    device = torch.device(\"cuda\")\n",
        "    print(f\"# available GPUs : {torch.cuda.device_count()}\")\n",
        "    print(f\"GPU name : {torch.cuda.get_device_name()}\")\n",
        "else:\n",
        "    device = torch.device(\"cpu\")\n",
        "print(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vT1ph0_Q1tMN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "df3def9b-e2b1-487c-b76e-87d7873d4e16"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loading file https://huggingface.co/klue/roberta-base/resolve/main/vocab.txt from cache at /root/.cache/huggingface/transformers/e8441a174492958462b6b16b6db8f1e7253cd149ca779522cadd812d55091b89.d1b86bed49516351c7bb29b19d7e7be2ab53b931bcb1f9b2aacfb71f2124d25a\n",
            "loading file https://huggingface.co/klue/roberta-base/resolve/main/tokenizer.json from cache at /root/.cache/huggingface/transformers/233a5b2c17873a8477b62dd92a02092a9937759e924a5f22b111becebb8aba5e.44c30ade4958fcfd446e66025e10a5b380cdd0bbe9b3fb7a794f357e7f0f34c2\n",
            "loading file https://huggingface.co/klue/roberta-base/resolve/main/added_tokens.json from cache at None\n",
            "loading file https://huggingface.co/klue/roberta-base/resolve/main/special_tokens_map.json from cache at /root/.cache/huggingface/transformers/9d0c87e44b00acfbfbae931b2e4068eb6311a0c3e71e23e5400bdf57cab4bfbf.70c17d6e4d492c8f24f5bb97ab56c7f272e947112c6faf9dd846da42ba13eb23\n",
            "loading file https://huggingface.co/klue/roberta-base/resolve/main/tokenizer_config.json from cache at /root/.cache/huggingface/transformers/9a9f77abeddd1bbd8de28608e78dd3604287ad91abd4796cd25ad936715b7640.5b0ba083b234382bb4c99ee0c9f4fca4cadaa053dd17c32dabfe0de2f629af1f\n"
          ]
        }
      ],
      "source": [
        "# argument setting\n",
        "train_batch_size = 58\n",
        "eval_batch_size = 64\n",
        "epochs=5\n",
        "patience=2\n",
        "lr=2.335281794951567e-05\n",
        "model_checkpoint = \"klue/roberta-base\"\n",
        "\n",
        "loss_fct = nn.CrossEntropyLoss()\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lghtyv1f72V2",
        "outputId": "1a742615-1349-4c40-aca3-4fdaa2d6f298"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive/AIBootCamp/NLP\n"
          ]
        }
      ],
      "source": [
        "cd /content/drive/MyDrive/AIBootCamp/NLP"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QfCe9d8572QH",
        "outputId": "6f1fe5f4-7f84-4293-80b8-0d0754335185"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "My current directory : /content/drive/MyDrive/AIBootCamp/NLP\n",
            "My data directory : /content/drive/MyDrive/AIBootCamp/NLP/data/klue-sts-v1.1\n"
          ]
        }
      ],
      "source": [
        "_CUR_DIR = os.path.abspath(os.curdir)\n",
        "print(f\"My current directory : {_CUR_DIR}\")\n",
        "_DATA_DIR = os.path.join(_CUR_DIR, \"data/klue-sts-v1.1\")\n",
        "print(f\"My data directory : {_DATA_DIR}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eD7F6DMt8Y9x"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "class Preprocess:\n",
        "    def __init__(self,train,test):\n",
        "        self.train=train\n",
        "        self.test=test\n",
        "\n",
        "    def check_st(self, text):\n",
        "      text = re.sub('a-zA-Z一-龥㐀-䶵豈-龎[-=+,#/\\?:^$.@*\\\"※~&%ㆍ·!』\\\\‘〈〉|\\(\\)\\[\\]\\<\\>`\\'…》《]','', text)\n",
        "      return text\n",
        "    \n",
        "    def robeta_base(self,dataset,mode=None):\n",
        "      '''\n",
        "      transform to the Input example\n",
        "      '''\n",
        "      #중복 제거\n",
        "      dataset = dataset.drop_duplicates(['sentence1','sentence2']).reset_index(drop=True)\n",
        "\n",
        "      sent_pairs = [sent1+' '+sent2 for sent1, sent2 in zip(dataset['sentence1'], dataset['sentence2'])]\n",
        "      sent_pairs = list(map(self.check_st, sent_pairs))\n",
        "\n",
        "      #real-label -> regression 활용 가능?\n",
        "      r_score=[i['real-label'] for i in dataset['labels']]\n",
        "\n",
        "      #custom label : int\n",
        "      c_score=[round(i['real-label']) for i in dataset['labels']]\n",
        "\n",
        "      #b_score label : [0,1] -> f1 scroe\n",
        "      b_score = [i['binary-label'] for i in dataset['labels']]\n",
        "      input_examples = pd.DataFrame({'guid': dataset['guid'], 'sent_pair' : sent_pairs, 'r_label' : r_score, 'c_label' : c_score, 'b_label' : b_score})\n",
        "\n",
        "      if mode == 'train':\n",
        "        train, valid = train_test_split(input_examples, test_size=0.1, random_state=seed)\n",
        "        return train, valid\n",
        "\n",
        "      return input_examples"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AxymFJDi9UJx"
      },
      "outputs": [],
      "source": [
        "# Custom Dataset\n",
        "class CustomDataset(Dataset):\n",
        "    \"\"\"\n",
        "    - input_data: list of string\n",
        "    - target_data: list of int\n",
        "    \"\"\"\n",
        "    \n",
        "    def __init__(self, input_data:list, target_data:list) -> None:\n",
        "        self.X = input_data\n",
        "        self.Y = target_data\n",
        "    \n",
        "    def __len__(self):\n",
        "        return len(self.X)\n",
        "    \n",
        "    def __getitem__(self, index):\n",
        "        return self.X[index], self.Y[index]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187,
          "referenced_widgets": [
            "b22751c7d5d942daa1baddc8af50a415",
            "21fbf305220b45df961157be2f29e946",
            "ff913db97f864449bc132df0c2b6179a",
            "d34a08431bee454c84ed07254612f7a0",
            "54371fe1b88845129e8ff18c5217797c",
            "16ae0c5c36184eb79d5b1b7d568e9a27",
            "45b834a8731445d2bf7e9175a9c6fc9a",
            "b2b27bb69e6641dd848736d5f7eb1af1",
            "9b55984eec5d401a8b21c80781459edf",
            "9de5f0ad569644c0a959f2863f988b79",
            "c8a4a9cb652f47c196e71e888483e469",
            "2f9f3e3fd9d94d34b2a53803991a55a9",
            "c6a1a214ad774544a0b3b73d9d6e83e3",
            "5cd6c658e0a14a8bb7cfd2922ec24b9c",
            "2f43bcf8eac94cb09e5dce4e3bd237af",
            "b35a973f34a44f12ae340309460dbce2",
            "4ff0ad2e30a448d7b3401ecbe57bc29a",
            "329334770ea243eaaab4be8cba3c4354",
            "4fed1bd1fcc04c3ba4c7875d5636e7fc",
            "c7d10acd90134798998a69d230263755",
            "e43e9351853f4d61a4487ae26b60f9a2",
            "04fa42a9d0a94ad7ae67cf17672d35b9",
            "851d223309af4c9e85e85a8988ef6387",
            "8ba2194fcebc41beb88f9096956379b7",
            "42f6f09b423445baab9855ae9baff887",
            "0e2708b3dc6c4eb287061a6cf1cde2a7",
            "38c3552b98d749798801af39ea52e1e0",
            "cff0787095694134bc91d9d55ce6899c",
            "3f8c6d0ff201492d9b5e76046f58b899",
            "ba98b1f992ce4c9aa16ec15f9a4a49a9",
            "865bf172aff24c2692183f6131d22b0c",
            "48c4d2786e6540a9af96768443d72021",
            "d7b5f453212440d79ec512f5b105d478",
            "f38df2abd62143e884a23b2c4397608a",
            "c6eb51c3c83e4de8b080b76bd5548ebe",
            "3f7f71a646c64a48991a0ffe3c2486c1",
            "dfd45244a5fe48f29d01a6f6f5d239ba",
            "1e4667fe913846999d65832709965966",
            "cdbc2596130c43eabdab2ac2560202d6",
            "528df8fbfcc343db9448df56f254c8b1",
            "9734c02d56af48db954229207064790f",
            "7bcd2912ca7744bb9a6ae49058dae326",
            "b58cee264d944914ac4b1b8f009bc2db",
            "da7abc45193242068c8987ab07ab6109",
            "e0d669251a114be5a79d8c4e5f3d1bfa",
            "47a08a359b0c40e3b66834625384dc20",
            "4c5f7287296e4c8791b9c94f1bba4dd1",
            "9906da54555a4741b7c91147571f462c",
            "63e1578bef1b46f999484690d8238042",
            "da526d19e34b495dad94e38ae9a20317",
            "a22bdd913bc8415d953bd45f4b3da9b7",
            "844730d076194decb6acc0237dde7fc8",
            "b691120249e44fa98e13271ac596a467",
            "9a4971f6714c4b14ace2fdc896bd481e",
            "91a848af4b894bc89e598a70e0180dd2"
          ]
        },
        "id": "bkN1xkZ1K-7b",
        "outputId": "27cc489d-a9c7-454a-ab47-44e99bfaf0e1"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/5.21k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b22751c7d5d942daa1baddc8af50a415"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading metadata:   0%|          | 0.00/2.93k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2f9f3e3fd9d94d34b2a53803991a55a9"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset klue/sts (download: 1.29 MiB, generated: 2.82 MiB, post-processed: Unknown size, total: 4.11 MiB) to /root/.cache/huggingface/datasets/klue/sts/1.0.0/e0fc3bc3de3eb03be2c92d72fd04a60ecc71903f821619cb28ca0e1e29e4233e...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/1.35M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "851d223309af4c9e85e85a8988ef6387"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/11668 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f38df2abd62143e884a23b2c4397608a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating validation split:   0%|          | 0/519 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e0d669251a114be5a79d8c4e5f3d1bfa"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset klue downloaded and prepared to /root/.cache/huggingface/datasets/klue/sts/1.0.0/e0fc3bc3de3eb03be2c92d72fd04a60ecc71903f821619cb28ca0e1e29e4233e. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Reusing dataset klue (/root/.cache/huggingface/datasets/klue/sts/1.0.0/e0fc3bc3de3eb03be2c92d72fd04a60ecc71903f821619cb28ca0e1e29e4233e)\n",
            "Reusing dataset klue (/root/.cache/huggingface/datasets/klue/sts/1.0.0/e0fc3bc3de3eb03be2c92d72fd04a60ecc71903f821619cb28ca0e1e29e4233e)\n"
          ]
        }
      ],
      "source": [
        "# dataset = load_dataset('klue', task)\n",
        "klue_sts_train = load_dataset(\"klue\", \"sts\", split='train[:90%]')\n",
        "klue_sts_valid = load_dataset(\"klue\", \"sts\", split='train[-10%:]') # train의 10%를 validation set으로 사용\n",
        "klue_sts_test = load_dataset(\"klue\", \"sts\", split='validation')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JTmFFg_ALGo-"
      },
      "outputs": [],
      "source": [
        "# real-label을 label로 변경 \n",
        "klue_sts_train = klue_sts_train.flatten()\n",
        "klue_sts_train = klue_sts_train.rename_column('labels.real-label','label')\n",
        "klue_sts_valid = klue_sts_valid.flatten()\n",
        "klue_sts_valid = klue_sts_valid.rename_column('labels.real-label','label')\n",
        "klue_sts_test = klue_sts_test.flatten()\n",
        "klue_sts_test = klue_sts_test.rename_column('labels.real-label','label')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jSOGK8SU9dcR"
      },
      "outputs": [],
      "source": [
        "# 최대 길이 설정 \n",
        "max_length = 512\n",
        "\n",
        "# 모델 입력으로 만들어 주는 전처리 함수 \n",
        "# truncation으로 길이 넘어가면 자를 수 있게 설정, padding 으로 길이 맞춰 주기 \n",
        "def preprocess_function(examples):\n",
        "    return tokenizer(examples['sentence1'], examples['sentence2'],\n",
        "                     truncation=True, max_length=max_length, padding=True)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "9ed00057998d4cd1846e86e1f6467abb",
            "dcebd258911448639e8a50457e2d7d1c",
            "b0ad2af7970a494cabddfaf08f06ed02",
            "41803b8153df46a798c0e5a518c34d6b",
            "c10e1a6e0f36455986609688df7bbe06",
            "e801e09faa1b42a2b4137889925d9335",
            "874c2fc7a826442fabb5dac3b07a0a75",
            "1b87ec51e122403c8940af5b71b8c5de",
            "37cd84313e9643efa907e69d62d65c91",
            "038bf42c4ae64b1b9e3d2c05c22f4894",
            "c16d8a5b79bc4e33a77ed13bab6b5020",
            "5c653abc118242ba925ed53ab6f07325",
            "5b46e7b42b6e4a92b8a703bd674ed944",
            "94b6c95ace144b20931a1f0d4888a65e",
            "a71a4c274a5f4453818abb24b429a5a7",
            "73554ceb4be34ef788898c7eebdec8dc",
            "56480527af3644dba1c090de3dbe3aef",
            "12bbd177fe164fa0a3681f5dfcb1540c",
            "668bf290194b4e82b7a792f5afc4302a",
            "f647bee7447c4601ab5c5ae93cfd7e0d",
            "9d900548ddb6411b9a4d1ad5188b63f8",
            "ce6417c26e204af6ae4a3539108a5ceb",
            "55f104a2dceb419dbcadd4e46515c2a2",
            "611f40b8e5f240f880106bf5af08d092",
            "ae1b15a0b2b04d828e00a8778527e13e",
            "c8a38212dedc4afaa22699c6d6f295d9",
            "09c06550e6e2494db2a8c6b04bd6c3d6",
            "523ba8c6193743abb8499655ec4a74f1",
            "3067407fb5194e3aa4de556d82f2586d",
            "c6f51004e26e4f9bb6b429140d523234",
            "ed288df2424642ef98bd3139bfe0e14b",
            "213cd178f01a439689bf513487778bcf",
            "df2b8316d57349abb97246e7b851d7ef"
          ]
        },
        "id": "eU4gJ35nAEMV",
        "outputId": "32598165-1f39-41ee-cb37-676c73b8bfb4"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/11 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9ed00057998d4cd1846e86e1f6467abb"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/2 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5c653abc118242ba925ed53ab6f07325"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "55f104a2dceb419dbcadd4e46515c2a2"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "# 전처리 함수 각 데이터셋에 적용 \n",
        "encoded_train = klue_sts_train.map(preprocess_function, batched=True)\n",
        "encoded_valid = klue_sts_valid.map(preprocess_function, batched=True)\n",
        "encoded_test = klue_sts_test.map(preprocess_function, batched=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 194,
          "referenced_widgets": [
            "43b4d7715b87477d806fe3193988e0ea",
            "02dbd9d85d6942f295a736144c0b306f",
            "f1297d192c234af5805b9eabb049919f",
            "d9a65770255547e99601a9a9f6995a39",
            "8cfed01f4c2549d0a599102951634551",
            "33ebb42888004ee0b37920608e763f0a",
            "be156d2353c847e5a49e7994bb9936fe",
            "c2518b3dc2314bac805c3c5d41923306",
            "3282a93542964a69904f02341a7dfe1e",
            "e0af530b11d44a85be40a5dab3615435",
            "3c8131c1104845b887c40d0009a11b7b",
            "e977b3566d18490680504fd253bc5b9c",
            "3293fb599a70432bb9d78ec3098ec40e",
            "065901872f554a6aa1dfc0c0ff1d07f4",
            "1e24fdd83fc74de1977bc23204e7a24b",
            "89549163a8464a05b2e272d417261bef",
            "9faebf2a37f64a3390b671f59a6dbcae",
            "0d5fd240ef044aad99cadb281eaf5a8c",
            "24ec8e0fdad144cf997decdd06195a3a",
            "503df178368d4ed994da85e53a4b3785",
            "35688c8cedaa4080a8ffc84bf8c4eee4",
            "46b2155e79a844caafce448e4690f0d2"
          ]
        },
        "id": "ujOtZxfW91c1",
        "outputId": "ed71d266-5e6d-43a7-ad47-39eed8a0457a"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/546 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "43b4d7715b87477d806fe3193988e0ea"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/422M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e977b3566d18490680504fd253bc5b9c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at klue/roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.decoder.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at klue/roberta-base and are newly initialized: ['classifier.out_proj.weight', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "num_labels = 1\n",
        "model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint,\n",
        "                                                           num_labels=num_labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RnbdtgJE-hIS"
      },
      "outputs": [],
      "source": [
        "# 텐서보드 dir 정의 \n",
        "logdir_path = os.path.join(_CUR_DIR, \"sts_log\")# log 파일을 저장할 경로를 지정\n",
        "writer = SummaryWriter(logdir_path) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wqDGgXa699WQ"
      },
      "outputs": [],
      "source": [
        "##### 베이스 라인 모델: batch_size= 32, 나머지는 default\n",
        "\n",
        "output_dir = os.path.join(_CUR_DIR, \"sts_output2\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZK-5cKY4-wOR"
      },
      "outputs": [],
      "source": [
        "args = TrainingArguments(\n",
        "    # checkpoint\n",
        "    output_dir=output_dir,\n",
        "\n",
        "    # Model Save & Load\n",
        "    save_strategy = \"epoch\", # 각 epoch 마지막에 저장 \n",
        "    load_best_model_at_end=True, # train 종료시 best model 로드할지 여부\n",
        "\n",
        "    # lerning_rate\n",
        "    # learning_rate = lr,\n",
        "    \n",
        "    # Dataset\n",
        "    num_train_epochs=epochs,\n",
        "    per_device_train_batch_size=train_batch_size,\n",
        "    per_device_eval_batch_size=eval_batch_size,\n",
        "    \n",
        "    # Evaluation \n",
        "    evaluation_strategy = \"epoch\",# 각 epoch 마지막에 평가\n",
        "\n",
        "    # Randomness\n",
        "    seed=seed,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "7fe1e70e19c041778b31de17182717e2",
            "8a59958566d6426aa08e5034e127eb1a",
            "6710b7603b3d49aa9f1886a757773738",
            "be23d4aadec24875a27ab2df73d64a7c",
            "13c4a05d05834d20bd7f7c47d7f69b72",
            "42a21255ef8c41299930f8ff35111bc9",
            "0ecd1e110314465bad1b3168acdc7eb6",
            "07c05fdc87b54bae996b8df5b75a9d3a",
            "53d7458317674c249290f8130ea11e26",
            "8527577021194ba3ac81a7920a0961b9",
            "b4ef8df2770e49e19e69837dc0c72ed9",
            "bb8ec8afbf214ec291f5b2211a56d674",
            "60b064f41d1f44fc9ae00d8d243688d6",
            "b087e9b218714c8ebcb52bf1e2be4cb9",
            "aa8c46c738c2402b90112dfc0a3d46ba",
            "1a591ac5a365427c88da1fd86b1bc140",
            "024915eaac2844d8a38524030661d38b",
            "50d6e0e505df45a7b82564e63289f9f8",
            "496fbbb37e724af08a0df51fc310a6f9",
            "2b12b9a4f7b34f1cab0d0573aa6b34f4",
            "dae939a41ac449adbd7f1111b960124e",
            "1937f00cb72149f499adb87f2f8ba4dd"
          ]
        },
        "id": "Ty3f4i8G_A91",
        "outputId": "185e1efd-839b-4771-fa0f-55f9bddbc67b"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/2.32k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7fe1e70e19c041778b31de17182717e2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/2.15k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bb8ec8afbf214ec291f5b2211a56d674"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "# f1과 pearson 메트릭 불러오기 \n",
        "metric_f1 = load_metric(\"f1\") \n",
        "metric_pearson = load_metric(\"pearsonr\") "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Bj9kPRQB_GJS"
      },
      "outputs": [],
      "source": [
        "# 출력시 메트릭 정의 하는 함수 \n",
        "def compute_metrics(eval_pred):\n",
        "    predictions, labels = eval_pred\n",
        "    predictions = predictions[:, 0]\n",
        "    predict_f1 = [1 if i>=3.0 else 0 for i in predictions] # 3.0 이상-> 1, 3.0 미만 -> 0으로 변환\n",
        "    labels_f1 = [1 if i>=3.0 else 0 for i in labels] # 3.0 이상-> 1, 3.0 미만 -> 0으로 변환\n",
        "\n",
        "    pr = metric_pearson.compute(predictions=predictions, # peason r 계산\n",
        "                                   references=labels)\n",
        "\n",
        "    f1 = metric_f1.compute(predictions=predict_f1,   # f1\n",
        "                                     references=labels_f1)\n",
        "    return {'pearsonr' : pr,\n",
        "             'f1' : f1}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PcyNbfmY-8Dd"
      },
      "outputs": [],
      "source": [
        "#옵티마이져 및 스케줄러 설정:32\n",
        "optimizer = AdamW(\n",
        "    model.parameters(),\n",
        "    lr=5e-5,\n",
        "    eps=1e-8\n",
        ")\n",
        "\n",
        "batch_size = 32\n",
        "total_steps = round(len(encoded_train.shard(index=1, num_shards=10))/batch_size)*5\n",
        "\n",
        "# lr 선형으로 감소시키는 스케줄러\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0,\n",
        "                                            num_training_steps = total_steps)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FSjB_VE3_JAF"
      },
      "outputs": [],
      "source": [
        "# trainer 정의 \n",
        "trainer = Trainer(\n",
        "    model,\n",
        "    args,\n",
        "    train_dataset=encoded_train,\n",
        "    eval_dataset=encoded_valid,\n",
        "    tokenizer=tokenizer,\n",
        "    compute_metrics=compute_metrics,\n",
        "    optimizers = (optimizer, scheduler),\n",
        "    callbacks=[EarlyStoppingCallback(early_stopping_patience = patience)] # earlystopping 추가\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "j5Urh7Cf_TJm",
        "outputId": "9ae936ee-3e90-4ee8-99e8-9ec6a72925d1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The following columns in the training set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running training *****\n",
            "  Num examples = 10501\n",
            "  Num Epochs = 25\n",
            "  Instantaneous batch size per device = 58\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 58\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 4550\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='546' max='4550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [ 546/4550 06:31 < 48:03, 1.39 it/s, Epoch 3/25]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Pearsonr</th>\n",
              "      <th>F1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>No log</td>\n",
              "      <td>0.201800</td>\n",
              "      <td>{'pearsonr': 0.9746447531475442}</td>\n",
              "      <td>{'f1': 0.9565987599645704}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>No log</td>\n",
              "      <td>0.201800</td>\n",
              "      <td>{'pearsonr': 0.9746447531475442}</td>\n",
              "      <td>{'f1': 0.9565987599645704}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.238000</td>\n",
              "      <td>0.201800</td>\n",
              "      <td>{'pearsonr': 0.9746447531475442}</td>\n",
              "      <td>{'f1': 0.9565987599645704}</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1167\n",
            "  Batch size = 64\n",
            "Trainer is attempting to log a value of \"{'pearsonr': 0.9746447531475442}\" of type <class 'dict'> for key \"eval/pearsonr\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
            "Trainer is attempting to log a value of \"{'f1': 0.9565987599645704}\" of type <class 'dict'> for key \"eval/f1\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
            "Saving model checkpoint to /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-182\n",
            "Configuration saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-182/config.json\n",
            "Model weights saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-182/pytorch_model.bin\n",
            "tokenizer config file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-182/tokenizer_config.json\n",
            "Special tokens file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-182/special_tokens_map.json\n",
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1167\n",
            "  Batch size = 64\n",
            "Trainer is attempting to log a value of \"{'pearsonr': 0.9746447531475442}\" of type <class 'dict'> for key \"eval/pearsonr\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
            "Trainer is attempting to log a value of \"{'f1': 0.9565987599645704}\" of type <class 'dict'> for key \"eval/f1\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
            "Saving model checkpoint to /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-364\n",
            "Configuration saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-364/config.json\n",
            "Model weights saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-364/pytorch_model.bin\n",
            "tokenizer config file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-364/tokenizer_config.json\n",
            "Special tokens file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-364/special_tokens_map.json\n",
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1167\n",
            "  Batch size = 64\n",
            "Trainer is attempting to log a value of \"{'pearsonr': 0.9746447531475442}\" of type <class 'dict'> for key \"eval/pearsonr\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
            "Trainer is attempting to log a value of \"{'f1': 0.9565987599645704}\" of type <class 'dict'> for key \"eval/f1\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
            "Saving model checkpoint to /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-546\n",
            "Configuration saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-546/config.json\n",
            "Model weights saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-546/pytorch_model.bin\n",
            "tokenizer config file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-546/tokenizer_config.json\n",
            "Special tokens file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-546/special_tokens_map.json\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-182 (score: 0.20179980993270874).\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=546, training_loss=0.230909757125072, metrics={'train_runtime': 392.5236, 'train_samples_per_second': 668.813, 'train_steps_per_second': 11.592, 'total_flos': 2039607796506498.0, 'train_loss': 0.230909757125072, 'epoch': 3.0})"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ],
      "source": [
        "trainer.train()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 299
        },
        "id": "bsvpl_SF_bDW",
        "outputId": "99da615a-a772-4cc5-96bb-bf0e22a3fa06"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1167\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='19' max='19' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [19/19 00:03]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Trainer is attempting to log a value of \"{'pearsonr': 0.9746447531475442}\" of type <class 'dict'> for key \"eval/pearsonr\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
            "Trainer is attempting to log a value of \"{'f1': 0.9565987599645704}\" of type <class 'dict'> for key \"eval/f1\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'epoch': 3.0,\n",
              " 'eval_f1': {'f1': 0.9565987599645704},\n",
              " 'eval_loss': 0.20179980993270874,\n",
              " 'eval_pearsonr': {'pearsonr': 0.9746447531475442},\n",
              " 'eval_runtime': 3.7256,\n",
              " 'eval_samples_per_second': 313.236,\n",
              " 'eval_steps_per_second': 5.1}"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ],
      "source": [
        "# baseline score\n",
        "trainer.evaluate()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 132
        },
        "id": "biwfHegA_YAB",
        "outputId": "40e536f6-7783-42a6-bca8-0ac16fcc08ca"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The following columns in the test set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Prediction *****\n",
            "  Num examples = 519\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='28' max='19' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [19/19 01:18]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "# test set으로 prediction \n",
        "predictions = trainer.predict(encoded_test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h1JZOtIrLvik",
        "outputId": "3153f665-8b32-4b69-827b-f47c44c0a1e1"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'test_f1': {'f1': 0.8346456692913387},\n",
              " 'test_loss': 0.5009527802467346,\n",
              " 'test_pearsonr': {'pearsonr': 0.8997988484485431},\n",
              " 'test_runtime': 1.738,\n",
              " 'test_samples_per_second': 298.616,\n",
              " 'test_steps_per_second': 5.178}"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ],
      "source": [
        "# test set score \n",
        "predictions.metrics"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CRpbCId_gqnG"
      },
      "source": [
        "# Hyper Parameter Tuning"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3OvmoDjeX69O",
        "outputId": "3ea6e24a-6069-4e56-c264-becd0acafd79"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: optuna in /usr/local/lib/python3.7/dist-packages (2.10.0)\n",
            "Requirement already satisfied: cliff in /usr/local/lib/python3.7/dist-packages (from optuna) (3.10.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from optuna) (4.64.0)\n",
            "Requirement already satisfied: sqlalchemy>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from optuna) (1.4.36)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from optuna) (1.21.6)\n",
            "Requirement already satisfied: alembic in /usr/local/lib/python3.7/dist-packages (from optuna) (1.8.0)\n",
            "Requirement already satisfied: colorlog in /usr/local/lib/python3.7/dist-packages (from optuna) (6.6.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from optuna) (21.3)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from optuna) (6.0)\n",
            "Requirement already satisfied: scipy!=1.4.0 in /usr/local/lib/python3.7/dist-packages (from optuna) (1.4.1)\n",
            "Requirement already satisfied: cmaes>=0.8.2 in /usr/local/lib/python3.7/dist-packages (from optuna) (0.8.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->optuna) (3.0.9)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from sqlalchemy>=1.1.0->optuna) (4.11.3)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.7/dist-packages (from sqlalchemy>=1.1.0->optuna) (1.1.2)\n",
            "Requirement already satisfied: Mako in /usr/local/lib/python3.7/dist-packages (from alembic->optuna) (1.2.0)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from alembic->optuna) (5.7.1)\n",
            "Requirement already satisfied: pbr!=2.1.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from cliff->optuna) (5.9.0)\n",
            "Requirement already satisfied: stevedore>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from cliff->optuna) (3.5.0)\n",
            "Requirement already satisfied: cmd2>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from cliff->optuna) (2.4.1)\n",
            "Requirement already satisfied: autopage>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from cliff->optuna) (0.5.1)\n",
            "Requirement already satisfied: PrettyTable>=0.7.2 in /usr/local/lib/python3.7/dist-packages (from cliff->optuna) (3.3.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from cmd2>=1.0.0->cliff->optuna) (4.2.0)\n",
            "Requirement already satisfied: pyperclip>=1.6 in /usr/local/lib/python3.7/dist-packages (from cmd2>=1.0.0->cliff->optuna) (1.8.2)\n",
            "Requirement already satisfied: attrs>=16.3.0 in /usr/local/lib/python3.7/dist-packages (from cmd2>=1.0.0->cliff->optuna) (21.4.0)\n",
            "Requirement already satisfied: wcwidth>=0.1.7 in /usr/local/lib/python3.7/dist-packages (from cmd2>=1.0.0->cliff->optuna) (0.2.5)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->sqlalchemy>=1.1.0->optuna) (3.8.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.9.2 in /usr/local/lib/python3.7/dist-packages (from Mako->alembic->optuna) (2.0.1)\n"
          ]
        }
      ],
      "source": [
        "#hyperparamer search\n",
        "! pip install optuna"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lEIWFfvxYWeJ"
      },
      "outputs": [],
      "source": [
        "# trainer 여러번 돌기 때문에 모델 초기화 해줘야함 \n",
        "def model_init():\n",
        "    model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=num_labels) \n",
        "    \n",
        "    return AutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=num_labels) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_Wxy-R6Pdi4Q",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ef99c02e-4929-441c-bcc5-ab88c943f558"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
          ]
        }
      ],
      "source": [
        "output_hyper_dir = os.path.join(_CUR_DIR, \"sts_hyper\")\n",
        "\n",
        "hyper_args = TrainingArguments(\n",
        "    # checkpoint\n",
        "    output_dir=output_hyper_dir,\n",
        "\n",
        "    # evaluation\n",
        "    evaluation_strategy=\"epoch\",\n",
        "\n",
        "    #log\n",
        "    logging_strategy=\"steps\",\n",
        "    logging_steps=1,\n",
        "    logging_first_step=False,\n",
        "\n",
        "    overwrite_output_dir=True,\n",
        "    \n",
        "    save_strategy=\"epoch\",\n",
        "    load_best_model_at_end=True,\n",
        "    \n",
        "    metric_for_best_model=\"eval_f1\",\n",
        "\n",
        "    seed=seed,\n",
        "\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "D_OvkyNOhs_R"
      },
      "outputs": [],
      "source": [
        "# 하이퍼파라미터 튜닝때 사용할 매트릭스 함수 \n",
        "def hyper_metrics(eval_pred):\n",
        "    predictions, labels = eval_pred\n",
        "    predictions = predictions[:, 0]\n",
        "    predict_f1 = [1 if i>=3.0 else 0 for i in predictions]\n",
        "    labels_f1 = [1 if i>=3.0 else 0 for i in labels]\n",
        "\n",
        "    f1 = metric_f1.compute(predictions=predict_f1,\n",
        "                                     references=labels_f1)\n",
        "    return f1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9MtnUwHth2Wj"
      },
      "outputs": [],
      "source": [
        "num_labels = 1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cvoZyHofB5ZW"
      },
      "outputs": [],
      "source": [
        "# 탐색할 파라미터 값 지정 \n",
        "def my_hp_space(trial):\n",
        "    return {\n",
        "        \"learning_rate\": trial.suggest_float(\"learning_rate\", 1e-5, 3e-5, log=True),\n",
        "        \"warmup_steps\":  trial.suggest_float(\"warmup_steps\", 0.01, 0.09, step=0.3),\n",
        "        \"weight_decay\":  trial.suggest_float(\"weight_decay\", 1e-6, 1e-1),   \n",
        "        \"per_device_train_batch_size\": trial.suggest_categorical(\"per_device_train_batch_size\", [4, 8, 16, 32]),\n",
        "    }"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AlX8r4kpYwQ3",
        "outputId": "b4ee5201-a439-45db-dc61-6caedea2cbed"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loading configuration file https://huggingface.co/klue/roberta-base/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/a96469ca2a238496d435a0e9e202f261119c146a0326444b6d68ae1adc35e04f.85b0b02ba2a483f3adb8a60ab70dbd875768fcd5e6cdb21a593c6e02fdffac3a\n",
            "Model config RobertaConfig {\n",
            "  \"_name_or_path\": \"klue/roberta-base\",\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"tokenizer_class\": \"BertTokenizer\",\n",
            "  \"transformers_version\": \"4.19.2\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 32000\n",
            "}\n",
            "\n",
            "loading weights file https://huggingface.co/klue/roberta-base/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/b204e0dc0a3b8fd45b35e7fcefd97c5f839b86c14aea510f1eb38fb8469e23d8.57d3cd0dfa80e5a249a776870dc87b6da993900685a271086750174009115320\n",
            "Some weights of the model checkpoint at klue/roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.decoder.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at klue/roberta-base and are newly initialized: ['classifier.out_proj.weight', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "# trainer 정의 \n",
        "trainer = Trainer(\n",
        "    args=hyper_args,\n",
        "    train_dataset=encoded_train,\n",
        "    eval_dataset=encoded_valid,\n",
        "    model_init=model_init,\n",
        "    tokenizer=tokenizer,\n",
        "    compute_metrics=hyper_metrics,\n",
        "    callbacks=[EarlyStoppingCallback(early_stopping_patience = patience)] # earlystopping 추가\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def my_objective(metrics):\n",
        "    return metrics[\"eval_f1\"]   "
      ],
      "metadata": {
        "id": "HJpPhJMTVSR-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3jakRgn6B73Y",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "f56dde86-931a-427f-a8de-690ba96586e4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[32m[I 2022-06-01 08:18:20,592]\u001b[0m A new study created in memory with name: no-name-36d1304b-457b-465d-8863-448930d10fef\u001b[0m\n",
            "/usr/local/lib/python3.7/dist-packages/optuna/distributions.py:548: UserWarning: The distribution is specified by [0.01, 0.09] and q=0.3, but the range is not divisible by `q`. It will be replaced by [0.01, 0.01].\n",
            "  low=low, old_high=old_high, high=high, step=q\n",
            "Trial:\n",
            "loading configuration file https://huggingface.co/klue/roberta-base/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/a96469ca2a238496d435a0e9e202f261119c146a0326444b6d68ae1adc35e04f.85b0b02ba2a483f3adb8a60ab70dbd875768fcd5e6cdb21a593c6e02fdffac3a\n",
            "Model config RobertaConfig {\n",
            "  \"_name_or_path\": \"klue/roberta-base\",\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"tokenizer_class\": \"BertTokenizer\",\n",
            "  \"transformers_version\": \"4.19.2\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 32000\n",
            "}\n",
            "\n",
            "loading weights file https://huggingface.co/klue/roberta-base/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/b204e0dc0a3b8fd45b35e7fcefd97c5f839b86c14aea510f1eb38fb8469e23d8.57d3cd0dfa80e5a249a776870dc87b6da993900685a271086750174009115320\n",
            "Some weights of the model checkpoint at klue/roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.decoder.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at klue/roberta-base and are newly initialized: ['classifier.out_proj.weight', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "The following columns in the training set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 10501\n",
            "  Num Epochs = 3\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3939\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3939' max='3939' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3939/3939 09:11, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>F1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.040700</td>\n",
              "      <td>0.180550</td>\n",
              "      <td>0.955816</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.045100</td>\n",
              "      <td>0.223150</td>\n",
              "      <td>0.952381</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.043300</td>\n",
              "      <td>0.176157</td>\n",
              "      <td>0.963295</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1167\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-0/checkpoint-1313\n",
            "Configuration saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-0/checkpoint-1313/config.json\n",
            "Model weights saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-0/checkpoint-1313/pytorch_model.bin\n",
            "tokenizer config file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-0/checkpoint-1313/tokenizer_config.json\n",
            "Special tokens file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-0/checkpoint-1313/special_tokens_map.json\n",
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1167\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-0/checkpoint-2626\n",
            "Configuration saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-0/checkpoint-2626/config.json\n",
            "Model weights saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-0/checkpoint-2626/pytorch_model.bin\n",
            "tokenizer config file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-0/checkpoint-2626/tokenizer_config.json\n",
            "Special tokens file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-0/checkpoint-2626/special_tokens_map.json\n",
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1167\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-0/checkpoint-3939\n",
            "Configuration saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-0/checkpoint-3939/config.json\n",
            "Model weights saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-0/checkpoint-3939/pytorch_model.bin\n",
            "tokenizer config file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-0/checkpoint-3939/tokenizer_config.json\n",
            "Special tokens file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-0/checkpoint-3939/special_tokens_map.json\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-0/checkpoint-3939 (score: 0.963294538943599).\n",
            "\u001b[32m[I 2022-06-01 08:27:33,584]\u001b[0m Trial 0 finished with value: 0.963294538943599 and parameters: {'learning_rate': 1.539880137960369e-05, 'warmup_steps': 0.01, 'weight_decay': 0.07028983728388466, 'per_device_train_batch_size': 16}. Best is trial 0 with value: 0.963294538943599.\u001b[0m\n",
            "/usr/local/lib/python3.7/dist-packages/optuna/distributions.py:548: UserWarning: The distribution is specified by [0.01, 0.09] and q=0.3, but the range is not divisible by `q`. It will be replaced by [0.01, 0.01].\n",
            "  low=low, old_high=old_high, high=high, step=q\n",
            "Trial:\n",
            "loading configuration file https://huggingface.co/klue/roberta-base/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/a96469ca2a238496d435a0e9e202f261119c146a0326444b6d68ae1adc35e04f.85b0b02ba2a483f3adb8a60ab70dbd875768fcd5e6cdb21a593c6e02fdffac3a\n",
            "Model config RobertaConfig {\n",
            "  \"_name_or_path\": \"klue/roberta-base\",\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"tokenizer_class\": \"BertTokenizer\",\n",
            "  \"transformers_version\": \"4.19.2\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 32000\n",
            "}\n",
            "\n",
            "loading weights file https://huggingface.co/klue/roberta-base/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/b204e0dc0a3b8fd45b35e7fcefd97c5f839b86c14aea510f1eb38fb8469e23d8.57d3cd0dfa80e5a249a776870dc87b6da993900685a271086750174009115320\n",
            "Some weights of the model checkpoint at klue/roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.decoder.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at klue/roberta-base and are newly initialized: ['classifier.out_proj.weight', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "The following columns in the training set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 10501\n",
            "  Num Epochs = 3\n",
            "  Instantaneous batch size per device = 32\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 32\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3939\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3939' max='3939' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3939/3939 09:11, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>F1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.036600</td>\n",
              "      <td>0.175341</td>\n",
              "      <td>0.959786</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.057100</td>\n",
              "      <td>0.221591</td>\n",
              "      <td>0.951456</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.047600</td>\n",
              "      <td>0.177857</td>\n",
              "      <td>0.961504</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1167\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-1/checkpoint-1313\n",
            "Configuration saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-1/checkpoint-1313/config.json\n",
            "Model weights saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-1/checkpoint-1313/pytorch_model.bin\n",
            "tokenizer config file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-1/checkpoint-1313/tokenizer_config.json\n",
            "Special tokens file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-1/checkpoint-1313/special_tokens_map.json\n",
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1167\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-1/checkpoint-2626\n",
            "Configuration saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-1/checkpoint-2626/config.json\n",
            "Model weights saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-1/checkpoint-2626/pytorch_model.bin\n",
            "tokenizer config file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-1/checkpoint-2626/tokenizer_config.json\n",
            "Special tokens file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-1/checkpoint-2626/special_tokens_map.json\n",
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1167\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-1/checkpoint-3939\n",
            "Configuration saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-1/checkpoint-3939/config.json\n",
            "Model weights saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-1/checkpoint-3939/pytorch_model.bin\n",
            "tokenizer config file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-1/checkpoint-3939/tokenizer_config.json\n",
            "Special tokens file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-1/checkpoint-3939/special_tokens_map.json\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from /content/drive/MyDrive/AIBootCamp/NLP/sts_hyper/run-1/checkpoint-3939 (score: 0.9615040286481648).\n",
            "\u001b[32m[I 2022-06-01 08:36:47,238]\u001b[0m Trial 1 finished with value: 0.9615040286481648 and parameters: {'learning_rate': 1.3645381074107456e-05, 'warmup_steps': 0.01, 'weight_decay': 0.004225918738145134, 'per_device_train_batch_size': 32}. Best is trial 0 with value: 0.963294538943599.\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "# 탐색 시작 \n",
        "best_run = trainer.hyperparameter_search(n_trials=2, \n",
        "                                         direction=\"maximize\", \n",
        "                                         hp_space=my_hp_space,\n",
        "                                         compute_objective=my_objective)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# best model 파라미터 값 \n",
        "best_run.hyperparameters"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5e-ZAVnQXfDZ",
        "outputId": "2bb8dc6c-85b8-4847-c32f-a5ab12d47311"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'learning_rate': 1.539880137960369e-05,\n",
              " 'per_device_train_batch_size': 16,\n",
              " 'warmup_steps': 0.01,\n",
              " 'weight_decay': 0.07028983728388466}"
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tune_args = TrainingArguments(\n",
        "    # checkpoint\n",
        "    output_dir=output_dir,\n",
        "\n",
        "    # Model Save & Load\n",
        "    save_strategy = \"epoch\", # 'steps'\n",
        "    load_best_model_at_end=True,\n",
        "\n",
        "    # Dataset\n",
        "    num_train_epochs=epochs,\n",
        "    per_device_train_batch_size=16,\n",
        "    per_device_eval_batch_size=32,\n",
        "    \n",
        "    # Optimizer (AdamW)\n",
        "    learning_rate= 1.539880137960369e-05, \n",
        "    warmup_steps=  0.01,\n",
        "    weight_decay= 0.07028983728388466,\n",
        "\n",
        "    # Evaluation \n",
        "    evaluation_strategy = \"epoch\",\n",
        "\n",
        "\n",
        "    # Randomness\n",
        "    seed= seed,\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y_XueOIOaTpW",
        "outputId": "6049b888-fc2e-4ad5-cd3f-7b5d8ac23752"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trainer = Trainer(\n",
        "    model,\n",
        "    tune_args,\n",
        "    train_dataset=encoded_train,\n",
        "    eval_dataset=encoded_valid,\n",
        "    tokenizer=tokenizer,\n",
        "    compute_metrics=compute_metrics,\n",
        "    callbacks=[EarlyStoppingCallback(early_stopping_patience=1)]\n",
        ")"
      ],
      "metadata": {
        "id": "yGL369Sva8-a"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 학습 시작\n",
        "trainer.train()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "HjIQx-c6a-iQ",
        "outputId": "1f4b7b0c-d2bf-4e59-de3a-5421dd409885"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The following columns in the training set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 10501\n",
            "  Num Epochs = 5\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3285\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='2628' max='3285' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [2628/3285 10:03 < 02:31, 4.35 it/s, Epoch 4/5]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Pearsonr</th>\n",
              "      <th>F1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.167700</td>\n",
              "      <td>0.329344</td>\n",
              "      <td>{'pearsonr': 0.9739637572281786}</td>\n",
              "      <td>{'f1': 0.9466316710411198}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.141300</td>\n",
              "      <td>0.201549</td>\n",
              "      <td>{'pearsonr': 0.9783806667507823}</td>\n",
              "      <td>{'f1': 0.9633601429848078}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.106300</td>\n",
              "      <td>0.147764</td>\n",
              "      <td>{'pearsonr': 0.9798616752331278}</td>\n",
              "      <td>{'f1': 0.9648964896489649}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.059800</td>\n",
              "      <td>0.191242</td>\n",
              "      <td>{'pearsonr': 0.9803258649831558}</td>\n",
              "      <td>{'f1': 0.9609929078014184}</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1167\n",
            "  Batch size = 32\n",
            "Trainer is attempting to log a value of \"{'pearsonr': 0.9739637572281786}\" of type <class 'dict'> for key \"eval/pearsonr\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
            "Trainer is attempting to log a value of \"{'f1': 0.9466316710411198}\" of type <class 'dict'> for key \"eval/f1\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
            "Saving model checkpoint to /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-657\n",
            "Configuration saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-657/config.json\n",
            "Model weights saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-657/pytorch_model.bin\n",
            "tokenizer config file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-657/tokenizer_config.json\n",
            "Special tokens file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-657/special_tokens_map.json\n",
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1167\n",
            "  Batch size = 32\n",
            "Trainer is attempting to log a value of \"{'pearsonr': 0.9783806667507823}\" of type <class 'dict'> for key \"eval/pearsonr\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
            "Trainer is attempting to log a value of \"{'f1': 0.9633601429848078}\" of type <class 'dict'> for key \"eval/f1\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
            "Saving model checkpoint to /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-1314\n",
            "Configuration saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-1314/config.json\n",
            "Model weights saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-1314/pytorch_model.bin\n",
            "tokenizer config file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-1314/tokenizer_config.json\n",
            "Special tokens file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-1314/special_tokens_map.json\n",
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1167\n",
            "  Batch size = 32\n",
            "Trainer is attempting to log a value of \"{'pearsonr': 0.9798616752331278}\" of type <class 'dict'> for key \"eval/pearsonr\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
            "Trainer is attempting to log a value of \"{'f1': 0.9648964896489649}\" of type <class 'dict'> for key \"eval/f1\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
            "Saving model checkpoint to /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-1971\n",
            "Configuration saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-1971/config.json\n",
            "Model weights saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-1971/pytorch_model.bin\n",
            "tokenizer config file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-1971/tokenizer_config.json\n",
            "Special tokens file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-1971/special_tokens_map.json\n",
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1167\n",
            "  Batch size = 32\n",
            "Trainer is attempting to log a value of \"{'pearsonr': 0.9803258649831558}\" of type <class 'dict'> for key \"eval/pearsonr\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
            "Trainer is attempting to log a value of \"{'f1': 0.9609929078014184}\" of type <class 'dict'> for key \"eval/f1\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
            "Saving model checkpoint to /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-2628\n",
            "Configuration saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-2628/config.json\n",
            "Model weights saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-2628/pytorch_model.bin\n",
            "tokenizer config file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-2628/tokenizer_config.json\n",
            "Special tokens file saved in /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-2628/special_tokens_map.json\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from /content/drive/MyDrive/AIBootCamp/NLP/sts_output2/checkpoint-1971 (score: 0.14776434004306793).\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=2628, training_loss=0.10827282883079448, metrics={'train_runtime': 603.7904, 'train_samples_per_second': 86.959, 'train_steps_per_second': 5.441, 'total_flos': 2707594004554080.0, 'train_loss': 0.10827282883079448, 'epoch': 4.0})"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# score\n",
        "trainer.evaluate()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 299
        },
        "id": "I0MKhh3Aa_4J",
        "outputId": "73de37e6-a1b6-4ca5-fb23-940c9b79a3e5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1167\n",
            "  Batch size = 32\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='37' max='37' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [37/37 00:04]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Trainer is attempting to log a value of \"{'pearsonr': 0.9798616752331278}\" of type <class 'dict'> for key \"eval/pearsonr\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
            "Trainer is attempting to log a value of \"{'f1': 0.9648964896489649}\" of type <class 'dict'> for key \"eval/f1\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'epoch': 4.0,\n",
              " 'eval_f1': {'f1': 0.9648964896489649},\n",
              " 'eval_loss': 0.14776434004306793,\n",
              " 'eval_pearsonr': {'pearsonr': 0.9798616752331278},\n",
              " 'eval_runtime': 4.9662,\n",
              " 'eval_samples_per_second': 234.988,\n",
              " 'eval_steps_per_second': 7.45}"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# test set으로 prediction \n",
        "predictions = trainer.predict(encoded_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 132
        },
        "id": "UIb6Pl1HbHbY",
        "outputId": "ab38f2e2-9ef8-44f1-c00a-2a3b65c6c77d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The following columns in the test set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: labels.binary-label, guid, sentence2, sentence1, labels.label, source. If labels.binary-label, guid, sentence2, sentence1, labels.label, source are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Prediction *****\n",
            "  Num examples = 519\n",
            "  Batch size = 32\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='54' max='37' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [37/37 00:12]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predictions.metrics"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tdvKsFf8daSW",
        "outputId": "2ba7977c-9a7f-4b58-8f9c-29b51b772ad6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'test_f1': {'f1': 0.8512396694214875},\n",
              " 'test_loss': 0.33514708280563354,\n",
              " 'test_pearsonr': {'pearsonr': 0.9250721379209141},\n",
              " 'test_runtime': 1.8916,\n",
              " 'test_samples_per_second': 274.37,\n",
              " 'test_steps_per_second': 8.987}"
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "{'test_f1': {'f1': 0.8346456692913387},\n",
        " 'test_loss': 0.5009527802467346,\n",
        " 'test_pearsonr': {'pearsonr': 0.8997988484485431},\n",
        " 'test_runtime': 1.738,\n",
        " 'test_samples_per_second': 298.616,\n",
        " 'test_steps_per_second': 5.178}"
      ],
      "metadata": {
        "id": "3rePUgyhf6kO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# result\n",
        "result= pd.DataFrame({'test_f1':[round(0.8512396694214875, 4),round(0.8346456692913387, 4),round(0.814498933901919, 4)],\n",
        "                             'test_loss':[ round(0.33514708280563354,4), round(0.5009527802467346, 4), round(0.4114389419555664, 4)],\n",
        "                             'test_pearsonr':[round(0.9250721379209141, 4), round(0.8997988484485431, 4), round(0.9010864050775359, 4)],\n",
        "                             'test_runtime':[1.8916,1.738,3.0923]}).T\n",
        "result.columns = ['hyper', 'optimizer', 'base']\n",
        "result"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "id": "KnqEanZ4eBUI",
        "outputId": "5f8f9d74-f6f5-4cf8-ea4a-563a7b89e777"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                hyper  optimizer    base\n",
              "test_f1        0.8512     0.8346  0.8145\n",
              "test_loss      0.3351     0.5010  0.4114\n",
              "test_pearsonr  0.9251     0.8998  0.9011\n",
              "test_runtime   1.8916     1.7380  3.0923"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8919b69f-30f8-49ad-891b-2aca5cb19611\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>hyper</th>\n",
              "      <th>optimizer</th>\n",
              "      <th>base</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>test_f1</th>\n",
              "      <td>0.8512</td>\n",
              "      <td>0.8346</td>\n",
              "      <td>0.8145</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>test_loss</th>\n",
              "      <td>0.3351</td>\n",
              "      <td>0.5010</td>\n",
              "      <td>0.4114</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>test_pearsonr</th>\n",
              "      <td>0.9251</td>\n",
              "      <td>0.8998</td>\n",
              "      <td>0.9011</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>test_runtime</th>\n",
              "      <td>1.8916</td>\n",
              "      <td>1.7380</td>\n",
              "      <td>3.0923</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8919b69f-30f8-49ad-891b-2aca5cb19611')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8919b69f-30f8-49ad-891b-2aca5cb19611 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8919b69f-30f8-49ad-891b-2aca5cb19611');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# save score "
      ],
      "metadata": {
        "id": "SrvkSS6cjY5H"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "predictions[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ost5aCc9lalg",
        "outputId": "337b7cbd-b588-45fd-ed4d-5d890620a841"
      },
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[ 4.653361  ],\n",
              "       [ 1.7810117 ],\n",
              "       [ 1.4303722 ],\n",
              "       [ 4.100582  ],\n",
              "       [ 2.887404  ],\n",
              "       [ 1.7765844 ],\n",
              "       [ 3.6381445 ],\n",
              "       [ 0.89057696],\n",
              "       [ 0.04111077],\n",
              "       [ 1.714973  ],\n",
              "       [ 3.7967744 ],\n",
              "       [ 2.7179408 ],\n",
              "       [ 0.53205067],\n",
              "       [ 2.452306  ],\n",
              "       [ 4.462683  ],\n",
              "       [ 3.402286  ],\n",
              "       [ 4.5356064 ],\n",
              "       [ 1.4683228 ],\n",
              "       [ 0.06471527],\n",
              "       [ 0.79742837],\n",
              "       [ 3.5034406 ],\n",
              "       [ 3.6744692 ],\n",
              "       [ 4.072261  ],\n",
              "       [ 3.863794  ],\n",
              "       [ 0.01405839],\n",
              "       [ 3.6003497 ],\n",
              "       [ 3.279758  ],\n",
              "       [ 2.6536767 ],\n",
              "       [ 0.40689546],\n",
              "       [ 3.6236727 ],\n",
              "       [ 1.6346698 ],\n",
              "       [ 0.07456977],\n",
              "       [ 0.27325472],\n",
              "       [ 3.6134775 ],\n",
              "       [ 4.639293  ],\n",
              "       [ 2.2422178 ],\n",
              "       [ 3.5118694 ],\n",
              "       [ 3.8391118 ],\n",
              "       [ 4.7428346 ],\n",
              "       [ 3.2471724 ],\n",
              "       [ 3.4478896 ],\n",
              "       [ 4.4284425 ],\n",
              "       [ 1.096538  ],\n",
              "       [ 2.968014  ],\n",
              "       [ 2.110668  ],\n",
              "       [ 3.98568   ],\n",
              "       [ 4.1480823 ],\n",
              "       [ 4.612698  ],\n",
              "       [ 0.7414808 ],\n",
              "       [ 3.9966974 ],\n",
              "       [ 0.3661978 ],\n",
              "       [ 3.9925694 ],\n",
              "       [ 3.5220525 ],\n",
              "       [ 1.472162  ],\n",
              "       [ 4.129498  ],\n",
              "       [ 0.44914877],\n",
              "       [ 0.76101804],\n",
              "       [ 3.0387573 ],\n",
              "       [ 3.3184428 ],\n",
              "       [ 3.0389156 ],\n",
              "       [ 2.5191963 ],\n",
              "       [ 1.97747   ],\n",
              "       [ 3.216861  ],\n",
              "       [ 0.9561499 ],\n",
              "       [ 3.8447926 ],\n",
              "       [ 3.906469  ],\n",
              "       [ 0.722819  ],\n",
              "       [ 0.00618024],\n",
              "       [ 4.6900525 ],\n",
              "       [ 4.127798  ],\n",
              "       [ 1.4840914 ],\n",
              "       [ 0.7101338 ],\n",
              "       [ 3.508813  ],\n",
              "       [ 3.8230453 ],\n",
              "       [ 3.0263443 ],\n",
              "       [ 4.3968763 ],\n",
              "       [-0.01683717],\n",
              "       [ 4.1687527 ],\n",
              "       [ 3.4896863 ],\n",
              "       [ 3.6000175 ],\n",
              "       [ 3.2675247 ],\n",
              "       [ 2.1125402 ],\n",
              "       [ 3.8398528 ],\n",
              "       [ 2.444708  ],\n",
              "       [ 3.140617  ],\n",
              "       [ 3.4001215 ],\n",
              "       [ 3.9672117 ],\n",
              "       [ 3.9663742 ],\n",
              "       [ 0.30725217],\n",
              "       [ 3.6911542 ],\n",
              "       [ 0.01486165],\n",
              "       [ 4.2910833 ],\n",
              "       [ 4.6570134 ],\n",
              "       [ 2.856808  ],\n",
              "       [ 4.5377107 ],\n",
              "       [ 3.4123645 ],\n",
              "       [ 1.5922216 ],\n",
              "       [ 2.5762029 ],\n",
              "       [ 3.9587982 ],\n",
              "       [ 1.1522838 ],\n",
              "       [ 0.65815043],\n",
              "       [ 3.8194988 ],\n",
              "       [ 4.201158  ],\n",
              "       [ 0.7193541 ],\n",
              "       [ 3.7824922 ],\n",
              "       [ 2.6326158 ],\n",
              "       [ 0.37370744],\n",
              "       [ 2.8776765 ],\n",
              "       [ 3.9009433 ],\n",
              "       [ 2.0919805 ],\n",
              "       [ 0.813532  ],\n",
              "       [ 1.2840846 ],\n",
              "       [ 3.6674447 ],\n",
              "       [ 1.7319906 ],\n",
              "       [ 0.6587003 ],\n",
              "       [ 4.0122232 ],\n",
              "       [ 3.3593488 ],\n",
              "       [ 1.742066  ],\n",
              "       [ 3.9771178 ],\n",
              "       [ 4.1386557 ],\n",
              "       [ 3.319165  ],\n",
              "       [ 1.0376887 ],\n",
              "       [ 0.573805  ],\n",
              "       [ 2.5656312 ],\n",
              "       [ 4.35048   ],\n",
              "       [ 3.8550591 ],\n",
              "       [ 0.5907598 ],\n",
              "       [ 3.8174398 ],\n",
              "       [ 2.2304068 ],\n",
              "       [ 4.159051  ],\n",
              "       [ 3.7647862 ],\n",
              "       [ 4.1202583 ],\n",
              "       [ 0.91509616],\n",
              "       [ 3.8321023 ],\n",
              "       [ 3.2392852 ],\n",
              "       [ 3.0028923 ],\n",
              "       [ 3.3638475 ],\n",
              "       [ 4.3087935 ],\n",
              "       [ 0.64156485],\n",
              "       [ 2.752291  ],\n",
              "       [ 3.8731742 ],\n",
              "       [ 3.474726  ],\n",
              "       [ 0.69445103],\n",
              "       [ 1.5802088 ],\n",
              "       [ 2.1280022 ],\n",
              "       [ 3.9010458 ],\n",
              "       [ 4.0295563 ],\n",
              "       [ 3.9346972 ],\n",
              "       [ 2.4040596 ],\n",
              "       [ 3.7275875 ],\n",
              "       [ 3.1470244 ],\n",
              "       [ 1.410428  ],\n",
              "       [ 3.3969972 ],\n",
              "       [ 1.1053749 ],\n",
              "       [ 4.153998  ],\n",
              "       [ 1.8805226 ],\n",
              "       [ 2.8433096 ],\n",
              "       [ 2.9988303 ],\n",
              "       [ 0.848103  ],\n",
              "       [ 0.98841965],\n",
              "       [ 2.2420866 ],\n",
              "       [ 1.5902741 ],\n",
              "       [ 2.6751604 ],\n",
              "       [ 2.0476482 ],\n",
              "       [ 1.6392909 ],\n",
              "       [ 4.165111  ],\n",
              "       [ 0.0248671 ],\n",
              "       [ 3.4195871 ],\n",
              "       [ 1.5236417 ],\n",
              "       [ 3.8156123 ],\n",
              "       [ 2.7562313 ],\n",
              "       [ 4.026107  ],\n",
              "       [ 4.540235  ],\n",
              "       [ 1.4006268 ],\n",
              "       [ 1.878074  ],\n",
              "       [ 4.0004954 ],\n",
              "       [ 3.9369123 ],\n",
              "       [ 3.698502  ],\n",
              "       [ 3.5159502 ],\n",
              "       [ 2.1021242 ],\n",
              "       [ 3.6412416 ],\n",
              "       [ 4.693074  ],\n",
              "       [ 2.5939448 ],\n",
              "       [ 3.1525772 ],\n",
              "       [ 2.5993276 ],\n",
              "       [ 1.0081371 ],\n",
              "       [ 2.6164956 ],\n",
              "       [ 1.1823803 ],\n",
              "       [ 4.06886   ],\n",
              "       [ 4.22022   ],\n",
              "       [ 3.6653466 ],\n",
              "       [ 4.4804277 ],\n",
              "       [ 1.6450807 ],\n",
              "       [ 2.7953017 ],\n",
              "       [ 2.7928936 ],\n",
              "       [ 2.9543998 ],\n",
              "       [ 0.20887317],\n",
              "       [ 2.80291   ],\n",
              "       [ 3.6069655 ],\n",
              "       [ 3.8087902 ],\n",
              "       [ 0.71607614],\n",
              "       [ 3.8484912 ],\n",
              "       [ 3.5955107 ],\n",
              "       [ 3.3146613 ],\n",
              "       [ 1.8778982 ],\n",
              "       [ 3.9231317 ],\n",
              "       [ 0.87243646],\n",
              "       [ 4.0500393 ],\n",
              "       [ 4.1154776 ],\n",
              "       [ 3.2325406 ],\n",
              "       [ 1.8982687 ],\n",
              "       [ 0.7863059 ],\n",
              "       [ 4.143476  ],\n",
              "       [ 1.5276483 ],\n",
              "       [ 3.363877  ],\n",
              "       [ 2.6897178 ],\n",
              "       [ 2.1004622 ],\n",
              "       [-0.03401637],\n",
              "       [ 4.1910663 ],\n",
              "       [ 3.717608  ],\n",
              "       [ 0.10996998],\n",
              "       [ 0.05806916],\n",
              "       [ 0.84870887],\n",
              "       [ 1.6486707 ],\n",
              "       [ 1.5591799 ],\n",
              "       [ 3.161826  ],\n",
              "       [ 2.4541314 ],\n",
              "       [ 3.4928806 ],\n",
              "       [ 0.56959695],\n",
              "       [ 1.2879131 ],\n",
              "       [ 2.467096  ],\n",
              "       [ 3.0305622 ],\n",
              "       [ 4.313698  ],\n",
              "       [ 0.5663848 ],\n",
              "       [ 4.062278  ],\n",
              "       [ 1.0130937 ],\n",
              "       [ 2.508952  ],\n",
              "       [ 3.595989  ],\n",
              "       [ 3.6057315 ],\n",
              "       [ 3.8966992 ],\n",
              "       [ 4.323829  ],\n",
              "       [ 3.8351626 ],\n",
              "       [ 0.9577581 ],\n",
              "       [ 4.154777  ],\n",
              "       [ 1.2148004 ],\n",
              "       [ 3.4008844 ],\n",
              "       [ 4.08606   ],\n",
              "       [ 2.0606267 ],\n",
              "       [ 4.0352926 ],\n",
              "       [ 2.7603643 ],\n",
              "       [ 3.576764  ],\n",
              "       [ 1.7834841 ],\n",
              "       [ 1.3636025 ],\n",
              "       [ 3.9622269 ],\n",
              "       [ 3.9073346 ],\n",
              "       [ 4.3943133 ],\n",
              "       [ 0.18310434],\n",
              "       [ 2.7177022 ],\n",
              "       [ 3.6416192 ],\n",
              "       [ 2.1679842 ],\n",
              "       [ 2.3597133 ],\n",
              "       [ 3.0752432 ],\n",
              "       [ 1.5172688 ],\n",
              "       [ 1.174509  ],\n",
              "       [ 0.37852105],\n",
              "       [ 2.1929495 ],\n",
              "       [ 4.228909  ],\n",
              "       [ 4.559686  ],\n",
              "       [ 3.8365717 ],\n",
              "       [ 4.148431  ],\n",
              "       [ 3.20731   ],\n",
              "       [ 2.0192108 ],\n",
              "       [ 0.2966166 ],\n",
              "       [ 3.519091  ],\n",
              "       [ 1.6814579 ],\n",
              "       [-0.04897961],\n",
              "       [ 1.7102959 ],\n",
              "       [ 1.1080054 ],\n",
              "       [ 0.22729614],\n",
              "       [ 3.5310814 ],\n",
              "       [ 3.4281673 ],\n",
              "       [ 4.140601  ],\n",
              "       [ 4.001452  ],\n",
              "       [ 1.7691425 ],\n",
              "       [ 3.5871258 ],\n",
              "       [ 1.6324992 ],\n",
              "       [ 0.20808245],\n",
              "       [ 1.3219343 ],\n",
              "       [ 0.40755287],\n",
              "       [ 1.2234914 ],\n",
              "       [ 3.4576962 ],\n",
              "       [ 2.4146316 ],\n",
              "       [ 4.6935496 ],\n",
              "       [ 0.31176525],\n",
              "       [ 4.310481  ],\n",
              "       [ 2.693192  ],\n",
              "       [ 4.543257  ],\n",
              "       [ 3.9032168 ],\n",
              "       [ 1.4019887 ],\n",
              "       [ 0.44532114],\n",
              "       [ 4.6518574 ],\n",
              "       [ 1.2899408 ],\n",
              "       [ 1.8456656 ],\n",
              "       [ 0.3966852 ],\n",
              "       [ 4.163582  ],\n",
              "       [ 4.5776973 ],\n",
              "       [ 3.5645418 ],\n",
              "       [ 4.0132003 ],\n",
              "       [ 0.10647923],\n",
              "       [ 3.451598  ],\n",
              "       [ 2.395111  ],\n",
              "       [ 3.3648493 ],\n",
              "       [ 1.299582  ],\n",
              "       [ 3.2871616 ],\n",
              "       [ 3.448217  ],\n",
              "       [ 2.651068  ],\n",
              "       [ 2.1671617 ],\n",
              "       [ 3.8976626 ],\n",
              "       [ 0.9111488 ],\n",
              "       [ 0.1403773 ],\n",
              "       [ 3.383412  ],\n",
              "       [ 0.5388686 ],\n",
              "       [ 3.6847053 ],\n",
              "       [ 1.1033733 ],\n",
              "       [ 3.456035  ],\n",
              "       [ 4.636573  ],\n",
              "       [ 3.395425  ],\n",
              "       [ 2.4104116 ],\n",
              "       [ 3.9831815 ],\n",
              "       [ 2.9566329 ],\n",
              "       [ 0.16528155],\n",
              "       [ 2.4339788 ],\n",
              "       [ 3.1558437 ],\n",
              "       [ 3.9194663 ],\n",
              "       [ 4.677365  ],\n",
              "       [ 3.336697  ],\n",
              "       [ 0.6556713 ],\n",
              "       [ 4.0224977 ],\n",
              "       [ 2.2790074 ],\n",
              "       [ 0.8215209 ],\n",
              "       [ 1.9827347 ],\n",
              "       [ 3.5305653 ],\n",
              "       [ 1.2613463 ],\n",
              "       [ 2.9183838 ],\n",
              "       [ 2.0391064 ],\n",
              "       [ 0.23022653],\n",
              "       [ 4.063673  ],\n",
              "       [ 4.11095   ],\n",
              "       [ 0.07896303],\n",
              "       [ 3.9302566 ],\n",
              "       [ 4.0846577 ],\n",
              "       [ 0.47951874],\n",
              "       [ 3.887531  ],\n",
              "       [ 0.51483876],\n",
              "       [ 2.0865276 ],\n",
              "       [ 0.43816695],\n",
              "       [ 0.7142692 ],\n",
              "       [ 3.5197287 ],\n",
              "       [ 3.2248156 ],\n",
              "       [ 3.4013207 ],\n",
              "       [ 2.0839465 ],\n",
              "       [ 3.8320153 ],\n",
              "       [ 4.2735972 ],\n",
              "       [ 2.3704278 ],\n",
              "       [ 3.8878992 ],\n",
              "       [ 0.12778871],\n",
              "       [ 0.4048727 ],\n",
              "       [ 0.2977751 ],\n",
              "       [ 3.6820962 ],\n",
              "       [ 1.1286175 ],\n",
              "       [ 1.9439359 ],\n",
              "       [ 3.6338527 ],\n",
              "       [ 0.890256  ],\n",
              "       [ 0.69125724],\n",
              "       [ 2.0262196 ],\n",
              "       [ 1.2580047 ],\n",
              "       [ 0.21177664],\n",
              "       [ 1.6940572 ],\n",
              "       [ 2.2935314 ],\n",
              "       [ 0.7898298 ],\n",
              "       [ 1.1241305 ],\n",
              "       [ 3.6692607 ],\n",
              "       [ 2.2054179 ],\n",
              "       [ 3.900295  ],\n",
              "       [ 3.946892  ],\n",
              "       [ 4.7206497 ],\n",
              "       [ 0.9305352 ],\n",
              "       [ 4.539364  ],\n",
              "       [ 3.0540435 ],\n",
              "       [ 4.0795474 ],\n",
              "       [ 0.07921311],\n",
              "       [ 0.18616247],\n",
              "       [ 0.24777058],\n",
              "       [ 4.298471  ],\n",
              "       [-0.03376787],\n",
              "       [ 3.5857852 ],\n",
              "       [ 1.9419852 ],\n",
              "       [ 3.908834  ],\n",
              "       [ 0.31630912],\n",
              "       [ 0.98120815],\n",
              "       [ 1.5693796 ],\n",
              "       [ 4.021842  ],\n",
              "       [ 4.3568664 ],\n",
              "       [ 4.1106935 ],\n",
              "       [ 3.9977658 ],\n",
              "       [ 3.4193928 ],\n",
              "       [ 3.8946745 ],\n",
              "       [ 2.8417094 ],\n",
              "       [ 4.0375633 ],\n",
              "       [ 0.95989114],\n",
              "       [ 3.0873408 ],\n",
              "       [ 3.615588  ],\n",
              "       [ 0.35349527],\n",
              "       [ 3.7883055 ],\n",
              "       [ 3.3472815 ],\n",
              "       [ 3.4360096 ],\n",
              "       [ 0.12383718],\n",
              "       [ 4.363663  ],\n",
              "       [ 3.1537416 ],\n",
              "       [ 1.1623995 ],\n",
              "       [ 1.215955  ],\n",
              "       [ 4.405833  ],\n",
              "       [ 3.5877364 ],\n",
              "       [ 1.9141349 ],\n",
              "       [ 4.4278116 ],\n",
              "       [ 0.08205288],\n",
              "       [ 0.8548405 ],\n",
              "       [ 0.43251923],\n",
              "       [-0.02494644],\n",
              "       [ 4.3564186 ],\n",
              "       [ 3.6314588 ],\n",
              "       [ 4.627309  ],\n",
              "       [ 2.1043715 ],\n",
              "       [ 2.9552805 ],\n",
              "       [ 4.5528574 ],\n",
              "       [ 3.738149  ],\n",
              "       [ 0.68270224],\n",
              "       [ 0.873018  ],\n",
              "       [ 4.4163446 ],\n",
              "       [ 4.0014124 ],\n",
              "       [ 2.9175487 ],\n",
              "       [ 2.7884905 ],\n",
              "       [ 2.5423543 ],\n",
              "       [ 3.5477211 ],\n",
              "       [ 3.4053445 ],\n",
              "       [ 3.3862998 ],\n",
              "       [ 3.3078868 ],\n",
              "       [ 0.45234412],\n",
              "       [ 0.9727074 ],\n",
              "       [ 3.795596  ],\n",
              "       [ 2.0279367 ],\n",
              "       [ 3.3634784 ],\n",
              "       [ 3.55712   ],\n",
              "       [ 3.544927  ],\n",
              "       [ 4.2532306 ],\n",
              "       [ 2.5687308 ],\n",
              "       [ 3.2223878 ],\n",
              "       [ 1.9887089 ],\n",
              "       [ 0.05676488],\n",
              "       [ 1.5197681 ],\n",
              "       [ 3.511648  ],\n",
              "       [ 3.8037977 ],\n",
              "       [ 1.2542819 ],\n",
              "       [ 3.7902951 ],\n",
              "       [ 0.22594118],\n",
              "       [ 4.364531  ],\n",
              "       [ 3.3966622 ],\n",
              "       [ 1.6597499 ],\n",
              "       [ 4.2790213 ],\n",
              "       [ 3.0084481 ],\n",
              "       [ 4.1227684 ],\n",
              "       [ 1.5206014 ],\n",
              "       [ 3.3351233 ],\n",
              "       [ 3.0215697 ],\n",
              "       [ 2.1846542 ],\n",
              "       [ 3.3647518 ],\n",
              "       [ 3.4003766 ],\n",
              "       [ 3.604226  ],\n",
              "       [ 4.1383104 ],\n",
              "       [ 3.5244415 ],\n",
              "       [ 1.7777723 ],\n",
              "       [ 2.8054812 ],\n",
              "       [ 1.414616  ],\n",
              "       [ 1.8324137 ],\n",
              "       [ 3.3692641 ],\n",
              "       [ 4.4519444 ],\n",
              "       [ 0.6507884 ],\n",
              "       [ 0.75647664],\n",
              "       [ 3.6377225 ],\n",
              "       [ 4.0314674 ],\n",
              "       [ 0.50707114],\n",
              "       [ 4.6423597 ],\n",
              "       [ 3.7277236 ],\n",
              "       [ 3.608195  ],\n",
              "       [ 3.2433953 ],\n",
              "       [ 3.6599824 ],\n",
              "       [ 0.24802725],\n",
              "       [ 0.63324606],\n",
              "       [ 1.3320705 ],\n",
              "       [ 3.6621938 ],\n",
              "       [ 4.2216773 ],\n",
              "       [ 4.133063  ],\n",
              "       [ 1.2353234 ],\n",
              "       [ 3.700397  ],\n",
              "       [ 2.8002713 ],\n",
              "       [ 2.7361064 ],\n",
              "       [ 0.16078079],\n",
              "       [ 4.1934586 ],\n",
              "       [ 2.97218   ],\n",
              "       [ 1.4819127 ],\n",
              "       [ 0.19056201],\n",
              "       [ 1.6070174 ],\n",
              "       [ 3.3415115 ],\n",
              "       [ 3.9225705 ],\n",
              "       [ 3.6131094 ],\n",
              "       [ 3.170947  ],\n",
              "       [ 1.8003072 ],\n",
              "       [ 0.47670743],\n",
              "       [ 3.3480523 ]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predict = [1 if i>=3.0 else 0 for i in predictions[0]]\n",
        "predict"
      ],
      "metadata": {
        "id": "ASKKQ51mkqBz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "encoded_test"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZXJv0R9bkNCx",
        "outputId": "d0d01194-8794-4119-e6f9-0c48892af7aa"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['guid', 'source', 'sentence1', 'sentence2', 'labels.label', 'label', 'labels.binary-label', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
              "    num_rows: 519\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "score = pd.DataFrame({'guid' : encoded_test['guid'],\n",
        "                      'true_real_label' : encoded_test['label'],\n",
        "                      'true_binary_label' : encoded_test['labels.binary-label'],\n",
        "})\n",
        "score"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "riXtQrygjqjF",
        "outputId": "2cca5c55-47fb-4126-ac5d-a848c6f06d79"
      },
      "execution_count": 79,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                      guid  true_real_label  true_binary_label\n",
              "0    klue-sts-v1_dev_00000         4.857143                  1\n",
              "1    klue-sts-v1_dev_00001         1.428571                  0\n",
              "2    klue-sts-v1_dev_00002         1.285714                  0\n",
              "3    klue-sts-v1_dev_00003         3.714286                  1\n",
              "4    klue-sts-v1_dev_00004         2.500000                  0\n",
              "..                     ...              ...                ...\n",
              "514  klue-sts-v1_dev_00514         2.200000                  0\n",
              "515  klue-sts-v1_dev_00515         2.833333                  0\n",
              "516  klue-sts-v1_dev_00516         0.333333                  0\n",
              "517  klue-sts-v1_dev_00517         0.333333                  0\n",
              "518  klue-sts-v1_dev_00518         2.857143                  0\n",
              "\n",
              "[519 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-86515c01-3d61-465e-b426-a81b2bd3f151\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>guid</th>\n",
              "      <th>true_real_label</th>\n",
              "      <th>true_binary_label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>klue-sts-v1_dev_00000</td>\n",
              "      <td>4.857143</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>klue-sts-v1_dev_00001</td>\n",
              "      <td>1.428571</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>klue-sts-v1_dev_00002</td>\n",
              "      <td>1.285714</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>klue-sts-v1_dev_00003</td>\n",
              "      <td>3.714286</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>klue-sts-v1_dev_00004</td>\n",
              "      <td>2.500000</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>514</th>\n",
              "      <td>klue-sts-v1_dev_00514</td>\n",
              "      <td>2.200000</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>515</th>\n",
              "      <td>klue-sts-v1_dev_00515</td>\n",
              "      <td>2.833333</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>516</th>\n",
              "      <td>klue-sts-v1_dev_00516</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>517</th>\n",
              "      <td>klue-sts-v1_dev_00517</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>518</th>\n",
              "      <td>klue-sts-v1_dev_00518</td>\n",
              "      <td>2.857143</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>519 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-86515c01-3d61-465e-b426-a81b2bd3f151')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-86515c01-3d61-465e-b426-a81b2bd3f151 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-86515c01-3d61-465e-b426-a81b2bd3f151');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "score['predict_real_label'] = predictions[0]"
      ],
      "metadata": {
        "id": "JijUDXuxlo0D"
      },
      "execution_count": 80,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "score['predict_binary_label'] = score.predict_real_label.apply(lambda x: 1 if x>=3.0 else 0)"
      ],
      "metadata": {
        "id": "_JNJS9Aolxwz"
      },
      "execution_count": 82,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "score"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "b5M9FgRgmCQF",
        "outputId": "2382e8f7-0b79-4632-d45e-48633622de41"
      },
      "execution_count": 84,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                      guid  true_real_label  true_binary_label  \\\n",
              "0    klue-sts-v1_dev_00000         4.857143                  1   \n",
              "1    klue-sts-v1_dev_00001         1.428571                  0   \n",
              "2    klue-sts-v1_dev_00002         1.285714                  0   \n",
              "3    klue-sts-v1_dev_00003         3.714286                  1   \n",
              "4    klue-sts-v1_dev_00004         2.500000                  0   \n",
              "..                     ...              ...                ...   \n",
              "514  klue-sts-v1_dev_00514         2.200000                  0   \n",
              "515  klue-sts-v1_dev_00515         2.833333                  0   \n",
              "516  klue-sts-v1_dev_00516         0.333333                  0   \n",
              "517  klue-sts-v1_dev_00517         0.333333                  0   \n",
              "518  klue-sts-v1_dev_00518         2.857143                  0   \n",
              "\n",
              "     predict_real_label  predict_binary_label  \n",
              "0              4.653361                     1  \n",
              "1              1.781012                     0  \n",
              "2              1.430372                     0  \n",
              "3              4.100582                     1  \n",
              "4              2.887404                     0  \n",
              "..                  ...                   ...  \n",
              "514            3.613109                     1  \n",
              "515            3.170947                     1  \n",
              "516            1.800307                     0  \n",
              "517            0.476707                     0  \n",
              "518            3.348052                     1  \n",
              "\n",
              "[519 rows x 5 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7e0f702f-4c96-42da-aebc-c0c54dc70e5d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>guid</th>\n",
              "      <th>true_real_label</th>\n",
              "      <th>true_binary_label</th>\n",
              "      <th>predict_real_label</th>\n",
              "      <th>predict_binary_label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>klue-sts-v1_dev_00000</td>\n",
              "      <td>4.857143</td>\n",
              "      <td>1</td>\n",
              "      <td>4.653361</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>klue-sts-v1_dev_00001</td>\n",
              "      <td>1.428571</td>\n",
              "      <td>0</td>\n",
              "      <td>1.781012</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>klue-sts-v1_dev_00002</td>\n",
              "      <td>1.285714</td>\n",
              "      <td>0</td>\n",
              "      <td>1.430372</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>klue-sts-v1_dev_00003</td>\n",
              "      <td>3.714286</td>\n",
              "      <td>1</td>\n",
              "      <td>4.100582</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>klue-sts-v1_dev_00004</td>\n",
              "      <td>2.500000</td>\n",
              "      <td>0</td>\n",
              "      <td>2.887404</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>514</th>\n",
              "      <td>klue-sts-v1_dev_00514</td>\n",
              "      <td>2.200000</td>\n",
              "      <td>0</td>\n",
              "      <td>3.613109</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>515</th>\n",
              "      <td>klue-sts-v1_dev_00515</td>\n",
              "      <td>2.833333</td>\n",
              "      <td>0</td>\n",
              "      <td>3.170947</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>516</th>\n",
              "      <td>klue-sts-v1_dev_00516</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>0</td>\n",
              "      <td>1.800307</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>517</th>\n",
              "      <td>klue-sts-v1_dev_00517</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>0</td>\n",
              "      <td>0.476707</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>518</th>\n",
              "      <td>klue-sts-v1_dev_00518</td>\n",
              "      <td>2.857143</td>\n",
              "      <td>0</td>\n",
              "      <td>3.348052</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>519 rows × 5 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7e0f702f-4c96-42da-aebc-c0c54dc70e5d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7e0f702f-4c96-42da-aebc-c0c54dc70e5d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7e0f702f-4c96-42da-aebc-c0c54dc70e5d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "score.to_csv('RoBERTa_hyper_dev_set_score.csv', index=False)"
      ],
      "metadata": {
        "id": "ng1hZW5ImJqY"
      },
      "execution_count": 85,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "Model_roberta_base.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyOd5SQWlcvNIVCapr1ktoei",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b22751c7d5d942daa1baddc8af50a415": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_21fbf305220b45df961157be2f29e946",
              "IPY_MODEL_ff913db97f864449bc132df0c2b6179a",
              "IPY_MODEL_d34a08431bee454c84ed07254612f7a0"
            ],
            "layout": "IPY_MODEL_54371fe1b88845129e8ff18c5217797c"
          }
        },
        "21fbf305220b45df961157be2f29e946": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_16ae0c5c36184eb79d5b1b7d568e9a27",
            "placeholder": "​",
            "style": "IPY_MODEL_45b834a8731445d2bf7e9175a9c6fc9a",
            "value": "Downloading builder script: "
          }
        },
        "ff913db97f864449bc132df0c2b6179a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b2b27bb69e6641dd848736d5f7eb1af1",
            "max": 5206,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9b55984eec5d401a8b21c80781459edf",
            "value": 5206
          }
        },
        "d34a08431bee454c84ed07254612f7a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9de5f0ad569644c0a959f2863f988b79",
            "placeholder": "​",
            "style": "IPY_MODEL_c8a4a9cb652f47c196e71e888483e469",
            "value": " 23.3k/? [00:00&lt;00:00, 722kB/s]"
          }
        },
        "54371fe1b88845129e8ff18c5217797c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "16ae0c5c36184eb79d5b1b7d568e9a27": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "45b834a8731445d2bf7e9175a9c6fc9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b2b27bb69e6641dd848736d5f7eb1af1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9b55984eec5d401a8b21c80781459edf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9de5f0ad569644c0a959f2863f988b79": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c8a4a9cb652f47c196e71e888483e469": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2f9f3e3fd9d94d34b2a53803991a55a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c6a1a214ad774544a0b3b73d9d6e83e3",
              "IPY_MODEL_5cd6c658e0a14a8bb7cfd2922ec24b9c",
              "IPY_MODEL_2f43bcf8eac94cb09e5dce4e3bd237af"
            ],
            "layout": "IPY_MODEL_b35a973f34a44f12ae340309460dbce2"
          }
        },
        "c6a1a214ad774544a0b3b73d9d6e83e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4ff0ad2e30a448d7b3401ecbe57bc29a",
            "placeholder": "​",
            "style": "IPY_MODEL_329334770ea243eaaab4be8cba3c4354",
            "value": "Downloading metadata: "
          }
        },
        "5cd6c658e0a14a8bb7cfd2922ec24b9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4fed1bd1fcc04c3ba4c7875d5636e7fc",
            "max": 2932,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c7d10acd90134798998a69d230263755",
            "value": 2932
          }
        },
        "2f43bcf8eac94cb09e5dce4e3bd237af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e43e9351853f4d61a4487ae26b60f9a2",
            "placeholder": "​",
            "style": "IPY_MODEL_04fa42a9d0a94ad7ae67cf17672d35b9",
            "value": " 22.7k/? [00:00&lt;00:00, 710kB/s]"
          }
        },
        "b35a973f34a44f12ae340309460dbce2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4ff0ad2e30a448d7b3401ecbe57bc29a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "329334770ea243eaaab4be8cba3c4354": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4fed1bd1fcc04c3ba4c7875d5636e7fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c7d10acd90134798998a69d230263755": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e43e9351853f4d61a4487ae26b60f9a2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "04fa42a9d0a94ad7ae67cf17672d35b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "851d223309af4c9e85e85a8988ef6387": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8ba2194fcebc41beb88f9096956379b7",
              "IPY_MODEL_42f6f09b423445baab9855ae9baff887",
              "IPY_MODEL_0e2708b3dc6c4eb287061a6cf1cde2a7"
            ],
            "layout": "IPY_MODEL_38c3552b98d749798801af39ea52e1e0"
          }
        },
        "8ba2194fcebc41beb88f9096956379b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cff0787095694134bc91d9d55ce6899c",
            "placeholder": "​",
            "style": "IPY_MODEL_3f8c6d0ff201492d9b5e76046f58b899",
            "value": "Downloading data: 100%"
          }
        },
        "42f6f09b423445baab9855ae9baff887": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ba98b1f992ce4c9aa16ec15f9a4a49a9",
            "max": 1349875,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_865bf172aff24c2692183f6131d22b0c",
            "value": 1349875
          }
        },
        "0e2708b3dc6c4eb287061a6cf1cde2a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_48c4d2786e6540a9af96768443d72021",
            "placeholder": "​",
            "style": "IPY_MODEL_d7b5f453212440d79ec512f5b105d478",
            "value": " 1.35M/1.35M [00:00&lt;00:00, 1.74MB/s]"
          }
        },
        "38c3552b98d749798801af39ea52e1e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cff0787095694134bc91d9d55ce6899c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3f8c6d0ff201492d9b5e76046f58b899": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ba98b1f992ce4c9aa16ec15f9a4a49a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "865bf172aff24c2692183f6131d22b0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "48c4d2786e6540a9af96768443d72021": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d7b5f453212440d79ec512f5b105d478": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f38df2abd62143e884a23b2c4397608a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c6eb51c3c83e4de8b080b76bd5548ebe",
              "IPY_MODEL_3f7f71a646c64a48991a0ffe3c2486c1",
              "IPY_MODEL_dfd45244a5fe48f29d01a6f6f5d239ba"
            ],
            "layout": "IPY_MODEL_1e4667fe913846999d65832709965966"
          }
        },
        "c6eb51c3c83e4de8b080b76bd5548ebe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cdbc2596130c43eabdab2ac2560202d6",
            "placeholder": "​",
            "style": "IPY_MODEL_528df8fbfcc343db9448df56f254c8b1",
            "value": "Generating train split:  96%"
          }
        },
        "3f7f71a646c64a48991a0ffe3c2486c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9734c02d56af48db954229207064790f",
            "max": 11668,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7bcd2912ca7744bb9a6ae49058dae326",
            "value": 11668
          }
        },
        "dfd45244a5fe48f29d01a6f6f5d239ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b58cee264d944914ac4b1b8f009bc2db",
            "placeholder": "​",
            "style": "IPY_MODEL_da7abc45193242068c8987ab07ab6109",
            "value": " 11155/11668 [00:01&lt;00:00, 10175.54 examples/s]"
          }
        },
        "1e4667fe913846999d65832709965966": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cdbc2596130c43eabdab2ac2560202d6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "528df8fbfcc343db9448df56f254c8b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9734c02d56af48db954229207064790f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7bcd2912ca7744bb9a6ae49058dae326": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b58cee264d944914ac4b1b8f009bc2db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "da7abc45193242068c8987ab07ab6109": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e0d669251a114be5a79d8c4e5f3d1bfa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_47a08a359b0c40e3b66834625384dc20",
              "IPY_MODEL_4c5f7287296e4c8791b9c94f1bba4dd1",
              "IPY_MODEL_9906da54555a4741b7c91147571f462c"
            ],
            "layout": "IPY_MODEL_63e1578bef1b46f999484690d8238042"
          }
        },
        "47a08a359b0c40e3b66834625384dc20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_da526d19e34b495dad94e38ae9a20317",
            "placeholder": "​",
            "style": "IPY_MODEL_a22bdd913bc8415d953bd45f4b3da9b7",
            "value": "Generating validation split:   9%"
          }
        },
        "4c5f7287296e4c8791b9c94f1bba4dd1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_844730d076194decb6acc0237dde7fc8",
            "max": 519,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b691120249e44fa98e13271ac596a467",
            "value": 519
          }
        },
        "9906da54555a4741b7c91147571f462c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9a4971f6714c4b14ace2fdc896bd481e",
            "placeholder": "​",
            "style": "IPY_MODEL_91a848af4b894bc89e598a70e0180dd2",
            "value": " 48/519 [00:00&lt;00:00, 479.50 examples/s]"
          }
        },
        "63e1578bef1b46f999484690d8238042": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "da526d19e34b495dad94e38ae9a20317": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a22bdd913bc8415d953bd45f4b3da9b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "844730d076194decb6acc0237dde7fc8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b691120249e44fa98e13271ac596a467": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9a4971f6714c4b14ace2fdc896bd481e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "91a848af4b894bc89e598a70e0180dd2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9ed00057998d4cd1846e86e1f6467abb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_dcebd258911448639e8a50457e2d7d1c",
              "IPY_MODEL_b0ad2af7970a494cabddfaf08f06ed02",
              "IPY_MODEL_41803b8153df46a798c0e5a518c34d6b"
            ],
            "layout": "IPY_MODEL_c10e1a6e0f36455986609688df7bbe06"
          }
        },
        "dcebd258911448639e8a50457e2d7d1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e801e09faa1b42a2b4137889925d9335",
            "placeholder": "​",
            "style": "IPY_MODEL_874c2fc7a826442fabb5dac3b07a0a75",
            "value": "100%"
          }
        },
        "b0ad2af7970a494cabddfaf08f06ed02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1b87ec51e122403c8940af5b71b8c5de",
            "max": 11,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_37cd84313e9643efa907e69d62d65c91",
            "value": 11
          }
        },
        "41803b8153df46a798c0e5a518c34d6b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_038bf42c4ae64b1b9e3d2c05c22f4894",
            "placeholder": "​",
            "style": "IPY_MODEL_c16d8a5b79bc4e33a77ed13bab6b5020",
            "value": " 11/11 [00:01&lt;00:00,  8.60ba/s]"
          }
        },
        "c10e1a6e0f36455986609688df7bbe06": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e801e09faa1b42a2b4137889925d9335": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "874c2fc7a826442fabb5dac3b07a0a75": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1b87ec51e122403c8940af5b71b8c5de": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "37cd84313e9643efa907e69d62d65c91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "038bf42c4ae64b1b9e3d2c05c22f4894": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c16d8a5b79bc4e33a77ed13bab6b5020": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5c653abc118242ba925ed53ab6f07325": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5b46e7b42b6e4a92b8a703bd674ed944",
              "IPY_MODEL_94b6c95ace144b20931a1f0d4888a65e",
              "IPY_MODEL_a71a4c274a5f4453818abb24b429a5a7"
            ],
            "layout": "IPY_MODEL_73554ceb4be34ef788898c7eebdec8dc"
          }
        },
        "5b46e7b42b6e4a92b8a703bd674ed944": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_56480527af3644dba1c090de3dbe3aef",
            "placeholder": "​",
            "style": "IPY_MODEL_12bbd177fe164fa0a3681f5dfcb1540c",
            "value": "100%"
          }
        },
        "94b6c95ace144b20931a1f0d4888a65e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_668bf290194b4e82b7a792f5afc4302a",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f647bee7447c4601ab5c5ae93cfd7e0d",
            "value": 2
          }
        },
        "a71a4c274a5f4453818abb24b429a5a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9d900548ddb6411b9a4d1ad5188b63f8",
            "placeholder": "​",
            "style": "IPY_MODEL_ce6417c26e204af6ae4a3539108a5ceb",
            "value": " 2/2 [00:00&lt;00:00,  6.65ba/s]"
          }
        },
        "73554ceb4be34ef788898c7eebdec8dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "56480527af3644dba1c090de3dbe3aef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "12bbd177fe164fa0a3681f5dfcb1540c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "668bf290194b4e82b7a792f5afc4302a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f647bee7447c4601ab5c5ae93cfd7e0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9d900548ddb6411b9a4d1ad5188b63f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce6417c26e204af6ae4a3539108a5ceb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "55f104a2dceb419dbcadd4e46515c2a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_611f40b8e5f240f880106bf5af08d092",
              "IPY_MODEL_ae1b15a0b2b04d828e00a8778527e13e",
              "IPY_MODEL_c8a38212dedc4afaa22699c6d6f295d9"
            ],
            "layout": "IPY_MODEL_09c06550e6e2494db2a8c6b04bd6c3d6"
          }
        },
        "611f40b8e5f240f880106bf5af08d092": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_523ba8c6193743abb8499655ec4a74f1",
            "placeholder": "​",
            "style": "IPY_MODEL_3067407fb5194e3aa4de556d82f2586d",
            "value": "100%"
          }
        },
        "ae1b15a0b2b04d828e00a8778527e13e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c6f51004e26e4f9bb6b429140d523234",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ed288df2424642ef98bd3139bfe0e14b",
            "value": 1
          }
        },
        "c8a38212dedc4afaa22699c6d6f295d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_213cd178f01a439689bf513487778bcf",
            "placeholder": "​",
            "style": "IPY_MODEL_df2b8316d57349abb97246e7b851d7ef",
            "value": " 1/1 [00:00&lt;00:00,  9.65ba/s]"
          }
        },
        "09c06550e6e2494db2a8c6b04bd6c3d6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "523ba8c6193743abb8499655ec4a74f1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3067407fb5194e3aa4de556d82f2586d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c6f51004e26e4f9bb6b429140d523234": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ed288df2424642ef98bd3139bfe0e14b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "213cd178f01a439689bf513487778bcf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "df2b8316d57349abb97246e7b851d7ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "43b4d7715b87477d806fe3193988e0ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_02dbd9d85d6942f295a736144c0b306f",
              "IPY_MODEL_f1297d192c234af5805b9eabb049919f",
              "IPY_MODEL_d9a65770255547e99601a9a9f6995a39"
            ],
            "layout": "IPY_MODEL_8cfed01f4c2549d0a599102951634551"
          }
        },
        "02dbd9d85d6942f295a736144c0b306f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_33ebb42888004ee0b37920608e763f0a",
            "placeholder": "​",
            "style": "IPY_MODEL_be156d2353c847e5a49e7994bb9936fe",
            "value": "Downloading: 100%"
          }
        },
        "f1297d192c234af5805b9eabb049919f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c2518b3dc2314bac805c3c5d41923306",
            "max": 546,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3282a93542964a69904f02341a7dfe1e",
            "value": 546
          }
        },
        "d9a65770255547e99601a9a9f6995a39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e0af530b11d44a85be40a5dab3615435",
            "placeholder": "​",
            "style": "IPY_MODEL_3c8131c1104845b887c40d0009a11b7b",
            "value": " 546/546 [00:00&lt;00:00, 17.5kB/s]"
          }
        },
        "8cfed01f4c2549d0a599102951634551": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "33ebb42888004ee0b37920608e763f0a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "be156d2353c847e5a49e7994bb9936fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c2518b3dc2314bac805c3c5d41923306": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3282a93542964a69904f02341a7dfe1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e0af530b11d44a85be40a5dab3615435": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3c8131c1104845b887c40d0009a11b7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e977b3566d18490680504fd253bc5b9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3293fb599a70432bb9d78ec3098ec40e",
              "IPY_MODEL_065901872f554a6aa1dfc0c0ff1d07f4",
              "IPY_MODEL_1e24fdd83fc74de1977bc23204e7a24b"
            ],
            "layout": "IPY_MODEL_89549163a8464a05b2e272d417261bef"
          }
        },
        "3293fb599a70432bb9d78ec3098ec40e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9faebf2a37f64a3390b671f59a6dbcae",
            "placeholder": "​",
            "style": "IPY_MODEL_0d5fd240ef044aad99cadb281eaf5a8c",
            "value": "Downloading: 100%"
          }
        },
        "065901872f554a6aa1dfc0c0ff1d07f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_24ec8e0fdad144cf997decdd06195a3a",
            "max": 442694866,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_503df178368d4ed994da85e53a4b3785",
            "value": 442694866
          }
        },
        "1e24fdd83fc74de1977bc23204e7a24b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_35688c8cedaa4080a8ffc84bf8c4eee4",
            "placeholder": "​",
            "style": "IPY_MODEL_46b2155e79a844caafce448e4690f0d2",
            "value": " 422M/422M [00:08&lt;00:00, 56.8MB/s]"
          }
        },
        "89549163a8464a05b2e272d417261bef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9faebf2a37f64a3390b671f59a6dbcae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0d5fd240ef044aad99cadb281eaf5a8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "24ec8e0fdad144cf997decdd06195a3a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "503df178368d4ed994da85e53a4b3785": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "35688c8cedaa4080a8ffc84bf8c4eee4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "46b2155e79a844caafce448e4690f0d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7fe1e70e19c041778b31de17182717e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8a59958566d6426aa08e5034e127eb1a",
              "IPY_MODEL_6710b7603b3d49aa9f1886a757773738",
              "IPY_MODEL_be23d4aadec24875a27ab2df73d64a7c"
            ],
            "layout": "IPY_MODEL_13c4a05d05834d20bd7f7c47d7f69b72"
          }
        },
        "8a59958566d6426aa08e5034e127eb1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_42a21255ef8c41299930f8ff35111bc9",
            "placeholder": "​",
            "style": "IPY_MODEL_0ecd1e110314465bad1b3168acdc7eb6",
            "value": "Downloading builder script: "
          }
        },
        "6710b7603b3d49aa9f1886a757773738": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_07c05fdc87b54bae996b8df5b75a9d3a",
            "max": 2318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_53d7458317674c249290f8130ea11e26",
            "value": 2318
          }
        },
        "be23d4aadec24875a27ab2df73d64a7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8527577021194ba3ac81a7920a0961b9",
            "placeholder": "​",
            "style": "IPY_MODEL_b4ef8df2770e49e19e69837dc0c72ed9",
            "value": " 6.50k/? [00:00&lt;00:00, 77.6kB/s]"
          }
        },
        "13c4a05d05834d20bd7f7c47d7f69b72": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "42a21255ef8c41299930f8ff35111bc9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0ecd1e110314465bad1b3168acdc7eb6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "07c05fdc87b54bae996b8df5b75a9d3a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "53d7458317674c249290f8130ea11e26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8527577021194ba3ac81a7920a0961b9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b4ef8df2770e49e19e69837dc0c72ed9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bb8ec8afbf214ec291f5b2211a56d674": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_60b064f41d1f44fc9ae00d8d243688d6",
              "IPY_MODEL_b087e9b218714c8ebcb52bf1e2be4cb9",
              "IPY_MODEL_aa8c46c738c2402b90112dfc0a3d46ba"
            ],
            "layout": "IPY_MODEL_1a591ac5a365427c88da1fd86b1bc140"
          }
        },
        "60b064f41d1f44fc9ae00d8d243688d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_024915eaac2844d8a38524030661d38b",
            "placeholder": "​",
            "style": "IPY_MODEL_50d6e0e505df45a7b82564e63289f9f8",
            "value": "Downloading builder script: "
          }
        },
        "b087e9b218714c8ebcb52bf1e2be4cb9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_496fbbb37e724af08a0df51fc310a6f9",
            "max": 2150,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2b12b9a4f7b34f1cab0d0573aa6b34f4",
            "value": 2150
          }
        },
        "aa8c46c738c2402b90112dfc0a3d46ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dae939a41ac449adbd7f1111b960124e",
            "placeholder": "​",
            "style": "IPY_MODEL_1937f00cb72149f499adb87f2f8ba4dd",
            "value": " 5.29k/? [00:00&lt;00:00, 145kB/s]"
          }
        },
        "1a591ac5a365427c88da1fd86b1bc140": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "024915eaac2844d8a38524030661d38b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "50d6e0e505df45a7b82564e63289f9f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "496fbbb37e724af08a0df51fc310a6f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2b12b9a4f7b34f1cab0d0573aa6b34f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "dae939a41ac449adbd7f1111b960124e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1937f00cb72149f499adb87f2f8ba4dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}