01. cross_attention_bahdanau_attention_implementation.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import re\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.layers import Embedding, LSTM, Dense\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences"
      ],
      "metadata": {
        "id": "nKBx5LFGnXHp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2Ej64XodTO85"
      },
      "outputs": [],
      "source": [
        "# downloading data\n",
        "!wget http://www.manythings.org/anki/ita-eng.zip\n",
        "!unzip ita-eng.zip"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Processing Data"
      ],
      "metadata": {
        "id": "idCMIoXenlLJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# reading data\n",
        "with open('ita.txt', 'r', encoding=\"utf8\") as f:\n",
        "    print(f.readlines(500))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TBxjssuAuxoO",
        "outputId": "fb3081cc-504f-47e7-f66a-74d1da3151e1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['Hi.\\tCiao!\\tCC-BY 2.0 (France) Attribution: tatoeba.org #538123 (CM) & #607364 (Cero)\\n', 'Hi.\\tCiao.\\tCC-BY 2.0 (France) Attribution: tatoeba.org #538123 (CM) & #4522287 (Guybrush88)\\n', 'Run!\\tCorri!\\tCC-BY 2.0 (France) Attribution: tatoeba.org #906328 (papabear) & #906347 (Guybrush88)\\n', 'Run!\\tCorra!\\tCC-BY 2.0 (France) Attribution: tatoeba.org #906328 (papabear) & #906348 (Guybrush88)\\n', 'Run!\\tCorrete!\\tCC-BY 2.0 (France) Attribution: tatoeba.org #906328 (papabear) & #906350 (Guybrush88)\\n', 'Who?\\tChi?\\tCC-BY 2.0 (France) Attribution: tatoeba.org #2083030 (CK) & #2126402 (Guybrush88)\\n']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# creating dataframe with english , italian sentences\n",
        "with open('ita.txt', 'r', encoding=\"utf8\") as f:\n",
        "    eng=[]\n",
        "    ita=[]\n",
        "    for i in f.readlines():\n",
        "      a = i.split(\"\\t\")\n",
        "      eng.append(a[0])\n",
        "      ita.append(a[1])\n",
        "data = pd.DataFrame()\n",
        "data['english'] = eng\n",
        "data['italian'] = ita\n",
        "data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "u28_4fTXvbJw",
        "outputId": "adabd71a-972d-413f-812f-a434c65cfc9a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  english  \\\n",
              "0                                                     Hi.   \n",
              "1                                                     Hi.   \n",
              "2                                                    Run!   \n",
              "3                                                    Run!   \n",
              "4                                                    Run!   \n",
              "...                                                   ...   \n",
              "362856  I know that adding sentences only in your nati...   \n",
              "362857  I know that adding sentences only in your nati...   \n",
              "362858  I know that adding sentences only in your nati...   \n",
              "362859  Doubtless there exists in this world precisely...   \n",
              "362860  Doubtless there exists in this world precisely...   \n",
              "\n",
              "                                                  italian  \n",
              "0                                                   Ciao!  \n",
              "1                                                   Ciao.  \n",
              "2                                                  Corri!  \n",
              "3                                                  Corra!  \n",
              "4                                                Correte!  \n",
              "...                                                   ...  \n",
              "362856  So che aggiungere frasi soltanto nella sua lin...  \n",
              "362857  So che aggiungere frasi solamente nella sua li...  \n",
              "362858  So che aggiungere frasi solamente nella sua li...  \n",
              "362859  Senza dubbio esiste in questo mondo proprio la...  \n",
              "362860  Senza dubbio esiste in questo mondo proprio la...  \n",
              "\n",
              "[362861 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a87f3148-a911-4be2-80d5-542b2dca9fc8\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>english</th>\n",
              "      <th>italian</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Hi.</td>\n",
              "      <td>Ciao!</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Hi.</td>\n",
              "      <td>Ciao.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Run!</td>\n",
              "      <td>Corri!</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Run!</td>\n",
              "      <td>Corra!</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Run!</td>\n",
              "      <td>Correte!</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>362856</th>\n",
              "      <td>I know that adding sentences only in your nati...</td>\n",
              "      <td>So che aggiungere frasi soltanto nella sua lin...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>362857</th>\n",
              "      <td>I know that adding sentences only in your nati...</td>\n",
              "      <td>So che aggiungere frasi solamente nella sua li...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>362858</th>\n",
              "      <td>I know that adding sentences only in your nati...</td>\n",
              "      <td>So che aggiungere frasi solamente nella sua li...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>362859</th>\n",
              "      <td>Doubtless there exists in this world precisely...</td>\n",
              "      <td>Senza dubbio esiste in questo mondo proprio la...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>362860</th>\n",
              "      <td>Doubtless there exists in this world precisely...</td>\n",
              "      <td>Senza dubbio esiste in questo mondo proprio la...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>362861 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a87f3148-a911-4be2-80d5-542b2dca9fc8')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a87f3148-a911-4be2-80d5-542b2dca9fc8 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a87f3148-a911-4be2-80d5-542b2dca9fc8');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 90
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XnDyE0ljbYor"
      },
      "outputs": [],
      "source": [
        "def decontractions(phrase): #https://stackoverflow.com/questions/19790188/expanding-english-language-contractions-in-python/47091490#47091490\n",
        "    phrase = re.sub(r\"won\\'t\", \"will not\", phrase)\n",
        "    phrase = re.sub(r\"can\\'t\", \"can not\", phrase)\n",
        "    phrase = re.sub(r\"won\\’t\", \"will not\", phrase)\n",
        "    phrase = re.sub(r\"can\\’t\", \"can not\", phrase)\n",
        "    phrase = re.sub(r\"n\\'t\", \" not\", phrase)\n",
        "    phrase = re.sub(r\"\\'re\", \" are\", phrase)\n",
        "    phrase = re.sub(r\"\\'s\", \" is\", phrase)\n",
        "    phrase = re.sub(r\"\\'d\", \" would\", phrase)\n",
        "    phrase = re.sub(r\"\\'ll\", \" will\", phrase)\n",
        "    phrase = re.sub(r\"\\'t\", \" not\", phrase)\n",
        "    phrase = re.sub(r\"\\'ve\", \" have\", phrase)\n",
        "    phrase = re.sub(r\"\\'m\", \" am\", phrase)\n",
        "    phrase = re.sub(r\"n\\’t\", \" not\", phrase)\n",
        "    phrase = re.sub(r\"\\’re\", \" are\", phrase)\n",
        "    phrase = re.sub(r\"\\’s\", \" is\", phrase)\n",
        "    phrase = re.sub(r\"\\’d\", \" would\", phrase)\n",
        "    phrase = re.sub(r\"\\’ll\", \" will\", phrase)\n",
        "    phrase = re.sub(r\"\\’t\", \" not\", phrase)\n",
        "    phrase = re.sub(r\"\\’ve\", \" have\", phrase)\n",
        "    phrase = re.sub(r\"\\’m\", \" am\", phrase)\n",
        "    return phrase"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def preprocess_eng(text):\n",
        "    text = decontractions(text)\n",
        "    text = re.sub('[^A-Za-z0-9 ]+', '', text)\n",
        "    text = text.lower()\n",
        "    return text"
      ],
      "metadata": {
        "id": "7ro9frBoxUZU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def preprocess_ita(text):\n",
        "    text = decontractions(text)\n",
        "    text = re.sub('[$)\\?\"’.°!;\\'€%:,(/]', '', text)\n",
        "    text = re.sub('\\u200b', ' ', text)\n",
        "    text = re.sub('\\xa0', ' ', text)\n",
        "    text = re.sub('-', ' ', text)\n",
        "    text = text.lower()\n",
        "    return text"
      ],
      "metadata": {
        "id": "QU027YfrxVN2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data['english'] = data['english'].apply(preprocess_eng)\n",
        "data['italian'] = data['italian'].apply(preprocess_ita)\n",
        "data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "aDwh-dptxZAb",
        "outputId": "71376d55-eec1-4cb1-deea-1dde8d23213a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  english  italian\n",
              "0      hi     ciao\n",
              "1      hi     ciao\n",
              "2     run    corri\n",
              "3     run    corra\n",
              "4     run  correte"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6c69f66d-7c51-44de-8c30-616ebb7d7f09\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>english</th>\n",
              "      <th>italian</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>hi</td>\n",
              "      <td>ciao</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>hi</td>\n",
              "      <td>ciao</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>run</td>\n",
              "      <td>corri</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>run</td>\n",
              "      <td>corra</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>run</td>\n",
              "      <td>correte</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6c69f66d-7c51-44de-8c30-616ebb7d7f09')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6c69f66d-7c51-44de-8c30-616ebb7d7f09 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6c69f66d-7c51-44de-8c30-616ebb7d7f09');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# getting length of sentences to select max_length\n",
        "def length_text(data):\n",
        "  len_data = []\n",
        "  for i in data:\n",
        "    len_data.append(len(i.split()))\n",
        "  return len_data"
      ],
      "metadata": {
        "id": "v6fTplFoy_Tf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ita_lengths = length_text(data['italian'].values)\n",
        "eng_lengths = length_text(data['english'].values)"
      ],
      "metadata": {
        "id": "IRFw5JSbzV7v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "np.quantile(ita_lengths , [0,0.25,0.5,0.75,0.9,0.95,0.99])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Fm9tekaxzhPO",
        "outputId": "7c6061f1-9799-4684-8064-b321b6580ab5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 1.,  4.,  5.,  7.,  8.,  9., 12.])"
            ]
          },
          "metadata": {},
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "np.quantile(eng_lengths , [0,0.25,0.5,0.75,0.9,0.95,0.99])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gi4hK0Vp0KdG",
        "outputId": "aa932b56-5979-4298-b57a-a47b558d1467"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 1.,  4.,  6.,  7.,  8.,  9., 12.])"
            ]
          },
          "metadata": {},
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "np.quantile(ita_lengths , np.arange(.99,1,0.001)) # getting percentile value between 0.99 to 1.0"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QQbR-wQ60ap8",
        "outputId": "95c91e31-be8b-4a63-fb28-c3833ed91fe1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([12., 12., 12., 13., 13., 13., 14., 15., 16., 22., 92.])"
            ]
          },
          "metadata": {},
          "execution_count": 67
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "np.quantile(eng_lengths , np.arange(.99,1,0.001))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6acz0qOg0jtQ",
        "outputId": "b779d7e2-08cc-421f-f8d7-753f34e19586"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 12.,  12.,  13.,  13.,  13.,  14.,  14.,  15.,  16.,  25., 101.])"
            ]
          },
          "metadata": {},
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IypVGvYKREOL",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "ita_lengths = data['italian'].str.split().apply(len)\n",
        "eng_lengths = data['english'].str.split().apply(len)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "frL1wvLwSz7_"
      },
      "outputs": [],
      "source": [
        "# selecting maximum input length of 20 words\n",
        "\n",
        "data['italian_len'] = data['italian'].str.split().apply(len)\n",
        "data = data[data['italian_len'] < 20]\n",
        "\n",
        "data['english_len'] = data['english'].str.split().apply(len)\n",
        "data = data[data['english_len'] < 20]\n",
        "\n",
        "data['english_inp'] = '<start> ' + data['english'].astype(str) # decoder input starts with <start> token\n",
        "data['english_out'] = data['english'].astype(str) + ' <end>'   # decoder output ends with <end> token\n",
        "\n",
        "data = data.drop(['english','italian_len','english_len'], axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(data.english_inp.values[15000])\n",
        "print(data.english_out.values[15000])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YX8QRj7zpj1p",
        "outputId": "75520544-4fc2-4772-8533-2ba05194f935"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<start> let tom drive\n",
            "let tom drive <end>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "svobsg_yXbux",
        "outputId": "6171ca36-eff7-4df8-c4e7-1118b9d31391"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                            italian                             english_inp  \\\n",
              "99788          tom sembra devastato            <start> tom looks devastated   \n",
              "59360    aggiungete una spiegazione              <start> add an explanation   \n",
              "250904  avevo tutto sotto controllo  <start> i had everything under control   \n",
              "12678                   fu lapidato                   <start> he was stoned   \n",
              "123290      io non ve lo posso dare        <start> i can not give it to you   \n",
              "\n",
              "                                 english_out  \n",
              "99788             tom looks devastated <end>  \n",
              "59360               add an explanation <end>  \n",
              "250904  i had everything under control <end>  \n",
              "12678                    he was stoned <end>  \n",
              "123290        i can not give it to you <end>  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b680214f-c05a-490e-86b8-5f9ea8d4c9fa\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>italian</th>\n",
              "      <th>english_inp</th>\n",
              "      <th>english_out</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>99788</th>\n",
              "      <td>tom sembra devastato</td>\n",
              "      <td>&lt;start&gt; tom looks devastated</td>\n",
              "      <td>tom looks devastated &lt;end&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59360</th>\n",
              "      <td>aggiungete una spiegazione</td>\n",
              "      <td>&lt;start&gt; add an explanation</td>\n",
              "      <td>add an explanation &lt;end&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>250904</th>\n",
              "      <td>avevo tutto sotto controllo</td>\n",
              "      <td>&lt;start&gt; i had everything under control</td>\n",
              "      <td>i had everything under control &lt;end&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12678</th>\n",
              "      <td>fu lapidato</td>\n",
              "      <td>&lt;start&gt; he was stoned</td>\n",
              "      <td>he was stoned &lt;end&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>123290</th>\n",
              "      <td>io non ve lo posso dare</td>\n",
              "      <td>&lt;start&gt; i can not give it to you</td>\n",
              "      <td>i can not give it to you &lt;end&gt;</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b680214f-c05a-490e-86b8-5f9ea8d4c9fa')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b680214f-c05a-490e-86b8-5f9ea8d4c9fa button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b680214f-c05a-490e-86b8-5f9ea8d4c9fa');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 101
        }
      ],
      "source": [
        "data.sample(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UG50P52vhMu8"
      },
      "outputs": [],
      "source": [
        "# train test split\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "trainval, test = train_test_split(data, test_size=0.003) # for test set getting appox. 1000 sentences\n",
        "train, val = train_test_split(trainval, test_size=0.15)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9mp9dzm1hwv-",
        "outputId": "3d442a19-5798-433c-a129-727cd15038b0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(307077, 3) (54191, 3) (1088, 3)\n"
          ]
        }
      ],
      "source": [
        "print(train.shape, val.shape, test.shape)\n",
        "train.iloc[0]['english_inp']= str(train.iloc[0]['english_inp'])+' <end>'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jAMqqi8GaGvq"
      },
      "outputs": [],
      "source": [
        "# tokenizing\n",
        "\n",
        "tknizer_ita = Tokenizer()\n",
        "tknizer_ita.fit_on_texts(train['italian'].values)\n",
        "tknizer_eng = Tokenizer(filters='!\"#$%&()*+,-./:;=?@[\\\\]^_`{|}~\\t\\n')\n",
        "tknizer_eng.fit_on_texts(train['english_inp'].values)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L1AC6vShbsyf",
        "outputId": "092a746f-6c06-4260-b904-9c52890c3b22"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13335\n",
            "27402\n"
          ]
        }
      ],
      "source": [
        "vocab_size_eng=len(tknizer_eng.word_index.keys())\n",
        "print(vocab_size_eng)\n",
        "vocab_size_ita=len(tknizer_ita.word_index.keys())\n",
        "print(vocab_size_ita)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "poK5koaDb7KO",
        "outputId": "f556b762-07bd-48d6-ec7c-34979f4b14be"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1, 10668)"
            ]
          },
          "metadata": {},
          "execution_count": 106
        }
      ],
      "source": [
        "tknizer_eng.word_index['<start>'], tknizer_eng.word_index['<end>']"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://www.dropbox.com/s/ddkmtqz01jc024u/glove.6B.100d.txt"
      ],
      "metadata": {
        "id": "r0gAaYHDniUu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4I8sqk4nefgN"
      },
      "outputs": [],
      "source": [
        "# embedding matrix using glove 100d embeddings\n",
        "\n",
        "embeddings_index = dict()\n",
        "f = open('glove.6B.100d.txt')\n",
        "for line in f:\n",
        "    values = line.split()\n",
        "    word = values[0]\n",
        "    coefs = np.asarray(values[1:], dtype='float32')\n",
        "    embeddings_index[word] = coefs\n",
        "f.close()\n",
        "\n",
        "embedding_matrix = np.zeros((vocab_size_eng+1, 100))\n",
        "for word, i in tknizer_eng.word_index.items():\n",
        "    embedding_vector = embeddings_index.get(word)\n",
        "    if embedding_vector is not None:\n",
        "        embedding_matrix[i] = embedding_vector"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_matrix.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8BNmkA74APKU",
        "outputId": "149eeaa3-6b9e-41ed-f3f4-b22d9726819d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(13336, 100)"
            ]
          },
          "metadata": {},
          "execution_count": 108
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Dataset Loader:"
      ],
      "metadata": {
        "id": "tJDATf05rIFX"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7mBXRd_sus3C"
      },
      "outputs": [],
      "source": [
        "class Dataset:\n",
        "    def __init__(self, data, tknizer_ita, tknizer_eng, max_len):\n",
        "        self.encoder_inps = data['italian'].values\n",
        "        self.decoder_inps = data['english_inp'].values\n",
        "        self.decoder_outs = data['english_out'].values\n",
        "        self.tknizer_eng = tknizer_eng\n",
        "        self.tknizer_ita = tknizer_ita\n",
        "        self.max_len = max_len\n",
        "\n",
        "    def __getitem__(self, i):\n",
        "        self.encoder_seq = self.tknizer_ita.texts_to_sequences([self.encoder_inps[i]])  # tokenizing input italian sentence\n",
        "        self.decoder_inp_seq = self.tknizer_eng.texts_to_sequences([self.decoder_inps[i]]) # tokenizing decoder input \n",
        "        self.decoder_out_seq = self.tknizer_eng.texts_to_sequences([self.decoder_outs[i]]) # tokenizing decoder output\n",
        "\n",
        "        self.encoder_seq = pad_sequences(self.encoder_seq, maxlen=self.max_len, dtype='int32', padding='post')\n",
        "        self.decoder_inp_seq = pad_sequences(self.decoder_inp_seq, maxlen=self.max_len, dtype='int32', padding='post')\n",
        "        self.decoder_out_seq = pad_sequences(self.decoder_out_seq, maxlen=self.max_len, dtype='int32', padding='post')\n",
        "        return self.encoder_seq, self.decoder_inp_seq, self.decoder_out_seq\n",
        "\n",
        "    def __len__(self): \n",
        "        return len(self.encoder_inps)\n",
        "\n",
        "    \n",
        "class Dataloder(tf.keras.utils.Sequence):    \n",
        "    def __init__(self, dataset, batch_size=1):\n",
        "        self.dataset = dataset\n",
        "        self.batch_size = batch_size\n",
        "        self.indexes = np.arange(len(self.dataset.encoder_inps))\n",
        "\n",
        "\n",
        "    def __getitem__(self, i):\n",
        "        start = i * self.batch_size\n",
        "        stop = (i + 1) * self.batch_size\n",
        "        data = []\n",
        "        for j in range(start, stop):\n",
        "            data.append(self.dataset[j])\n",
        "\n",
        "        batch = [np.squeeze(np.stack(samples, axis=1), axis=0) for samples in zip(*data)]\n",
        "        \n",
        "        return tuple([[batch[0],batch[1]],batch[2]]) # input is made of italian sent for encoder and eng. input for decoder, output is decoder output eng sent\n",
        "\n",
        "    def __len__(self): \n",
        "        return len(self.indexes) // self.batch_size\n",
        "\n",
        "    def on_epoch_end(self):\n",
        "        self.indexes = np.random.permutation(self.indexes)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Building Model:"
      ],
      "metadata": {
        "id": "LNhofLP204vr"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "![seq2seq_attention_mechanism_new.svg](data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4bWxuczpsdWNpZD0ibHVjaWQiIHdpZHRoPSIxMzAwIiBoZWlnaHQ9IjEwMzQiPjxnIHRyYW5zZm9ybT0idHJhbnNsYXRlKDAgLTMyMCkiIGx1Y2lkOnBhZ2UtdGFiLWlkPSIwXzAiPjxwYXRoIGQ9Ik0xMDYwLjQgMTA4MGMwLTExLjA1IDguOTUtMjAgMjAtMjBoMTQuNGMxMS4wNSAwIDIwIDguOTUgMjAgMjB2MjM0YzAgMTEuMDUtOC45NSAyMC0yMCAyMGgtMTQuNGMtMTEuMDUgMC0yMC04Ljk1LTIwLTIweiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utb3BhY2l0eT0iMCIgc3Ryb2tlLXdpZHRoPSIuNSIgZmlsbD0iI2QxYmNkMiIgZmlsbC1vcGFjaXR5PSIuMyIvPjxwYXRoIGQ9Ik02MCAzNjBjMC0xMS4wNSA4Ljk1LTIwIDIwLTIwaDU4MGMxMS4wNSAwIDIwIDguOTUgMjAgMjB2NDYwYzAgMTEuMDUtOC45NSAyMC0yMCAyMEg4MGMtMTEuMDUgMC0yMC04Ljk1LTIwLTIweiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utb3BhY2l0eT0iMCIgc3Ryb2tlLXdpZHRoPSIuNSIgZmlsbD0iI2Y1YjVjOCIgZmlsbC1vcGFjaXR5PSIuMyIvPjxwYXRoIGQ9Ik02MCA4ODFjMC0xMS4wNSA4Ljk1LTIwIDIwLTIwaDU4MGMxMS4wNSAwIDIwIDguOTUgMjAgMjB2MTE4YzAgMTEuMDUtOC45NSAyMC0yMCAyMEg4MGMtMTEuMDUgMC0yMC04Ljk1LTIwLTIweiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utb3BhY2l0eT0iMCIgc3Ryb2tlLXdpZHRoPSIuNSIgZmlsbD0iI2QxYmNkMiIgZmlsbC1vcGFjaXR5PSIuMyIvPjxwYXRoIGQ9Ik0xMDAgODk2YzAtOC4yOCA2LjcyLTE1IDE1LTE1aDcwYzguMjggMCAxNSA2LjcyIDE1IDE1djUwYzAgOC4yOC02LjcyIDE1LTE1IDE1aC03MGMtOC4yOCAwLTE1LTYuNzItMTUtMTV6IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0iI2ZmZiIgZmlsbC1vcGFjaXR5PSIuNDgiLz48dXNlIHhsaW5rOmhyZWY9IiNhIiB0cmFuc2Zvcm09Im1hdHJpeCgxLDAsMCwxLDEwMCw4ODEuMDAwMDAwMDAwMDAwMSkgdHJhbnNsYXRlKDE3Ljk5NDc5MTY2NjY2NjY2NCA0Ny45OTQ3OTE2NjY2NjY2NjQpIi8+PHBhdGggZD0iTTMyMCA4OTZjMC04LjI4IDYuNzItMTUgMTUtMTVoNzBjOC4yOCAwIDE1IDYuNzIgMTUgMTV2NTBjMCA4LjI4LTYuNzIgMTUtMTUgMTVoLTcwYy04LjI4IDAtMTUtNi43Mi0xNS0xNXoiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSIjZmZmIiBmaWxsLW9wYWNpdHk9Ii40OCIvPjx1c2UgeGxpbms6aHJlZj0iI2EiIHRyYW5zZm9ybT0ibWF0cml4KDEsMCwwLDEsMzE5Ljk5OTk5OTk5OTk5OTk0LDg4MS4wMDAwMDAwMDAwMDAxKSB0cmFuc2xhdGUoMTcuOTk0NzkxNjY2NjY2NjY0IDQ3Ljk5NDc5MTY2NjY2NjY2NCkiLz48cGF0aCBkPSJNNTQwIDg5NmMwLTguMjggNi43Mi0xNSAxNS0xNWg3MGM4LjI4IDAgMTUgNi43MiAxNSAxNXY1MGMwIDguMjgtNi43MiAxNS0xNSAxNWgtNzBjLTguMjggMC0xNS02LjcyLTE1LTE1eiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9IiNmZmYiIGZpbGwtb3BhY2l0eT0iLjQ4Ii8+PHVzZSB4bGluazpocmVmPSIjYSIgdHJhbnNmb3JtPSJtYXRyaXgoMSwwLDAsMSw1NDAsODgxLjAwMDAwMDAwMDAwMDEpIHRyYW5zbGF0ZSgxNy45OTQ3OTE2NjY2NjY2NjQgNDcuOTk0NzkxNjY2NjY2NjY0KSIvPjxwYXRoIGQ9Ik0yMDAgOTIxaDExOC41IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0ibm9uZSIvPjxwYXRoIGQ9Ik0yMDAgOTIyLjV2LTN6TTMxOC41IDkyMWwtMTQuMjcgNC42NHYtOS4yOHoiLz48cGF0aCBkPSJNMzIwIDkxOS45djIuMmwtMTcuMjcgNS42di0xMy40em0tMTQuMjcgMy42N2w3LjkyLTIuNTctNy45Mi0yLjU3eiIvPjxwYXRoIGQ9Ik00MjAgOTIxaDEyMCIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9Im5vbmUiLz48cGF0aCBkPSJNNDIwIDkyMi41di0zek01NDAgOTIxbC0xNC4yNyA0LjY0di05LjI4eiIvPjxwYXRoIGQ9Ik01NDAgOTE5LjQydjMuMTZsLTE1Ljc3IDUuMTJ2LTEzLjR6bS0xMi43NyA0LjE1bDcuOTItMi41Ny03LjkyLTIuNTd6Ii8+PHBhdGggZD0iTTE2NC42MiA3MDYuMzhjMC0yLjk3LTIuNTUtNS4zOC01LjctNS4zOGgtMTcuMTZjLTMuMTYgMC01LjcyIDIuNC01LjcyIDUuMzh2MTA5LjI0YzAgMi45NyAyLjU2IDUuMzggNS43MiA1LjM4aDE3LjE1YzMuMTcgMCA1LjcyLTIuNCA1LjcyLTUuMzh6IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0iIzY2NiIgZmlsbC1vcGFjaXR5PSIuNiIvPjxwYXRoIGQ9Ik0xNTkuODYgNzE5LjVjMCA0Ljk1LTQuMjYgOC45Ni05LjUzIDguOTYtNS4yNiAwLTkuNTItNC05LjUyLTguOTYgMC00Ljk0IDQuMjctOC45NSA5LjUzLTguOTUgNS4yNyAwIDkuNTMgNCA5LjUzIDguOTV6IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIvPjxwYXRoIGQ9Ik0xNTkuODYgNzQ2LjM3YzAgNC45NS00LjI2IDguOTUtOS41MyA4Ljk1LTUuMjYgMC05LjUyLTQtOS41Mi04Ljk1IDAtNC45NCA0LjI3LTguOTUgOS41My04Ljk1IDUuMjcgMCA5LjUzIDQgOS41MyA4Ljk1eiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9IiNjY2MiLz48cGF0aCBkPSJNMTU5Ljg2IDc3My4yNGMwIDQuOTQtNC4yNiA4Ljk1LTkuNTMgOC45NS01LjI2IDAtOS41Mi00LjAyLTkuNTItOC45NiAwLTQuOTQgNC4yNy04Ljk1IDkuNTMtOC45NSA1LjI3IDAgOS41MyA0IDkuNTMgOC45NHoiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSIjNjY2Ii8+PHBhdGggZD0iTTE1OS44NiA4MDAuMWMwIDQuOTUtNC4yNiA4Ljk2LTkuNTMgOC45Ni01LjI2IDAtOS41Mi00LTkuNTItOC45NSAwLTQuOTQgNC4yNy04Ljk2IDkuNTMtOC45NiA1LjI3IDAgOS41MyA0LjAyIDkuNTMgOC45N3oiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSIjZmZmIi8+PHBhdGggZD0iTTM4NC4wNCA3MDYuMzhjMC0yLjk3LTIuNTYtNS4zOC01LjcyLTUuMzhoLTE3LjE1Yy0zLjE1IDAtNS43IDIuNC01LjcgNS4zOHYxMDkuMjRjMCAyLjk3IDIuNTUgNS4zOCA1LjcgNS4zOGgxNy4xNWMzLjE2IDAgNS43Mi0yLjQgNS43Mi01LjM4eiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9IiNmZmYiIGZpbGwtb3BhY2l0eT0iLjYiLz48cGF0aCBkPSJNMzc5LjI4IDcxOS41YzAgNC45NS00LjI3IDguOTYtOS41MyA4Ljk2LTUuMjcgMC05LjUzLTQtOS41My04Ljk2IDAtNC45NCA0LjI2LTguOTUgOS41My04Ljk1IDUuMjYgMCA5LjUzIDQgOS41MyA4Ljk1eiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiLz48cGF0aCBkPSJNMzc5LjI4IDc0Ni4zN2MwIDQuOTUtNC4yNyA4Ljk1LTkuNTMgOC45NS01LjI3IDAtOS41My00LTkuNTMtOC45NSAwLTQuOTQgNC4yNi04Ljk1IDkuNTMtOC45NSA1LjI2IDAgOS41MyA0IDkuNTMgOC45NXoiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSIjY2NjIi8+PHBhdGggZD0iTTM3OS4yOCA3NzMuMjRjMCA0Ljk0LTQuMjcgOC45NS05LjUzIDguOTUtNS4yNyAwLTkuNTMtNC4wMi05LjUzLTguOTYgMC00Ljk0IDQuMjYtOC45NSA5LjUzLTguOTUgNS4yNiAwIDkuNTMgNCA5LjUzIDguOTR6IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0iIzY2NiIvPjxwYXRoIGQ9Ik0zNzkuMjggODAwLjFjMCA0Ljk1LTQuMjcgOC45Ni05LjUzIDguOTYtNS4yNyAwLTkuNTMtNC05LjUzLTguOTUgMC00Ljk0IDQuMjYtOC45NiA5LjUzLTguOTYgNS4yNiAwIDkuNTMgNC4wMiA5LjUzIDguOTd6IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0iI2ZmZiIvPjxwYXRoIGQ9Ik02MDQgNzA2LjM4YzAtMi45Ny0yLjUtNS4zOC01LjU4LTUuMzhoLTE2Ljc1Yy0zLjEgMC01LjYgMi40LTUuNiA1LjM4djEwOS4yNGMwIDIuOTcgMi41IDUuMzggNS42IDUuMzhoMTYuNzVjMy4wOCAwIDUuNTgtMi40IDUuNTgtNS4zOHoiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSIjY2NjIiBmaWxsLW9wYWNpdHk9Ii42Ii8+PHBhdGggZD0iTTU5OS4zNSA3MTkuNWMwIDQuOTUtNC4xNyA4Ljk2LTkuMyA4Ljk2LTUuMTUgMC05LjMtNC05LjMtOC45NiAwLTQuOTQgNC4xNS04Ljk1IDkuMy04Ljk1IDUuMTMgMCA5LjMgNCA5LjMgOC45NXoiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIi8+PHBhdGggZD0iTTU5OS4zNSA3NDYuMzdjMCA0Ljk1LTQuMTcgOC45NS05LjMgOC45NS01LjE1IDAtOS4zLTQtOS4zLTguOTUgMC00Ljk0IDQuMTUtOC45NSA5LjMtOC45NSA1LjEzIDAgOS4zIDQgOS4zIDguOTV6IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0iI2NjYyIvPjxwYXRoIGQ9Ik01OTkuMzUgNzczLjI0YzAgNC45NC00LjE3IDguOTUtOS4zIDguOTUtNS4xNSAwLTkuMy00LjAyLTkuMy04Ljk2IDAtNC45NCA0LjE1LTguOTUgOS4zLTguOTUgNS4xMyAwIDkuMyA0IDkuMyA4Ljk0eiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9IiM2NjYiLz48cGF0aCBkPSJNNTk5LjM1IDgwMC4xYzAgNC45NS00LjE3IDguOTYtOS4zIDguOTYtNS4xNSAwLTkuMy00LTkuMy04Ljk1IDAtNC45NCA0LjE1LTguOTYgOS4zLTguOTYgNS4xMyAwIDkuMyA0LjAyIDkuMyA4Ljk3eiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9IiNmZmYiLz48cGF0aCBkPSJNNTkwIDg4MXYtNTguNSIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9Im5vbmUiLz48cGF0aC8+PHBhdGggZD0iTTU5NC42NCA4MzYuNzdoLTkuMjhMNTkwIDgyMi41eiIvPjxwYXRoIGQ9Ik01OTYuNyA4MzguMjdoLTEzLjRsNS42LTE3LjI3aDIuMnptLTkuMjctM2g1LjE0bC0yLjU3LTcuOTJ6Ii8+PHBhdGggZD0iTTM3MCA4ODF2LTU4LjUiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSJub25lIi8+PHBhdGgvPjxwYXRoIGQ9Ik0zNzQuNjQgODM2Ljc3aC05LjI4TDM3MCA4MjIuNXoiLz48cGF0aCBkPSJNMzc2LjcgODM4LjI3aC0xMy40bDUuNi0xNy4yN2gyLjJ6bS05LjI3LTNoNS4xNGwtMi41Ny03LjkyeiIvPjxwYXRoIGQ9Ik0xNTAuMDggODgxbC4xMy02MCIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9Im5vbmUiLz48cGF0aCBkPSJNMTUwLjA4IDg4MWgtMS41ek0xNTQuOCA4MzUuMjhsLTkuMjYtLjAyTDE1MC4yIDgyMXoiLz48cGF0aCBkPSJNMTU2Ljg4IDgzNi43OGwtMTMuNC0uMDMgNS4xNS0xNS43NWgzLjE1em0tOS4yNy0zLjAyaDUuMTVsLTIuNTUtNy45eiIvPjxwYXRoIGQ9Ik01NjEuNSA2MjBjMC0xMS4wNSA4Ljk1LTIwIDIwLTIwaDE3YzExLjA1IDAgMjAgOC45NSAyMCAyMHMtOC45NSAyMC0yMCAyMGgtMTdjLTExLjA1IDAtMjAtOC45NS0yMC0yMHoiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSIjZDFiY2QyIi8+PHVzZSB4bGluazpocmVmPSIjYiIgdHJhbnNmb3JtPSJtYXRyaXgoMSwwLDAsMSw1NjEuNSw2MDApIHRyYW5zbGF0ZSgyMy4wOTkxMjEwOTM3NSAyNS4zOTY0ODQzNzUpIi8+PHBhdGggZD0iTTExOC41IDYyMGMwLTExLjA1IDguOTUtMjAgMjAtMjBoMTdjMTEuMDUgMCAyMCA4Ljk1IDIwIDIwcy04Ljk1IDIwLTIwIDIwaC0xN2MtMTEuMDUgMC0yMC04Ljk1LTIwLTIweiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9IiNkMWJjZDIiLz48dXNlIHhsaW5rOmhyZWY9IiNiIiB0cmFuc2Zvcm09Im1hdHJpeCgxLDAsMCwxLDExOC41LDYwMCkgdHJhbnNsYXRlKDIzLjA5OTEyMTA5Mzc1IDI1LjM5NjQ4NDM3NSkiLz48cGF0aCBkPSJNMzQwIDYyMGMwLTExLjA1IDguOTUtMjAgMjAtMjBoMTdjMTEuMDUgMCAyMCA4Ljk1IDIwIDIwcy04Ljk1IDIwLTIwIDIwaC0xN2MtMTEuMDUgMC0yMC04Ljk1LTIwLTIweiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9IiNkMWJjZDIiLz48dXNlIHhsaW5rOmhyZWY9IiNiIiB0cmFuc2Zvcm09Im1hdHJpeCgxLDAsMCwxLDM0MCw2MDApIHRyYW5zbGF0ZSgyMy4wOTkxMjEwOTM3NSAyNS4zOTY0ODQzNzUpIi8+PHBhdGggZD0iTTU5MC4wNCA3MDF2LTU5LjUiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSJub25lIi8+PHBhdGggZD0iTTU5MS41NCA3MDFoLTMgM3pNNTk0LjY4IDY1NS43N2gtOS4yN2w0LjY0LTE0LjI3eiIvPjxwYXRoIGQ9Ik01OTYuNzQgNjU3LjI3aC0xMy40bDUuNi0xNy4yN2gyLjJ6bS05LjI3LTNoNS4xNGwtMi41Ni03LjkyeiIvPjxwYXRoIGQ9Ik0xNTAuMzMgNzAxdi01OS41IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0ibm9uZSIvPjxwYXRoIGQ9Ik0xNTEuODMgNzAxaC0zIDN6TTE1NC45NyA2NTUuNzdoLTkuMjdsNC42My0xNC4yN3oiLz48cGF0aCBkPSJNMTU3LjAzIDY1Ny4yN2gtMTMuNGw1LjYtMTcuMjdoMi4yem0tOS4yNy0zaDUuMTRsLTIuNTctNy45MnoiLz48cGF0aCBkPSJNMzY5Ljc1IDcwMXYtNTkuNSIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9Im5vbmUiLz48cGF0aCBkPSJNMzcxLjI1IDcwMWgtMyAzek0zNzQuMzggNjU1Ljc3aC05LjI3bDQuNjUtMTQuMjd6Ii8+PHBhdGggZD0iTTM3Ni40NSA2NTcuMjdoLTEzLjRsNS42LTE3LjI3aDIuMnptLTkuMjctM2g1LjE0bC0yLjU3LTcuOTJ6Ii8+PHBhdGggZD0iTTExOSA1MjkuNWMwLTMuODcgMy4xMy03IDctN2g0ODZjMy44NyAwIDcgMy4xMyA3IDd2MjFjMCAzLjg3LTMuMTMgNy03IDdIMTI2Yy0zLjg3IDAtNy0zLjEzLTctN3oiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSIjZTVlNWU1Ii8+PHBhdGggZD0iTTU5NyA1MzkuNWMwIDUuNTItNC40OCAxMC0xMCAxMHMtMTAtNC40OC0xMC0xMCA0LjQ4LTEwIDEwLTEwIDEwIDQuNDggMTAgMTB6IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0iI2ZmZiIvPjxwYXRoIGQ9Ik0zODAgNTQwYzAgNS41Mi00LjQ4IDEwLTEwIDEwcy0xMC00LjQ4LTEwLTEwIDQuNDgtMTAgMTAtMTAgMTAgNC40OCAxMCAxMHoiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSIjZDFiY2QyIi8+PHBhdGggZD0iTTE1OCA1MzkuNWMwIDUuNTItNC45MiAxMC0xMSAxMHMtMTEtNC40OC0xMS0xMCA0LjkyLTEwIDExLTEwIDExIDQuNDggMTEgMTB6IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0iIzgzNDE4NyIvPjxwYXRoIGQ9Ik0xNDcgNjAwdi00OSIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9Im5vbmUiLz48cGF0aC8+PHBhdGggZD0iTTE1MS42NCA1NjUuMjhoLTkuMjhMMTQ3IDU1MXoiLz48cGF0aCBkPSJNMTQ3IDU0OS41bDEuMDQtLjE0IDUuNjYgMTcuNDJoLTEzLjRsNS42Ni0xNy40MnptLTIuNTcgMTQuMjhoNS4xNGwtMi41Ny03Ljl6Ii8+PHBhdGggZD0iTTM2OC41IDYwMHYtNDguNzIiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSJub25lIi8+PHBhdGgvPjxwYXRoIGQ9Ik0zNzMuMTQgNTY1LjU0aC05LjI4bDQuNjQtMTQuMjZ6Ii8+PHBhdGggZD0iTTM2OS42NCA1NDkuOTRsNS41NiAxNy4xaC0xMy40bDUuNjctMTcuNDV6bS0zLjcgMTQuMWg1LjEzbC0yLjU3LTcuOXoiLz48cGF0aCBkPSJNNTkwIDYwMHYtNDkuNDYiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSJub25lIi8+PHBhdGgvPjxwYXRoIGQ9Ik01OTQuNjQgNTY0LjhoLTkuMjhsNC42NC0xNC4yNnoiLz48cGF0aCBkPSJNNTk2LjcgNTY2LjNoLTEzLjRsNS41Ni0xNy4xIDEuMy0uMi43OC0uNHptLTkuMjctM2g1LjE0bC0yLjU3LTcuOXoiLz48cGF0aCBkPSJNMzIwIDQ4MGMwLTExLjA1IDguOTUtMjAgMjAtMjBoNThjMTEuMDUgMCAyMCA4Ljk1IDIwIDIwcy04Ljk1IDIwLTIwIDIwaC01OGMtMTEuMDUgMC0yMC04Ljk1LTIwLTIweiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9IiNkMWJjZDIiLz48dXNlIHhsaW5rOmhyZWY9IiNjIiB0cmFuc2Zvcm09Im1hdHJpeCgxLDAsMCwxLDMyMCw0NjApIHRyYW5zbGF0ZSgxMS4xOTM4NDc2NTYyNSAyNS4zOTY0ODQzNzUpIi8+PHBhdGggZD0iTTM2OSA1MjIuNXYtMjEiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSJub25lIi8+PHBhdGgvPjxwYXRoIGQ9Ik0zNzMuNjQgNTE1Ljc3aC05LjI4TDM2OSA1MDEuNXoiLz48cGF0aCBkPSJNMzc1LjcgNTE3LjI3aC0xMy40bDUuNi0xNy4yN2gyLjJ6bS05LjI3LTNoNS4xNGwtMi41Ny03LjkyeiIvPjxwYXRoIGQ9Ik0xMjAgNDA5LjVjMC0zLjg3IDMuMTMtNyA3LTdoNDg2YzMuODcgMCA3IDMuMTMgNyA3djIxYzAgMy44Ny0zLjEzIDctNyA3SDEyN2MtMy44NyAwLTctMy4xMy03LTd6IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0iI2U1ZTVlNSIvPjxwYXRoIGQ9Ik01OTggNDE5LjVjMCA1LjUyLTQuNDggMTAtMTAgMTBzLTEwLTQuNDgtMTAtMTAgNC40OC0xMCAxMC0xMCAxMCA0LjQ4IDEwIDEweiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9IiNmZmYiLz48cGF0aCBkPSJNMzgxIDQyMGMwIDUuNTItNC40OCAxMC0xMCAxMHMtMTAtNC40OC0xMC0xMCA0LjQ4LTEwIDEwLTEwIDEwIDQuNDggMTAgMTB6IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0iI2QxYmNkMiIvPjxwYXRoIGQ9Ik0xNTkgNDE5LjVjMCA1LjUyLTQuOTIgMTAtMTEgMTBzLTExLTQuNDgtMTEtMTAgNC45Mi0xMCAxMS0xMCAxMSA0LjQ4IDExIDEweiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9IiM4MzQxODciLz48cGF0aCBkPSJNMzY5IDQ2MHYtMjEiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSJub25lIi8+PHBhdGgvPjxwYXRoIGQ9Ik0zNzMuNjQgNDUzLjI3aC05LjI4TDM2OSA0Mzl6Ii8+PHBhdGggZD0iTTM3NS43IDQ1NC43N2gtMTMuNGw1LjYtMTcuMjdoMi4yem0tOS4yNy0zaDUuMTRsLTIuNTctNy45MnoiLz48cGF0aCBkPSJNMjUxLjUzIDc1MGg2Ny45N3YyNGgtNjcuOTd6IiBmaWxsPSIjZmZmIiBmaWxsLW9wYWNpdHk9IjAiLz48cGF0aCBkPSJNMjcxLjI0IDc2MmgtOC4wOHY4LjFoLTIuMDVWNzYyaC04LjA4di0yLjAzaDguMXYtOC4xaDIuMDR2OC4xaDguMDhWNzYyTTMwNS4zIDc1NS41N2wtNC40IDguMXYuNTRjMCAxLjUuMzggMy4wOCAxLjA2IDMuMDguNDMgMCAuOTMtLjYgMS41OC0yLjNsLjUuMjNjLS43IDIuMy0xLjY1IDMuOS0zLjEzIDMuOS0xLjE3IDAtMS41NC0xLjM1LTEuNi0zLjI0LTEuNTYgMS45LTMuNyAzLjMtNi4wMiAzLjMtMyAwLTQuMTUtMi4xLTQuMTUtNC43IDAtMi41IDEuMDgtNS4xNCAyLjgyLTYuOSAxLjM2LTEuNCAzLjQ0LTIuNCA1LjM2LTIuNCAyLjY0IDAgMy45MiAxLjg2IDMuOTIgMy43bDEuNDItMy4zaDIuNjdtLTUuODggNC44NGMwLTIuMS0uNC00LjU4LTIuNC00LjU4LTEuOTQgMC01LjM1IDQuMi01LjM1IDkuMTQgMCAxLjc2LjU2IDMuNTYgMi4xNCAzLjU2IDIuMTggMCA0LjMtNC4xOCA1LjEyLTUuOTIuMTMtLjYuNS0xLjM2LjUtMi4yTTMwMS42NSA3NTIuMjJIMjkydi0xLjY3aDkuNjV2MS42N00zMTQuMDQgNzczLjVIMzA4di0uMzNjMS42Mi0uMSAyLjA4LS41NSAyLjA4LTEuNzV2LTkuODRjMC0uNzUtLjItMS4wOC0uNjYtMS4wOC0uMjIgMC0uNi4xLTEgLjI3bC0uNTguMjJ2LS4zMmwzLjkyLTIgLjIuMDd2MTMuMDhjMCAuOTUuNDQgMS4zNCAyLjA4IDEuMzR2LjMzIi8+PHBhdGggZD0iTTQ3My41NSA3NTAuNTRINTQwVjc3NGgtNjYuNDV6IiBmaWxsPSIjZmZmIiBmaWxsLW9wYWNpdHk9IjAiLz48cGF0aCBkPSJNNDkyLjgyIDc2Mi4yOGgtNy45djcuOWgtMnYtNy45SDQ3NXYtMmg3Ljl2LTcuOWgydjcuOWg3Ljl2Mk01MjYuMTIgNzU1Ljk4bC00LjMgNy45di41NWMwIDEuNDYuMzcgMyAxLjAzIDMgLjQzIDAgLjktLjU3IDEuNTUtMi4yNGwuNDguMmMtLjY2IDIuMjQtMS42IDMuODItMy4wNiAzLjgyLTEuMTUgMC0xLjUtMS4zMy0xLjU3LTMuMTgtMS41MiAxLjg1LTMuNiAzLjI0LTUuODggMy4yNC0yLjk0IDAtNC4wNi0yLjA2LTQuMDYtNC42IDAtMi40NiAxLjA3LTUuMDMgMi43Ni02Ljc2IDEuMzQtMS4zNyAzLjM3LTIuMzQgNS4yNS0yLjM0IDIuNTggMCAzLjgyIDEuODIgMy44MiAzLjZsMS40LTMuMmgyLjZtLTUuNzUgNC43M2MwLTIuMDUtLjQtNC40OC0yLjM0LTQuNDgtMS45IDAtNS4yNCA0LjEyLTUuMjQgOC45NCAwIDEuNzMuNTQgMy40OCAyLjA4IDMuNDggMi4xMiAwIDQuMTgtNC4xIDUtNS43OC4xMi0uNTguNS0xLjM0LjUtMi4xNU01MjIuNTUgNzUyLjdoLTkuNDJ2LTEuNjNoOS40MnYxLjY0TTUzNi4zOCA3NzAuNThsLTEuMTYgMi45M2gtOC4zOHYtLjI1bDMuODItNC4wNWMyLTIuMTIgMi43OC0zLjc0IDIuNzgtNS41NyAwLTEuOTUtMS4xNS0zLjAyLTIuOTctMy4wMi0xLjU1IDAtMi4zLjctMy4xNSAyLjhsLS40NS0uMWMuNDUtMi41MiAxLjgyLTQuMjcgNC40NS00LjI3IDIuNDIgMCAzLjk3IDEuNjUgMy45NyAzLjc3IDAgMS43LS44NSAzLjMtMi43NiA1LjNMNTI5IDc3MS45aDUuMDJjLjkgMCAxLjM1LS4yMyAyLjA2LTEuNDMiLz48cGF0aCBkPSJNODAgNzUxaDMwLjV2MjMuNzZIODB6IiBmaWxsPSIjZmZmIiBmaWxsLW9wYWNpdHk9IjAiLz48cGF0aCBkPSJNOTYuNjQgNzU2LjQ0bC00LjMgNy45di41NmMwIDEuNDUuMzYgMyAxLjAyIDMgLjQzIDAgLjktLjU4IDEuNTUtMi4yNWwuNS4yMmMtLjY3IDIuMjQtMS42IDMuOC0zLjA3IDMuOC0xLjE1IDAtMS41LTEuMzItMS41Ny0zLjE3LTEuNTIgMS44NS0zLjYgMy4yNS01Ljg4IDMuMjUtMi45NCAwLTQuMDYtMi4wNy00LjA2LTQuNiAwLTIuNDYgMS4wNi01LjA0IDIuNzYtNi43NyAxLjMzLTEuMzYgMy4zNi0yLjMzIDUuMjQtMi4zMyAyLjU3IDAgMy44MiAxLjgyIDMuODIgMy42bDEuNC0zLjJoMi42bS01Ljc2IDQuNzJjMC0yLjA2LS40LTQuNDgtMi4zMy00LjQ4LTEuOSAwLTUuMjUgNC4xLTUuMjUgOC45MiAwIDEuNzMuNTUgMy41IDIuMSAzLjUgMi4xIDAgNC4xOC00LjEgNS01LjguMS0uNTcuNDgtMS4zMy40OC0yLjE1TTkzLjA2IDc1My4xN2gtOS40MnYtMS42NGg5LjQydjEuNjRNMTA2LjkzIDc2Ni45YzAgMy43LTEuMzUgNy4zOC00Ljg0IDcuMzgtMy42NyAwLTQuODUtNC00Ljg1LTcuNSAwLTMuOCAxLjQ4LTcuMyA0LjkzLTcuMyAyLjggMCA0Ljc1IDMuMDMgNC43NSA3LjQzbS0yLjA1LjFjMC00LjQ0LS45NS02Ljk1LTIuODMtNi45NS0xLjc4IDAtMi43NCAyLjUzLTIuNzQgNi44OHMuOTUgNi44IDIuOCA2LjhjMS44IDAgMi43OC0yLjQ4IDIuNzgtNi43MiIvPjxwYXRoIGQ9Ik03NDcuMzMgNzA0LjQ3YzAtMy4wMi0yLjQ0LTUuNDctNS40Ni01LjQ3aC0xNi40Yy0zLjAyIDAtNS40NyAyLjQ1LTUuNDcgNS40N3YxMTEuMDZjMCAzLjAyIDIuNDUgNS40NyA1LjQ3IDUuNDdoMTYuNGMzLjAyIDAgNS40Ni0yLjQ1IDUuNDYtNS40N3oiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSIjY2NjIiBmaWxsLW9wYWNpdHk9Ii42Ii8+PHBhdGggZD0iTTc0Mi43OCA3MTcuOGMwIDUuMDQtNC4wOCA5LjEtOS4xIDkuMS01LjA1IDAtOS4xMi00LjA2LTkuMTItOS4xIDAtNSA0LjA3LTkuMSA5LjEtOS4xIDUuMDQgMCA5LjEyIDQuMSA5LjEyIDkuMXoiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIi8+PHBhdGggZD0iTTc0Mi43OCA3NDUuMTNjMCA1LjAyLTQuMDggOS4xLTkuMSA5LjEtNS4wNSAwLTkuMTItNC4wOC05LjEyLTkuMSAwLTUuMDMgNC4wNy05LjEgOS4xLTkuMSA1LjA0IDAgOS4xMiA0LjA3IDkuMTIgOS4xeiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9IiNjY2MiLz48cGF0aCBkPSJNNzQyLjc4IDc3Mi40NGMwIDUuMDMtNC4wOCA5LjEtOS4xIDkuMS01LjA1IDAtOS4xMi00LjA3LTkuMTItOS4xIDAtNS4wMiA0LjA3LTkuMSA5LjEtOS4xIDUuMDQgMCA5LjEyIDQuMDggOS4xMiA5LjF6IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0iIzY2NiIvPjxwYXRoIGQ9Ik03NDIuNzggNzk5Ljc2YzAgNS4wMy00LjA4IDkuMS05LjEgOS4xLTUuMDUgMC05LjEyLTQuMDctOS4xMi05LjEgMC01LjAzIDQuMDctOS4xIDkuMS05LjEgNS4wNCAwIDkuMTIgNC4wNyA5LjEyIDkuMXoiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSIjZmZmIi8+PHBhdGggZD0iTTExMzggMTExMy42NGgxMy40N3YxNC43MkgxMTM4eiIgZmlsbD0iI2ZmZiIgZmlsbC1vcGFjaXR5PSIwIi8+PHBhdGggZD0iTTExNDguNiAxMTI0LjI3bC41LjNjLTEuODYgMi4zNC0zLjQ2IDMuMjgtNS43NCAzLjI4LTIuOCAwLTQuNDUtMS41OC00LjQ1LTQuNjcgMC0yLjgyIDEuNTgtNS40NSAzLjc0LTcuMTggMS40OC0xLjE4IDMuMi0xLjg1IDUuMDYtMS44NSAxLjg0IDAgMy4xOCAxLjAzIDMuMTggMi40NiAwIC44Mi0uNjcgMS40Ni0xLjQ2IDEuNDYtLjg0IDAtMS4xOC0uNi0xLjE4LTEuMiAwLS41Ni40Ni0uOS40Ni0xLjQ3IDAtLjM3LS40My0uNi0xLjEyLS42LTEuMjIgMC0yLjQzLjUzLTMuMyAxLjQ0LTEuNyAxLjgtMi43NyA0LjM3LTIuNzcgNy4wNiAwIDIuMTguOTggMy40NiAyLjcgMy40NiAxLjYyIDAgMi43Ny0uNzYgNC40LTIuNSIvPjxwYXRoIGQ9Ik0zMiA5ODBoMjI3djM5SDMyeiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utb3BhY2l0eT0iMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSIjZmZmIiBmaWxsLW9wYWNpdHk9IjAiLz48dXNlIHhsaW5rOmhyZWY9IiNkIiB0cmFuc2Zvcm09Im1hdHJpeCgxLDAsMCwxLDM3LDk4NSkgdHJhbnNsYXRlKDU5Ljg5MjA4OTg0Mzc1IDE5Ljg5NjQ4NDM3NSkiLz48cGF0aCBkPSJNNzgyIDg4MWMwLTExLjA1IDguOTUtMjAgMjAtMjBoNDU4YzExLjA1IDAgMjAgOC45NSAyMCAyMHYxMThjMCAxMS4wNS04Ljk1IDIwLTIwIDIwSDgwMmMtMTEuMDUgMC0yMC04Ljk1LTIwLTIweiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utb3BhY2l0eT0iMCIgc3Ryb2tlLXdpZHRoPSIuNSIgZmlsbD0iI2QxYmNkMiIgZmlsbC1vcGFjaXR5PSIuMyIvPjxwYXRoIGQ9Ik04MDYuMTQgODk2YzAtOC4yOCA2LjcyLTE1IDE1LTE1aDcwYzguMyAwIDE1IDYuNzIgMTUgMTV2NTBjMCA4LjI4LTYuNyAxNS0xNSAxNWgtNzBjLTguMjggMC0xNS02LjcyLTE1LTE1eiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9IiNmZmYiIGZpbGwtb3BhY2l0eT0iLjQ4Ii8+PHVzZSB4bGluazpocmVmPSIjYSIgdHJhbnNmb3JtPSJtYXRyaXgoMSwwLDAsMSw4MDYuMTQwOTY4NTUyNzI5NCw4ODEpIHRyYW5zbGF0ZSgxNy45OTQ3OTE2NjY2NjY2NjQgNDcuOTk0NzkxNjY2NjY2NjY0KSIvPjxwYXRoIGQ9Ik0xMDM4IDg5NmMwLTguMjggNi43Mi0xNSAxNS0xNWg3MGM4LjI4IDAgMTUgNi43MiAxNSAxNXY1MGMwIDguMjgtNi43MiAxNS0xNSAxNWgtNzBjLTguMjggMC0xNS02LjcyLTE1LTE1eiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9IiNmZmYiIGZpbGwtb3BhY2l0eT0iLjQ4Ii8+PHVzZSB4bGluazpocmVmPSIjYSIgdHJhbnNmb3JtPSJtYXRyaXgoMSwwLDAsMSwxMDM4LDg4MSkgdHJhbnNsYXRlKDE3Ljk5NDc5MTY2NjY2NjY2NCA0Ny45OTQ3OTE2NjY2NjY2NjQpIi8+PHBhdGggZD0iTTkwNi4xNCA5MjFIMTAzOCIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9Im5vbmUiLz48cGF0aC8+PHBhdGggZD0iTTEwMzggOTIxbC0xNC4yNyA0LjY0di05LjI4eiIvPjxwYXRoIGQ9Ik0xMDM4IDkxOS40MnYzLjE2bC0xNS43NyA1LjEydi0xMy40em0tMTIuNzcgNC4xNWw3LjkyLTIuNTctNy45Mi0yLjU3eiIvPjxwYXRoIGQ9Ik04MjAgMTI1MGMwLTUuNTIgNC40OC0xMCAxMC0xMGgxMjJjNS41MiAwIDEwIDQuNDggMTAgMTB2MjBjMCA1LjUyLTQuNDggMTAtMTAgMTBIODMwYy01LjUyIDAtMTAtNC40OC0xMC0xMHoiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIyIiBmaWxsPSIjZmZmIiBmaWxsLW9wYWNpdHk9IjAiLz48dXNlIHhsaW5rOmhyZWY9IiNlIiB0cmFuc2Zvcm09Im1hdHJpeCgxLDAsMCwxLDgyNSwxMjQ1KSB0cmFuc2xhdGUoMTQuNjU4MzExNjMxOTQ0NDQzIDIyLjMyODU1OTAyNzc3Nzc4KSIvPjxwYXRoIGQ9Ik0xMTAyLjU4IDEyMDguODhjMC0yLjk3LTIuNTUtNS4zOC01LjctNS4zOGgtMTcuMTZjLTMuMTYgMC01LjcyIDIuNC01LjcyIDUuMzh2MTA5LjI0YzAgMi45NyAyLjU2IDUuMzggNS43MiA1LjM4aDE3LjE1YzMuMTYgMCA1LjctMi40IDUuNy01LjM4eiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9IiNmZmYiIGZpbGwtb3BhY2l0eT0iLjYiLz48cGF0aCBkPSJNMTA5Ny44MiAxMjIyYzAgNC45NS00LjI2IDguOTYtOS41MyA4Ljk2cy05LjU0LTQtOS41NC04Ljk2YzAtNC45NCA0LjI3LTguOTUgOS41My04Ljk1czkuNTIgNCA5LjUyIDguOTV6IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIvPjxwYXRoIGQ9Ik0xMDk3LjgyIDEyNDguODdjMCA0Ljk1LTQuMjYgOC45NS05LjUzIDguOTVzLTkuNTQtNC05LjU0LTguOTVjMC00Ljk0IDQuMjctOC45NSA5LjUzLTguOTVzOS41MiA0IDkuNTIgOC45NXoiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSIjY2NjIi8+PHBhdGggZD0iTTEwOTcuODIgMTI3NS43NGMwIDQuOTQtNC4yNiA4Ljk1LTkuNTMgOC45NXMtOS41NC00LjAyLTkuNTQtOC45NmMwLTQuOTQgNC4yNy04Ljk1IDkuNTMtOC45NXM5LjUyIDQgOS41MiA4Ljk0eiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9IiM2NjYiLz48cGF0aCBkPSJNMTA5Ny44MiAxMzAyLjZjMCA0Ljk1LTQuMjYgOC45Ni05LjUzIDguOTZzLTkuNTQtNC05LjU0LTguOTVjMC00Ljk0IDQuMjctOC45NiA5LjUzLTguOTZzOS41MiA0LjAyIDkuNTIgOC45N3oiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSIjZmZmIi8+PHBhdGggZD0iTTEwODcuNiAxMDYwbC40LTk5IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0ibm9uZSIvPjxwYXRoIGQ9Ik0xMDg3LjYgMTA2MGgtMS41ek0xMDkyLjU4IDk3NS4yOGwtOS4yNy0uMDNMMTA4OCA5NjF6Ii8+PHBhdGggZD0iTTEwOTQuNjQgOTc2LjhsLTEzLjQtLjA2IDUuMTgtMTUuNzRoMy4xNnptLTkuMjYtMy4wNGw1LjE0LjAyLTIuNTQtNy45M3oiLz48cGF0aCBkPSJNNzM1IDk4MGgyMjd2MzlINzM1eiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utb3BhY2l0eT0iMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSIjZmZmIiBmaWxsLW9wYWNpdHk9IjAiLz48dXNlIHhsaW5rOmhyZWY9IiNmIiB0cmFuc2Zvcm09Im1hdHJpeCgxLDAsMCwxLDc0MCw5ODUpIHRyYW5zbGF0ZSg1OS44OTIwODk4NDM3NSAxOS44OTY0ODQzNzUpIi8+PHBhdGggZD0iTTg1Ni4xNCA4ODFsLS4yNy0xMi45LS44LTEyLjc1LTEuMzItMTIuNTQtMS44Mi0xMi4zMi0yLjMtMTIuMDgtMi43Ny0xMS44Mi0zLjIyLTExLjUzLTMuNjUtMTEuMjMtNC4wNi0xMC45LTQuNDQtMTAuNTUtNC44LTEwLjItNS4xMy05LjgyLTUuNDYtOS40NS01Ljc0LTkuMDYtNi4wMi04LjY3LTYuMjgtOC4yNy02LjUyLTcuODgtNi43My03LjQ4LTYuOTQtNy4xLTcuMTMtNi43LTcuMzItNi4zLTcuNDgtNS45NS03LjY1LTUuNTctNy44LTUuMi03Ljk4LTQuOC04LjEzLTQuNDctOC4yNy00LjEtOC40My0zLjc2LTguNi0zLjQtOC43NC0zLjA1LTguOTItMi43LTkuMS0yLjMzLTkuMjgtMS45OC05LjUtMS42LTkuNy0xLjIzLTkuOTQtLjgyLTEwLjItLjQzLTUuMTgtLjA1IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0ibm9uZSIvPjxwYXRoIGQ9Ik04NTcuNjQgODgxaC0xLjY2bDEuNjYtLjA0ek02MzIuNjIgNjI1LjA3TDYxOC41IDYyMGwxNC40LTQuMnoiLz48cGF0aCBkPSJNNjM0LjA1IDYyNy4xOGwtMTUuNzQtNS42NS4yLTEuNTMtLjA1LTEuNDQtLjAyLS4xIDE2LjA0LTQuNjd6bS0xMC43LTcuMDNsNy44MyAyLjguMTYtNS4xMnoiLz48cGF0aCBkPSJNNjQwLjA4IDc1OC40aDE1Ljd2OS42aC0xNS43eiIgZmlsbD0iI2ZmZiIgZmlsbC1vcGFjaXR5PSIwIi8+PHBhdGggZD0iTTY1NC42OCA3NjAuM2gtMTMuNXYtMS41aDEzLjV2MS41bTAgNC41OGgtMTMuNXYtMS41aDEzLjV2MS41Ii8+PHBhdGggZD0iTTM3MSA0MzBsLTcuMSAyLjQ3LTYuMzcgMi41Mi01Ljc2IDIuNTUtNS4yMyAyLjYtNC44IDIuNjMtNC40IDIuNjYtNC4wNiAyLjctMy43OCAyLjc1LTMuNSAyLjc3LTMuMyAyLjg0LTMuMDggMi45LTIuOSAyLjk3LTIuNzUgMy4wNC0yLjYgMy4xNC0yLjQ0IDMuMjQtMi4zMyAzLjM1LTIuMjIgMy41LTIuMSAzLjY0LTIgMy44Mi0xLjkgNC4wMy0xLjggNC4zLTEuNyA0LjU2LTEuNiA0LjktMS40OCA1LjMtMS4zNiA1LjgtMS4yMiA2LjQyLTEuMDUgNy4xNi0uODUgOC4xMy0uNiA5LjQ2LS4yMiAxMS4zMy4zIDE0LjM3IDEuMzQgMjAuNjIgMTAuMyAxMDUuMjUuMyAxNC4yNi0uMDcgNyIgZmlsbD0ibm9uZSIvPjxwYXRoIGQ9Ik0zNzEgNDMwbC00LjIzIDEuNDdtLTUuNiAyLjA4bC0zLjY0IDEuNDQtNC42IDIuMDRtLTUuMzcgMi42bC0xLjAyLjUtNC44IDIuNjQtMiAxLjJtLTUuMDMgMy4yMmwtMS40Mi45NS0zLjc4IDIuNzQtMiAxLjU4bS00LjU4IDMuODRsLS4yMi4yLTMuMDggMi45LTIuOSAyLjk1LS4yLjJtLTMuOTIgNC41bC0xLjIyIDEuNDctMi40NSAzLjI0LTEuNyAyLjQ1bS0zLjIyIDUuMDNsLTEuNzIgMy0yIDMuODMtLjUgMS4wNG0tMi40MyA1LjQ1bC0uNzcgMS44Mi0xLjcgNC41Ny0uNjUgMS45OG0tMS43MyA1LjdsLS43IDIuNTMtMS4zNiA1LjgtLjA3LjM4bS0xLjEgNS44NmwtLjA1LjE4LTEuMDUgNy4xNi0uMTYgMS41M20tLjYgNS45NGwtLjA4LjY3LS41MiA4LjI4bS0uMTcgNS45NmwtLjEzIDYuNTUuMDUgMi40bS4xMyA1Ljk2bC4xMyA2IC4yIDIuOTRtLjQgNS45NmwuNTYgOC45M20uNSA1Ljk0bC44NiA4LjltLjU4IDUuOTVsLjg4IDguOW0uNTggNS45NGwuODcgOC45bS41OCA1Ljk1bC44OCA4LjltLjU4IDUuOTRsLjg3IDguOW0uNTggNS45NWwuOSA4LjltLjU3IDUuOTNsLjg3IDguOW0uNDQgNS45NmwuMiA4Ljk0bS4wNCA1Ljk3bC0uMDUgNC41IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0ibm9uZSIvPjxwYXRoIGQ9Ik0zNzQuMzMgNDMwLjM2bC0yLjggMS4wNS0uNTMtMS40IDMuMDItLjV6Ii8+PHBhdGggZD0iTTI5OC42NyA3MTlsLTQuNzItMTQuMjUgOS4yNy0uMDZ6IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIvPjxwYXRoIGQ9Ik01ODggNDI5LjVsLTYuOSAzLjg2LTYuMTggMy44LTUuNiAzLjc4LTUuMDcgMy43NS00LjY0IDMuNzMtNC4yNSAzLjczLTMuOTMgMy43NS0zLjY0IDMuOC0zLjQgMy44LTMuMTUgMy44OC0yLjk1IDMuOTYtMi43NyA0LjA1LTIuNiA0LjE2LTIuNDQgNC4zMi0yLjMgNC40OC0yLjE2IDQuNy0yLjAzIDQuOTQtMS45IDUuMjUtMS43OCA1LjYyLTEuNjUgNi4wNi0xLjUgNi42My0xLjM0IDcuMzUtMS4xNSA4LjMtLjkzIDkuNjItLjYzIDExLjU3LS4yIDE0LjkuNzIgMjIuOTMgNS40NiA5OS4xNS0uMjMgMTkuMDMtLjE4IDQuNiIgZmlsbD0ibm9uZSIvPjxwYXRoIGQ9Ik01ODggNDI5LjVsLTMuOTcgMi4yMm0tNS4yNCAzLjA2bC0zLjg4IDIuNC0zLjc3IDIuNTNtLTQuOTMgMy41NGwtMS45NyAxLjQ1LTQuNjQgMy43My0uNS40NW0tNC41MyA0LjAzbC0zLjE1IDMtMy4zIDMuNDNtLTQuMDIgNC41M2wtMi44NyAzLjUyLTIuNzIgMy42NW0tMy40IDUuMDJsLTIuMiAzLjUtMi40MyA0LjMybS0yLjczIDUuNGwtMS43NCAzLjgtMS44OCA0LjU1bS0yLjA4IDUuN2wtMS43NSA1LjU1LS44NyAzLjE2bS0xLjQ2IDUuODhsLS44MiAzLjY2LS45NiA1LjI3bS0uOTQgNmwtLjYgNC4zOC0uNDYgNC42NW0tLjUzIDYuMDRsLS41IDkuMDdtLS4xMyA2LjA2bC0uMSA5LjFtLjEyIDYuMDVsLjI4IDkuMW0uMiA2LjA1bC4wOCAyLjkuMzQgNi4ybS4zMyA2LjA0bC41IDkuMDhtLjMzIDYuMDZsLjUgOS4xbS4zMyA2LjA0bC41IDkuMDhtLjM0IDYuMDZsLjUgOS4wOG0uMzMgNi4wNWwuNSA5LjFtLjMzIDYuMDVsLjUgOS4wOG0uMDcgNi4wNmwtLjEgOS4wOG0tLjA4IDYuMDZsLS4xOCA0LjU1IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0ibm9uZSIvPjxwYXRoIGQ9Ik01OTEuMzQgNDI5LjI4bC0yLjYgMS41Mi0uNzQtMS4zIDMuMTYtLjV2LS4wMnoiLz48cGF0aCBkPSJNNTE4LjY3IDcxOWwtNC4zNC0xNC4zNyA5LjI3LjJ6IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIvPjxwYXRoIGQ9Ik0yMCAzNDBoMzMwdjQyLjVIMjB6IiBzdHJva2U9IiMwMDAiIHN0cm9rZS1vcGFjaXR5PSIwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9IiNmZmYiIGZpbGwtb3BhY2l0eT0iMCIvPjx1c2UgeGxpbms6aHJlZj0iI2ciIHRyYW5zZm9ybT0ibWF0cml4KDEsMCwwLDEsMjUsMzQ1KSB0cmFuc2xhdGUoNDYuNTgxNTQyOTY4NzUgMjEuODk2NDg0Mzc1KSIvPjx1c2UgeGxpbms6aHJlZj0iI2giIHRyYW5zZm9ybT0ibWF0cml4KDEsMCwwLDEsMjUsMzQ1KSB0cmFuc2xhdGUoMTY1LjQwMDg3ODkwNjI1IDIxLjg5NjQ4NDM3NSkiLz48cGF0aCBkPSJNMTQ4IDQyOS41bC00Ljk3IDIuNC00LjU1IDIuNDQtNC4yIDIuNDctMy44OCAyLjUzLTMuNjIgMi41Ny0zLjM3IDIuNi0zLjE2IDIuNy0yLjk4IDIuNzMtMi44MiAyLjgtMi42NyAyLjktMi41NCAzLTIuNDIgMy4xMy0yLjMgMy4yNC0yLjIgMy40LTIuMTIgMy41Ny0yLjAyIDMuNzctMS45NCA0LTEuODYgNC4zLTEuNzcgNC42LTEuNyA1LjAyLTEuNiA1LjUtMS41IDYuMDctMS4zNiA2LjgyLTEuMjQgNy44LTEuMDUgOS4xMi0uOCAxMS4wMi0uNDUgMTQuMS4yNCAyMC40IDUuNzYgMTI0LjkzLS41OCAxOC42LS4zMiA1LjMiIGZpbGw9Im5vbmUiLz48cGF0aCBkPSJNMTQ4IDQyOS41bC00LjEgMS45OG0tNS40IDIuODRsLS4wMi4wMi00LjIgMi40Ny0zLjUzIDIuM20tNC45NCAzLjU1bC0yLjQgMS44Ni0zLjE2IDIuNy0xLjQzIDEuM20tNC4zNSA0LjIzbC0yLjY4IDIuOS0yLjU0IDMtLjc2IDFtLTMuNiA0Ljg4bC0uMzcuNS0yLjIgMy40LTIuMTIgMy41Ny0uMTQuMjdtLTIuOCA1LjRsLTEuMDIgMi4xLTEuODYgNC4zLS43NSAxLjk3bS0yLjA1IDUuNzNsLS42NSAxLjkzLTEuNiA1LjUtLjMyIDEuMzJtLTEuNCA1LjlsLTEuMTQgNS42Ny0uNTMgMy4zbS0uODggNi4wMmwtLjg3IDcuNi0uMSAxLjQ3bS0uNDUgNi4wN2wtLjI2IDMuNS0uMTcgNS42Mm0tLjIgNi4wN2wtLjA3IDIuNC4wOCA2LjdtLjA3IDYuMWwuMSA3LjYuMDYgMS41bS4zIDYuMWwuNCA5LjFtLjMgNi4wOGwuNCA5LjFtLjMgNi4wOGwuNCA5LjEybS4zIDYuMDdsLjQgOS4xbS4zIDYuMWwuNCA5LjFtLjMgNi4wN2wuNCA5LjEybS4zIDYuMDdsLjQgOS4xMm0uMyA2LjA4bC40IDkuMW0tLjAzIDYuMWwtLjMgOS4xbS0uMiA2LjA4bC0uMjcgNC41NSIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9Im5vbmUiLz48cGF0aCBkPSJNMTUxLjM1IDQyOS40MmwtMi42NSAxLjQtLjctMS4zMiAzLjEyLS40LjA1LS4wM3oiLz48cGF0aCBkPSJNODggNzIzLjMzbC00LjA0LTE0LjQ0IDkuMjYuMzd6IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIvPjxwYXRoIGQ9Ik0xMDg4IDg4MXYtNDciIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSJub25lIi8+PHBhdGggZD0iTTEwODkuNSA4ODFoLTMgM3pNMTA5Mi42NCA4NDguMjdoLTkuMjhMMTA4OCA4MzR6Ii8+PHBhdGggZD0iTTEwOTQuNyA4NDkuNzdoLTEzLjRsNS4xMi0xNS43N2gzLjE2em0tOS4yNy0zaDUuMTRsLTIuNTctNy45MnoiLz48cGF0aCBkPSJNMTA2MCA4MTFjMC0xMS4wNSA4Ljk1LTIwIDIwLTIwaDE2YzExLjA1IDAgMjAgOC45NSAyMCAyMHYzYzAgMTEuMDUtOC45NSAyMC0yMCAyMGgtMTZjLTExLjA1IDAtMjAtOC45NS0yMC0yMHoiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSIjZDFiY2QyIi8+PHVzZSB4bGluazpocmVmPSIjaSIgdHJhbnNmb3JtPSJtYXRyaXgoMSwwLDAsMSwxMDYwLDc5MSkgdHJhbnNsYXRlKDIyLjU5OTEyMTA5Mzc1IDI2Ljg5NjQ4NDM3NSkiLz48cGF0aCBkPSJNMTA4OCA3OTFsMS00MCIgZmlsbD0ibm9uZSIvPjxwYXRoIGQ9Ik0xMDg4IDc5MWwuMS00bS4xMy01LjMzbC4yLThtLjE0LTUuMzRsLjItOG0uMTMtNS4zM2wuMS00IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0ibm9uZSIvPjxwYXRoIGQ9Ik0xMDg4LjAyIDc5MWgtMS41MnYtLjA0eiIvPjxwYXRoIGQ9Ik0xMDg5IDc1MWw0LjI4IDE0LjM4LTkuMjctLjIzeiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiLz48cGF0aCBkPSJNMTEwMSAxMDc0LjI3YzAtMy4wMi0yLjQ1LTUuNDctNS40Ny01LjQ3aC0xNi40Yy0zLjAyIDAtNS40NiAyLjQ1LTUuNDYgNS40N3YxMTEuMDZjMCAzLjAyIDIuNDQgNS40NyA1LjQ2IDUuNDdoMTYuNGMzLjAyIDAgNS40Ny0yLjQ1IDUuNDctNS40N3oiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSIjY2NjIiBmaWxsLW9wYWNpdHk9Ii42Ii8+PHBhdGggZD0iTTEwOTYuNDQgMTA4Ny42YzAgNS4wNC00LjA3IDkuMS05LjEgOS4xLTUuMDQgMC05LjEyLTQuMDYtOS4xMi05LjEgMC01IDQuMDgtOS4xIDkuMS05LjEgNS4wNSAwIDkuMTIgNC4xIDkuMTIgOS4xeiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiLz48cGF0aCBkPSJNMTA5Ni40NCAxMTE0LjkzYzAgNS4wMi00LjA3IDkuMS05LjEgOS4xLTUuMDQgMC05LjEyLTQuMDgtOS4xMi05LjEgMC01LjAzIDQuMDgtOS4xIDkuMS05LjEgNS4wNSAwIDkuMTIgNC4wNyA5LjEyIDkuMXoiIHN0cm9rZT0iIzAwMCIgc3Ryb2tlLXdpZHRoPSIzIiBmaWxsPSIjY2NjIi8+PHBhdGggZD0iTTEwOTYuNDQgMTE0Mi4yNGMwIDUuMDMtNC4wNyA5LjEtOS4xIDkuMS01LjA0IDAtOS4xMi00LjA3LTkuMTItOS4xIDAtNS4wMiA0LjA4LTkuMSA5LjEtOS4xIDUuMDUgMCA5LjEyIDQuMDggOS4xMiA5LjF6IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0iIzY2NiIvPjxwYXRoIGQ9Ik0xMDk2LjQ0IDExNjkuNTZjMCA1LjAzLTQuMDcgOS4xLTkuMSA5LjEtNS4wNCAwLTkuMTItNC4wNy05LjEyLTkuMSAwLTUuMDMgNC4wOC05LjEgOS4xLTkuMSA1LjA1IDAgOS4xMiA0LjA3IDkuMTIgOS4xeiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9IiNmZmYiLz48cGF0aCBkPSJNNzMzLjY3IDgyMWwuNzIgMzUuNjcgMS44OCAzMC4zIDIuNyAyNS43NSAzLjIzIDIxLjk2IDMuNiAxOC44NSAzLjggMTYuMyAzLjk1IDE0LjE2IDQuMDMgMTIuNCA0LjA3IDEwLjk0IDQuMDcgOS43IDQuMDcgOC42NiA0LjA0IDcuNzYgNC4wNCA3IDQgNi4zMiA0IDUuNzUgNCA1LjI1IDMuOTYgNC44IDMuOTcgNC40IDMuOTggNC4wNyA0IDMuNzQuMjIuMiA0LjEgMy41MyA0LjM4IDMuNDUgNC42NiAzLjM2IDQuOTggMy4zIDUuMzUgMy4yMiA1LjggMy4xNiA2LjI3IDMuMSA2Ljg4IDMuMDMgNy41NiAyLjk3IDguNCAyLjkgOS40NCAyLjgyIDEwLjcgMi43MiAxMi4zOCAyLjYgMTQuNiAyLjQgMTcuOCAyLjIgMjMuMDQgMS44NiAzNC41IDEuMyA3My41Ni4wOCIgZmlsbD0ibm9uZSIvPjxwYXRoIGQ9Ik03MzMuNjcgODIxbC4xIDQuNDdtLjEgNS45NWwuMiA4LjkybS4xIDUuOTVsLjIgOC45Mm0uMyA1Ljk0bC41NiA4LjltLjM3IDUuOTVsLjU2IDguOTJtLjUzIDUuOTNsLjkyIDguODdtLjYyIDUuOTJsLjc0IDcuMDguMjYgMS44bS44NyA1Ljg4bDEuMyA4LjgzbS45IDUuOWwxLjY2IDguNzZtMS4xIDUuODRsLjczIDMuOCAxLjE1IDQuOTJtMS4zNSA1LjhsMS4zIDUuNTYuODcgMy4xbTEuNiA1LjczbDEuNDggNS4zNCAxLjA1IDMuMm0xLjg0IDUuNjdsMS4xNCAzLjUzIDEuODIgNC45bTIuMDcgNS41N2wuMTguNDcgMy4yNiA3Ljc2bTIuNDYgNS40M2wyLjQzIDUuMTcgMS40OCAyLjg1bTIuNzcgNS4yNmwzLjgzIDYuNjUuNjggMS4wN20zLjE4IDUuMDJsLjE1LjI0IDQgNS43NS45OCAxLjNtMy42MyA0LjczbDMuMzQgNC4wMiAyLjQ4IDIuNzVtNC4xIDQuMzJsMS4zNiAxLjQgNCAzLjc0LjIyLjIuOS43OG00LjU3IDMuODJsMyAyLjM4IDQuMTMgMi45OG00Ljk1IDMuM2wuNTcuMzcgNS4zNSAzLjIzIDEuNzYuOTVtNS4yNSAyLjgybDUuMDYgMi41IDMgMS4zMm01LjQ4IDIuMzRsNS45NiAyLjM0IDIuMzguODJtNS42MyAxLjk1bC40LjEyIDguMTQgMi40NG01Ljc1IDEuNTJsNi4yMyAxLjU4IDIuNDQuNW01LjgyIDEuMjNsNC4xLjg1IDQuNjcuNzdtNS44Ny45OGw0LjA2LjY3IDQuNzcuNm01LjkuNzJsNy4xNC44NiAxLjc1LjE0bTUuOTMuNDhsOC45LjcybTUuOTMuNDdsLjU0LjA1IDguNC4zbTUuOTQuMjRsOC45Mi4zM201Ljk1LjJsNS4zLjIgMy42Mi4wMm01Ljk1IDBoOC45M201Ljk1LjAyaDguOTNtNS45NS4wMmg4LjkzbTUuOTYuMDJoOC45Mm01Ljk2IDBsNC40Ni4wMiIgc3Ryb2tlPSIjMDAwIiBzdHJva2Utd2lkdGg9IjMiIGZpbGw9Im5vbmUiLz48cGF0aCBkPSJNNzMyLjE3IDgyMS4wM1Y4MjFoMS42M3pNMTA2MC40IDExMjNsLTE0LjI3IDQuNjJ2LTkuMjd6Ii8+PHBhdGggZD0iTTEwNjAuNCAxMTIxLjQydjMuMTZsLTE1Ljc3IDUuMXYtMTMuNHptLTEyLjc3IDQuMTRsNy45Mi0yLjU3LTcuOS0yLjZ6Ii8+PHBhdGggZD0iTTk2MiAxMjYwaDExMC41IiBzdHJva2U9IiMwMDAiIHN0cm9rZS13aWR0aD0iMyIgZmlsbD0ibm9uZSIvPjxwYXRoIGQ9Ik05NjIgMTI2MS41di0zek0xMDcyLjUgMTI2MGwtMTQuMjcgNC42NHYtOS4yOHoiLz48cGF0aCBkPSJNMTA3NCAxMjU4Ljl2Mi4ybC0xNy4yNyA1LjZ2LTEzLjR6bS0xNC4yNyAzLjY3bDcuOTItMi41Ny03LjkyLTIuNTd6Ii8+PGRlZnM+PHBhdGggZD0iTTIzNyAwdi0xMzQ5aDE5MXYxMTkzaDY3MlYwSDIzNyIgaWQ9ImoiLz48cGF0aCBkPSJNNjE0LTEyMjZjLTE2NyAxLTI4MyA1My0yODMgMjEzIDAgMTgzIDE4NiAxOTMgMzM0IDIzNCAyMzAgNjMgNDYzIDEyMCA0NjMgNDA5IDAgMjg2LTIxOSAzODctNTE4IDM5MEMzMDkgMjMgMTMxLTk4IDc5LTMzOGwxODUtMzdjMzQgMTY1IDE0OSAyNDggMzUxIDI0NiAxODQtMiAzMjQtNTggMzI0LTIzOCAwLTIwMy0yMDctMjIxLTM3Mi0yNjYtMjEwLTU3LTQyMi0xMTEtNDIyLTM3NyAwLTI2NyAyMDEtMzU2IDQ3MC0zNjAgMjc5LTUgNDMwIDEwMSA0ODAgMzI0bC0xODggMzNjLTI4LTE0MS0xMjEtMjE1LTI5My0yMTMiIGlkPSJrIi8+PHBhdGggZD0iTTcwOS0xMTkzVjBINTE5di0xMTkzSDc2di0xNTZoMTA3NnYxNTZINzA5IiBpZD0ibCIvPjxwYXRoIGQ9Ik0yODUtMTE2OWM4IDM4MiAyIDc4MCA0IDExNjlIMTI5di0xMzQ5aDIzN2M4NiAyMzkgMTg4IDQ2MSAyNTMgNzIwIDY5LTI1OCAxNjktNDgxIDI1NS03MjBoMjI1VjBIOTM3YzItMzkwLTUtNzg4IDYtMTE2OS03NSAyNTUtMTcwIDQ4OC0yNTkgNzI5SDU0N2MtOTAtMjQwLTE4NS00NzUtMjYyLTcyOSIgaWQ9Im0iLz48ZyBpZD0iYSI+PHVzZSB0cmFuc2Zvcm09Im1hdHJpeCgwLjAxMzAyMDgzMzMzMzMzMzMzNCwwLDAsMC4wMTMwMjA4MzMzMzMzMzMzMzQsMCwwKSIgeGxpbms6aHJlZj0iI2oiLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDEzMDIwODMzMzMzMzMzMzM0LDAsMCwwLjAxMzAyMDgzMzMzMzMzMzMzNCwxNi4wMDI2MDQxNjY2NjY2NjgsMCkiIHhsaW5rOmhyZWY9IiNrIi8+PHVzZSB0cmFuc2Zvcm09Im1hdHJpeCgwLjAxMzAyMDgzMzMzMzMzMzMzNCwwLDAsMC4wMTMwMjA4MzMzMzMzMzMzMzQsMzIuMDA1MjA4MzMzMzMzMzM2LDApIiB4bGluazpocmVmPSIjbCIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMTMwMjA4MzMzMzMzMzMzMzQsMCwwLDAuMDEzMDIwODMzMzMzMzMzMzM0LDQ4LjAwNzgxMjUsMCkiIHhsaW5rOmhyZWY9IiNtIi8+PC9nPjxwYXRoIGQ9Ik04MzktMTMzNWMtMTgyLTYtMjY5IDY3LTI1OSAyNTNoNDkxdjE0Mkg1ODBWMEg0MDB2LTk0MEgxMzh2LTE0MmgyNjJjLTE1LTI5MyAxMzItNDA4IDQxOC00MDIgOTQgMiAyMDAgNyAyODEgMjF2MTQ1Yy03NS0xMC0xNzctMTQtMjYwLTE3IiBpZD0ibiIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsMCwwKSIgeGxpbms6aHJlZj0iI24iIGlkPSJiIi8+PHBhdGggZD0iTTg3My04MTljLTE4LTExNC0xMTktMTQ2LTI1MC0xNDYtMTYzIDAtMjQ1IDUwLTI0NSAxNTEgMCAxNTEgMTcwIDE0OCAyOTQgMTg1IDE4MiA1NCAzODggOTQgMzg4IDMyMCAwIDI0MC0xODkgMzI1LTQzOSAzMjktMjQ1IDQtNDEwLTY5LTQ1NC0yNjhsMTU5LTMxYzI0IDEzMyAxMzYgMTY4IDI5NSAxNjUgMTQ0LTIgMjcwLTMxIDI3MC0xNzEgMC0xNjQtMTk1LTE2MC0zMzEtMjAyLTE2Ny01Mi0zNTAtODctMzUwLTI5OSAwLTIxOCAxNzMtMzE1IDQxMy0zMTMgMjIwIDIgMzczIDc3IDQxMiAyNjAiIGlkPSJvIi8+PHBhdGggZD0iTTYxNS0xMTAyYzM0MyAwIDQ4NCAyMDMgNDgyIDU2MC0xIDM0Ny0xNDcgNTYyLTQ4OCA1NjItMzM2IDAtNDc1LTIxOS00NzktNTYyLTQtMzQ5IDE1Ni01NjAgNDg1LTU2MHptLTggOTg5YzI0MCAwIDMwMS0xODAgMzAxLTQyOSAwLTI0NS01NS00MjctMjkwLTQyNy0yMzYgMC0yOTkgMTgxLTI5OSA0MjcgMCAyNDMgNjEgNDI5IDI4OCA0MjkiIGlkPSJwIi8+PHBhdGggZD0iTTY4MiAxNmMtMjA5IDAtMzIzLTgwLTMyNC0yODV2LTY3MUgxOTB2LTE0MmgxNzBsNTgtMjgyaDEyMHYyODJoNDMydjE0Mkg1Mzh2NjUyYzIgMTE0IDYwIDE1NSAxODIgMTU1IDEwNiAwIDIwOS0xNiAyOTctMzR2MTM3QzkyMS00IDgwNiAxNiA2ODIgMTYiIGlkPSJxIi8+PHBhdGggZD0iTTkwNC0xMTAyYzE5OSAwIDIyMCAxNzcgMjIwIDM4MVYwSDk1NnYtNjg2Yy0zLTExNCAwLTIxNS02MC0yNjQtNzAtMzMtMTI1LTQtMTU4IDcxLTI2IDU2LTM5IDE0MC0zOSAyNTJWMEg1MzF2LTY4NmMtMy0xMTQtMS0yMTUtNjEtMjY0LTc4LTQxLTEzNiAyNC0xNTcgODQtMjQgNjktMzkgMTU5LTM5IDI1OVYwSDEwNWMtMy0zNjAgNi03MzItNi0xMDgyaDE0OWM2IDUwIDMgMTIzIDggMTc1IDM2LTEwMCA4My0xOTUgMjE2LTE5NSAxMzUgMCAxNjYgNzkgMTk2IDE5NiA0Mi0xMDUgOTMtMTk2IDIzNi0xOTYiIGlkPSJyIi8+PHBhdGggZD0iTTEwMDAtMjcyYzMgOTUgMTIgMTU5IDEwMSAxNjEgMjEgMCA0MS0zIDU5LTdWLTZjLTQ0IDEwLTg2IDE2LTEzOSAxNi0xNDEgMi0xOTEtODQtMTk3LTIxN2gtNkM3NDgtNzYgNjQ4IDIwIDQ0NiAyMGMtMjA3IDAtMzE4LTEyMC0zMTgtMzIyIDAtMjY2IDE5NC0zNDggNDU0LTM1NGwyMzYtNGMxMi0xOTEtNDAtMzA1LTIyMi0zMDUtMTQwIDAtMjIwIDQ3LTIzMiAxNzJsLTE4OC0xN2MzMy0yMDQgMTgxLTI5MiA0MjMtMjkyIDI1NSAwIDQwMSAxMTggNDAxIDM2NHY0NjZ6bS02ODMtMjdjMCAxMDkgNjMgMTg0IDE3NSAxODIgMTY2LTMgMjU5LTk2IDMwNi0yMTcgMjQtNjUgMjAtMTIwIDIwLTIwMC0yMzIgNy01MDEtMjgtNTAxIDIzNSIgaWQ9InMiLz48cGF0aCBkPSJNOTMyIDBMNjExLTQ0NCAyODggMEg5NGw0MTUtNTU2LTM5Ny01MjZoMTk5bDMwMCA0MjEgMjk4LTQyMWgyMDFMNzEzLTU1OCAxMTMzIDBIOTMyIiBpZD0idCIvPjxnIGlkPSJjIj48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDAsMCkiIHhsaW5rOmhyZWY9IiNvIi8+PHVzZSB0cmFuc2Zvcm09Im1hdHJpeCgwLjAwODc4OTA2MjUsMCwwLDAuMDA4Nzg5MDYyNSwxMC44MDE3NTc4MTI1LDApIiB4bGluazpocmVmPSIjcCIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsMjEuNjAzNTE1NjI1LDApIiB4bGluazpocmVmPSIjbiIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsMzIuNDA1MjczNDM3NSwwKSIgeGxpbms6aHJlZj0iI3EiLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDQzLjIwNzAzMTI1LDApIiB4bGluazpocmVmPSIjciIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsNTQuMDA4Nzg5MDYyNSwwKSIgeGxpbms6aHJlZj0iI3MiLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDY0LjgxMDU0Njg3NSwwKSIgeGxpbms6aHJlZj0iI3QiLz48L2c+PHBhdGggZD0iTTYyMS01MzBjMCA0MjEgMTI3IDcwMCAzMDEgOTU1SDY0MUM0NjYgMTczIDM0NC0xMTAgMzQ0LTUzMXMxMjItNzAyIDI5Ny05NTNoMjgxYy0xNzQgMjU0LTMwMSA1MzQtMzAxIDk1NCIgaWQ9InUiLz48cGF0aCBkPSJNMTExMy0yNzRDMTAzOC05NiA4ODQgMjAgNjI2IDIwYy0zNTIgMC01MjYtMjA5LTUyNi01NjYgMC0zNTggMTg1LTU1NiA1MzAtNTU2IDM1OCAwIDQ5NiAyNTEgNDk5IDYxNUgzOTVjNSAxODAgNjggMzE5IDI0NSAzMTkgMTAyIDAgMTg2LTQ1IDIwOC0xMjl6TTg1Ny02NjNjLTYtMTc3LTEyOC0zMDctMzE1LTI0Ny05NCAzMC0xNDEgMTIxLTE0NSAyNDdoNDYwIiBpZD0idiIvPjxwYXRoIGQ9Ik03NjgtMTEwM2MyNDcgMCAzMzYgMTY3IDMzNiA0MTZWMEg4MjR2LTYxOWMtMy0xNjQtMjQtMjczLTE3MS0yNzMtMTY0IDAtMjI5IDE0Mi0yMjkgMzEyVjBIMTQzYy00LTM1OSA5LTczNi04LTEwODJoMjY4YzcgNjMgMTIgMTQ2IDEzIDIxNWg0YzY3LTE0NSAxNjAtMjM2IDM0OC0yMzYiIGlkPSJ3Ii8+PHBhdGggZD0iTTExMDktMzYwQzEwNjYtMTIyIDkwMiAyMCA2MjQgMjBjLTM0NyAwLTUxNC0yMDktNTE0LTU1NSAwLTM1MyAxNjgtNTY3IDUxOC01NjcgMjcwIDAgNDI1IDEzNiA0NzMgMzYxbC0yODMgMTRjLTE2LTEwNi03NS0xODMtMTk2LTE4Mi03NSAwLTEzMCAzMS0xNjUgOTNzLTUyIDE1Mi01MiAyNzBjMCAyNDkgNzQgMzc0IDIyMSAzNzQgMTIwIDAgMTg4LTgyIDIwMS0yMDEiIGlkPSJ4Ii8+PHBhdGggZD0iTTYxNi0xMTAyYzM2MSAwIDUyMSAyMDIgNTIxIDU2MCAwIDM1NC0xNzEgNTYyLTUyNyA1NjJDMjU5IDIwIDkyLTE5MiA5Mi01NDJjMC0zNTEgMTY5LTU2MCA1MjQtNTYwem0tOSA5MzBjMjAwLTIgMjM2LTE2NSAyMzYtMzcwIDAtMTk5LTI3LTM2Ny0yMjMtMzY3LTE5NSAwLTIzMyAxNDYtMjMzIDM2NyAwIDEyOSAxOCAyMjMgNTUgMjgyczkyIDg4IDE2NSA4OCIgaWQ9InkiLz48cGF0aCBkPSJNODAyLTkwOWMtMy0xODktMi0zODMtMi01NzVoMjgxdjEyNDhjLTEgODMgNCAxNjggOCAyMzZIODE3Yy05LTUwLTE0LTExNy0xNS0xNzZoLTRDNzM1LTQ5IDY1MSAyMCA0ODIgMjAgMTkxIDIwIDk3LTIzOCA5Ny01MzljMC0yNDMgNjYtNDMyIDIyMS01MTkgNTAtMjggMTA4LTQyIDE3NS00MiAxNjIgMiAyNTMgNzggMzA5IDE5MXptLTcwIDY1OGM5MS0xMTMgOTMtNDYyIDItNTg0LTQ2LTYxLTE0NS0xMDAtMjI0LTUzLTk2IDU2LTExOSAxOTAtMTIwIDM0OSAwIDEyMyAxNyAyMTQgNTIgMjc1IDQ2IDgyIDEzMCAxMTUgMjIyIDc0IDI1LTEyIDQ2LTMzIDY4LTYxIiBpZD0ieiIvPjxwYXRoIGQ9Ik04NzUtMTEwMmM3MCAwIDE0OCA3IDIwNiAxN3YyMzdjLTExMC0xNy0yNjctMzctMzU5IDE1LTEyMyA2OS0xOTIgMjA3LTE5MiAzOTVWMEgyNTBjLTEwLTM2NyAzMi03ODktNTItMTA4MmgyNzFjMTkgNjkgMzYgMTQ5IDQ2IDIyOGg0YzUyLTE1MSAxNTEtMjUwIDM1Ni0yNDgiIGlkPSJBIi8+PHBhdGggZD0iTTg4NS01MzFjMCA0MjEtMTIyIDcwNC0yOTcgOTU2SDMwN2MxNzQtMjU1IDMwMS01MzQgMzAxLTk1NSAwLTQyMC0xMjctNzAwLTMwMS05NTRoMjgxYzE3NSAyNTEgMjk3IDUzMiAyOTcgOTUzIiBpZD0iQiIvPjxnIGlkPSJkIj48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDAsMCkiIHhsaW5rOmhyZWY9IiN1Ii8+PHVzZSB0cmFuc2Zvcm09Im1hdHJpeCgwLjAwODc4OTA2MjUsMCwwLDAuMDA4Nzg5MDYyNSwxMC44MDE3NTc4MTI1LDApIiB4bGluazpocmVmPSIjdiIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsMjEuNjAzNTE1NjI1LDApIiB4bGluazpocmVmPSIjdyIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsMzIuNDA1MjczNDM3NSwwKSIgeGxpbms6aHJlZj0iI3giLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDQzLjIwNzAzMTI1LDApIiB4bGluazpocmVmPSIjeSIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsNTQuMDA4Nzg5MDYyNSwwKSIgeGxpbms6aHJlZj0iI3oiLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDY0LjgxMDU0Njg3NSwwKSIgeGxpbms6aHJlZj0iI3YiLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDc1LjYxMjMwNDY4NzUsMCkiIHhsaW5rOmhyZWY9IiNBIi8+PHVzZSB0cmFuc2Zvcm09Im1hdHJpeCgwLjAwODc4OTA2MjUsMCwwLDAuMDA4Nzg5MDYyNSw4Ni40MTQwNjI1LDApIiB4bGluazpocmVmPSIjQiIvPjwvZz48cGF0aCBkPSJNOTM0LTExMDJjMTk5IDAgMjIwIDE3NyAyMjAgMzgxVjBIOTI2Yy0yLTI2MCAxMC01MzMtNC03ODUtNC03Mi0xNC0xMzgtODMtMTM4LTM0IDAtNjEgMjctODEgODFzLTI5IDEzNC0yOSAyNDFWMEg1MDFjLTItMjYwIDktNTMzLTUtNzg1LTQtNzItMTMtMTM4LTgyLTEzOC00NiAwLTY1IDQ3LTgwIDkzLTIxIDY3LTMwIDE1NS0zMCAyNDhWMEg3NWMtMy0zNjAgNi03MzItNi0xMDgyaDIwOWM2IDUwIDMgMTIzIDggMTc1IDM2LTEwMCA4My0xOTUgMjE2LTE5NSAxMzUgMCAxNjYgNzkgMTk2IDE5NiA0Mi0xMDUgOTMtMTk2IDIzNi0xOTYiIGlkPSJDIi8+PHBhdGggZD0iTTcxMSAxM2MtMjUwIDAtMzg2LTk5LTM4NS0zNDlsMi01NTZIMTYxdi0xOTBoMTgxbDg4LTI4MmgxNzZ2MjgyaDM4NXYxOTBINjA2djUzMGMtMTAgMTMwIDQ1IDE4NiAxNzQgMTg0IDkxLTIgMTc2LTExIDI1NS0yN3YxODZDOTMzLTEgODMzIDEzIDcxMSAxMyIgaWQ9IkQiLz48ZyBpZD0iZSI+PHVzZSB0cmFuc2Zvcm09Im1hdHJpeCgwLjAxMTkzNTc2Mzg4ODg4ODg5LDAsMCwwLjAxMTkzNTc2Mzg4ODg4ODg5LDAsMCkiIHhsaW5rOmhyZWY9IiN4Ii8+PHVzZSB0cmFuc2Zvcm09Im1hdHJpeCgwLjAxMTkzNTc2Mzg4ODg4ODg5LDAsMCwwLjAxMTkzNTc2Mzg4ODg4ODg5LDE0LjY2OTA1MzgxOTQ0NDQ0NiwwKSIgeGxpbms6aHJlZj0iI3kiLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDExOTM1NzYzODg4ODg4ODksMCwwLDAuMDExOTM1NzYzODg4ODg4ODksMjkuMzM4MTA3NjM4ODg4ODkzLDApIiB4bGluazpocmVmPSIjQyIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMTE5MzU3NjM4ODg4ODg4OSwwLDAsMC4wMTE5MzU3NjM4ODg4ODg4OSw0NC4wMDcxNjE0NTgzMzMzNCwwKSIgeGxpbms6aHJlZj0iI0MiLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDExOTM1NzYzODg4ODg4ODksMCwwLDAuMDExOTM1NzYzODg4ODg4ODksNTguNjc2MjE1Mjc3Nzc3Nzg2LDApIiB4bGluazpocmVmPSIjdiIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMTE5MzU3NjM4ODg4ODg4OSwwLDAsMC4wMTE5MzU3NjM4ODg4ODg4OSw3My4zNDUyNjkwOTcyMjIyMywwKSIgeGxpbms6aHJlZj0iI3ciLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDExOTM1NzYzODg4ODg4ODksMCwwLDAuMDExOTM1NzYzODg4ODg4ODksODguMDE0MzIyOTE2NjY2NjcsMCkiIHhsaW5rOmhyZWY9IiNEIi8+PC9nPjxnIGlkPSJmIj48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDAsMCkiIHhsaW5rOmhyZWY9IiN1Ii8+PHVzZSB0cmFuc2Zvcm09Im1hdHJpeCgwLjAwODc4OTA2MjUsMCwwLDAuMDA4Nzg5MDYyNSwxMC44MDE3NTc4MTI1LDApIiB4bGluazpocmVmPSIjeiIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsMjEuNjAzNTE1NjI1LDApIiB4bGluazpocmVmPSIjdiIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsMzIuNDA1MjczNDM3NSwwKSIgeGxpbms6aHJlZj0iI3giLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDQzLjIwNzAzMTI1LDApIiB4bGluazpocmVmPSIjeSIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsNTQuMDA4Nzg5MDYyNSwwKSIgeGxpbms6aHJlZj0iI3oiLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDY0LjgxMDU0Njg3NSwwKSIgeGxpbms6aHJlZj0iI3YiLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDc1LjYxMjMwNDY4NzUsMCkiIHhsaW5rOmhyZWY9IiNBIi8+PHVzZSB0cmFuc2Zvcm09Im1hdHJpeCgwLjAwODc4OTA2MjUsMCwwLDAuMDA4Nzg5MDYyNSw4Ni40MTQwNjI1LDApIiB4bGluazpocmVmPSIjQiIvPjwvZz48cGF0aCBkPSJNNzc1LTE5M0M3MDgtNzUgNjIxIDIwIDQzOSAyMCAyMjYgMjAgMTA2LTkzIDEwNi0zMDZjMC0yNTEgMTcwLTM0MSA0MTctMzQ2bDIyMy00YzQtMTQzLTEyLTI2NC0xNDgtMjY0LTEwMyAwLTE0MSA1OC0xNTAgMTUzbC0yOTMtMTRjMzctMjE5IDE5NC0zMjEgNDU1LTMyMSAyNjMgMCA0MTggMTMwIDQxNyAzOTB2MzkyYzEgODQgNSAxNjEgODkgMTYwIDIxIDAgNDItMiA2Mi02djE1MmMtNTcgMTUtMTA4IDI3LTE4MCAyNi0xNDUgMC0yMDItNzMtMjE3LTIwNWgtNnptLTI1NiAxN2MxNjggMCAyMzQtMTM5IDIyNy0zMjUtMTY2IDQtMzQ3LTI1LTM0NyAxNzMgMCA5NCAzMyAxNTIgMTIwIDE1MiIgaWQ9IkUiLz48cGF0aCBkPSJNNzk0LTE5MGgzNTNWMEgxMTh2LTE5MGgzOTV2LTcwMkgyMjN2LTE5MGg1NzF2ODkyek01MTMtMTI3N3YtMjA3aDI4MXYyMDdINTEzIiBpZD0iRiIvPjxnIGlkPSJnIj48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDAsMCkiIHhsaW5rOmhyZWY9IiN1Ii8+PHVzZSB0cmFuc2Zvcm09Im1hdHJpeCgwLjAwODc4OTA2MjUsMCwwLDAuMDA4Nzg5MDYyNSwxMC44MDE3NTc4MTI1LDApIiB4bGluazpocmVmPSIjRSIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsMjEuNjAzNTE1NjI1LDApIiB4bGluazpocmVmPSIjRCIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsMzIuNDA1MjczNDM3NSwwKSIgeGxpbms6aHJlZj0iI0QiLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDQzLjIwNzAzMTI1LDApIiB4bGluazpocmVmPSIjdiIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsNTQuMDA4Nzg5MDYyNSwwKSIgeGxpbms6aHJlZj0iI3ciLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDY0LjgxMDU0Njg3NSwwKSIgeGxpbms6aHJlZj0iI0QiLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDc1LjYxMjMwNDY4NzUsMCkiIHhsaW5rOmhyZWY9IiNGIi8+PHVzZSB0cmFuc2Zvcm09Im1hdHJpeCgwLjAwODc4OTA2MjUsMCwwLDAuMDA4Nzg5MDYyNSw4Ni40MTQwNjI1LDApIiB4bGluazpocmVmPSIjeSIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsOTcuMjE1ODIwMzEyNSwwKSIgeGxpbms6aHJlZj0iI3ciLz48L2c+PHBhdGggZD0iTTQyNC0xNDg0Yy0yIDIwNiA1IDQyNC04IDYxOGg0YzY3LTE0NSAxNjAtMjM0IDM0OC0yMzYgMjUxLTMgMzM1IDE2NyAzMzYgNDE2VjBIODI0di02MjFjMC0xNTAtMjMtMjc2LTE3My0yNzAtMTYzIDctMjI3IDE0Mi0yMjcgMzEyVjBIMTQzdi0xNDg0aDI4MSIgaWQ9IkciLz48cGF0aCBkPSJNNjIyLTkxNmMtMTEyIDItMjAwIDE0LTIwMCAxMTEgMCA5MCA4OSAxMDEgMTY4IDEyMSAyMzQgNTggNTE0IDc3IDUxNCAzNjhDMTEwNC00IDc1MSA1NCA0MzQgNSAyNjctMjEgMTYxLTExMiAxMjEtMjcwbDI0Ny0zN2MyNCAxMTEgMTEzIDE0NSAyNTIgMTQxIDExNC0zIDIyNi0xMiAyMjYtMTI0IDAtMTAxLTEwMy0xMTMtMTkyLTEzNC0yMjUtNTQtNDkwLTc2LTQ5MC0zNTEgMC0yNDcgMjAxLTMyOCA0NTgtMzI4IDI0NSAwIDQxMyA4OCA0NTcgMjkybC0yNDkgMjZjLTE2LTk4LTkyLTEzMy0yMDgtMTMxIiBpZD0iSCIvPjxnIGlkPSJoIj48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDAsMCkiIHhsaW5rOmhyZWY9IiNDIi8+PHVzZSB0cmFuc2Zvcm09Im1hdHJpeCgwLjAwODc4OTA2MjUsMCwwLDAuMDA4Nzg5MDYyNSwxMC44MDE3NTc4MTI1LDApIiB4bGluazpocmVmPSIjdiIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsMjEuNjAzNTE1NjI1LDApIiB4bGluazpocmVmPSIjeCIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsMzIuNDA1MjczNDM3NSwwKSIgeGxpbms6aHJlZj0iI0ciLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDQzLjIwNzAzMTI1LDApIiB4bGluazpocmVmPSIjRSIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsNTQuMDA4Nzg5MDYyNSwwKSIgeGxpbms6aHJlZj0iI3ciLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDY0LjgxMDU0Njg3NSwwKSIgeGxpbms6aHJlZj0iI0YiLz48dXNlIHRyYW5zZm9ybT0ibWF0cml4KDAuMDA4Nzg5MDYyNSwwLDAsMC4wMDg3ODkwNjI1LDc1LjYxMjMwNDY4NzUsMCkiIHhsaW5rOmhyZWY9IiNIIi8+PHVzZSB0cmFuc2Zvcm09Im1hdHJpeCgwLjAwODc4OTA2MjUsMCwwLDAuMDA4Nzg5MDYyNSw4Ni40MTQwNjI1LDApIiB4bGluazpocmVmPSIjQyIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsOTcuMjE1ODIwMzEyNSwwKSIgeGxpbms6aHJlZj0iI0IiLz48L2c+PHBhdGggZD0iTTEwNDgtMzJjLTIgMzAwLTEzNSA0NTYtNDMzIDQ1Ni0yMjItMS0zNTgtODgtNDAwLTI2N2wxODQtMjVjMjIgOTkgMTAwIDE1NyAyMjIgMTU2IDE4NC0yIDI0OC0xMjUgMjQ4LTMxNSAwLTY0IDMtMTMzLTItMTk0QzgwNy0xMDAgNzA2LTEzIDUyNC0xMmMtMzA2IDAtMzgxLTIyOC0zODEtNTM3IDAtMzE4IDg1LTU1MCA0MDAtNTUwIDE2NCAwIDI3MSA4MyAzMjUgMjAyaDNjMS02MCAzLTEzNCAxMi0xODVoMTcxYy0xMyAzMzktNCA3MDItNiAxMDUwek01ODUtMTQ1YzIxMC04IDI4NC0xNzggMjg0LTQwNiAwLTE5Mi01Mi0zMzEtMTc3LTM5Mi0zMy0xNi02OS0yMi0xMDQtMjItMjIzIDItMjU5IDE4NC0yNTkgNDE0IDAgMjI5IDMxIDQxNSAyNTYgNDA2IiBpZD0iSSIvPjx1c2UgdHJhbnNmb3JtPSJtYXRyaXgoMC4wMDg3ODkwNjI1LDAsMCwwLjAwODc4OTA2MjUsMCwwKSIgeGxpbms6aHJlZj0iI0kiIGlkPSJpIi8+PC9kZWZzPjwvZz48L3N2Zz4=)\n",
        "\n",
        "ref: https://guillaumegenthial.github.io/sequence-to-sequence.html"
      ],
      "metadata": {
        "id": "5x6nSO3J0Kvy"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Encoder layer:"
      ],
      "metadata": {
        "id": "P280QZRsrEaC"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9Kb4esWRNSUg"
      },
      "outputs": [],
      "source": [
        "class Encoder(tf.keras.layers.Layer):\n",
        "    '''class that enocodes input sentence using lstm and returns encoded vector of shape [batch_size,input_length,lstm_units]'''\n",
        "    def __init__(self, vocab_size, embedding_dim_enc, input_length, enc_units, dropout=0.0, recurrent_dropout=0.0):\n",
        "      super().__init__()\n",
        "      self.vocab_size = vocab_size\n",
        "      self.embedding_dim_enc = embedding_dim_enc\n",
        "      self.input_length = input_length\n",
        "      self.enc_units= enc_units\n",
        "      self.dropout = dropout \n",
        "      self.recurrent_dropout = recurrent_dropout \n",
        "\n",
        "    def get_config(self):\n",
        "      config = super().get_config()\n",
        "      config.update({'vocab_size': self.vocab_size, 'embedding_dim_enc': self.embedding_dim_enc,\\\n",
        "                     'input_length': self.input_length, 'enc_units': self.enc_units})\n",
        "      return config\n",
        "\n",
        "    def build(self, input_shape):\n",
        "      self.embedding = Embedding(input_dim=self.vocab_size, output_dim=self.embedding_dim_enc, input_length=self.input_length,\n",
        "                          mask_zero=True, name=\"embedding_layer_encoder\") \n",
        "      self.lstm = LSTM(self.enc_units, return_state=True, return_sequences=True,dropout=self.dropout,\\\n",
        "                        recurrent_dropout=self.recurrent_dropout ,name=\"Encoder_LSTM\")\n",
        "      \n",
        "    def call(self, input_sentances, training=True):\n",
        "      '''input sentence is embeded and then passed to lstm'''\n",
        "      input_embedd = self.embedding(input_sentances) # [b, max_len, embed-size]\n",
        "      self.lstm_output, _, _ = self.lstm(input_embedd) # [b, max_len, lstm-units], \n",
        "      return self.lstm_output\n",
        "    \n",
        "    def initialize_states(self,batch_size):\n",
        "      '''Given a batch size it will return intial hidden state and intial cell state'''\n",
        "      return tf.zeros([batch_size, self.enc_units ]), tf.zeros([batch_size, self.enc_units ])\n",
        "\n",
        "    def get_states(self):\n",
        "      return self.lstm_state_h, self.lstm_state_c"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# checking code\n",
        "\n",
        "vocab_size = 150\n",
        "embedding_dim_enc = 50\n",
        "input_length = 10\n",
        "enc_units= 32\n",
        "dropout = 0.3\n",
        "recurrent_dropout = 0.3\n",
        "batch_size = 16\n",
        "\n",
        "input_sentances = tf.random.uniform(shape=[batch_size,input_length])\n",
        "\n",
        "e = Encoder(vocab_size, embedding_dim_enc, input_length, enc_units, dropout, recurrent_dropout)\n",
        "e(input_sentances).shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WabI2lairy-n",
        "outputId": "b46752f1-4789-4c89-8d8f-dde1b49ff04b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([16, 10, 32])"
            ]
          },
          "metadata": {},
          "execution_count": 118
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Attention Mechanism Layer:"
      ],
      "metadata": {
        "id": "qAjT4vcjrb52"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ab5SNdPZLlur"
      },
      "outputs": [],
      "source": [
        "class AttentionMechanism(tf.keras.layers.Layer):\n",
        "  '''Class the calculates attention weights and corresponding weighted vector using simple dot product operation.'''\n",
        "  def __init__(self, initializer = tf.keras.initializers.GlorotUniform() ):\n",
        "    super().__init__()\n",
        "    self.initializer = initializer\n",
        "\n",
        "  def get_config(self):\n",
        "    config = super().get_config()\n",
        "    config.update({'initializer': self.initializer })\n",
        "    return config\n",
        "\n",
        "  def call(self,decoder_hidden_state,encoder_output):\n",
        "    '''decoder input is transformed to match encoder output dimension and attention weights are calculated\n",
        "    based on similarity using dot products and weighted sum of encoder hidden state vector is returned as context vector\n",
        "    to be used by decoder'''\n",
        "    initializer = self.initializer\n",
        "    # initializing decoder transformation matrix\n",
        "    values = initializer(shape=(decoder_hidden_state.shape[2],encoder_output.shape[2])) # [dec_embed_dim, encoder_lstm_units]\n",
        "    # tranforming decoder input\n",
        "    similarity1 = tf.matmul(decoder_hidden_state,values) # [b,1,dec_embed_dim] X [dec_embed_dim, encoder_lstm_units] = [b,1,encoder_lstm_units]\n",
        "    # finding similarity score\n",
        "    similarity = tf.matmul(similarity1,encoder_output, transpose_b=True) # [b,1,encoder_lstm_units] X [b,encoder_lstm_units,max_len] = [b,1,max_len]\n",
        "    # normalizing scores using softmax\n",
        "    attn_weights = tf.nn.softmax( similarity,axis=-1 ) # [b,1,max_len]\n",
        "    # calculating weighted sum\n",
        "    context_vector = tf.matmul(attn_weights,encoder_output) # [b,1,max_len] X -[b,max_len,encoder_lstm_units] = [b,1,encoder_lstm_units]\n",
        "    context_vector = tf.squeeze(context_vector,axis=1) # [b,encoder_lstm_units]\n",
        "    attn_weights = tf.transpose(attn_weights, perm=[0, 2, 1]) # [b,max_len,1]\n",
        "    return context_vector, attn_weights"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OrxpKs1LJ7T1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f846e084-ffb2-426c-d029-0f1b6da60189"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(TensorShape([16, 32]), TensorShape([16, 10, 1]))"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ],
      "source": [
        "# checking code\n",
        "\n",
        "input_length=10\n",
        "batch_size=16\n",
        "encoder_lstm_units=32\n",
        "dec_embed_dim=50\n",
        "\n",
        "decoder_hidden_state=tf.random.uniform(shape=[batch_size,1,dec_embed_dim])\n",
        "encoder_output=tf.random.uniform(shape=[batch_size,input_length,encoder_lstm_units])\n",
        "\n",
        "attn=AttentionMechanism()\n",
        "c,wt = attn(decoder_hidden_state,encoder_output)\n",
        "c.shape , wt.shape"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Decoder Encoder Cross Attention:"
      ],
      "metadata": {
        "id": "ChIW-4Y_sl5D"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b7BPvFlmoPVC"
      },
      "outputs": [],
      "source": [
        "class DecoderEncoderCrossAttention(tf.keras.Model):\n",
        "  '''class that performs cross attention on decoder input, pass the attention updated input to\n",
        "  lstm and then to final dense layer having units equal to output vocab size. the decoder input is passed\n",
        "  one word at a time over batch'''\n",
        "  def __init__(self,tar_vocab_size, embedding_dim_dec, input_length, dec_units, lstm_dropout, recurrent_dropout, trainable=True):\n",
        "    super().__init__()\n",
        "    self.tar_vocab_size = tar_vocab_size\n",
        "    self.embedding_dim_dec = embedding_dim_dec\n",
        "    self.input_length = input_length\n",
        "    self.dec_units = dec_units\n",
        "    self.lstm_dropout = lstm_dropout\n",
        "    self.recurrent_dropout = recurrent_dropout \n",
        "    self.trainable = True\n",
        "\n",
        "  def get_config(self):\n",
        "      config = super().get_config()\n",
        "      config.update({\n",
        "          'tar_vocab_size': self.tar_vocab_size, 'embedding_dim_dec': self.embedding_dim_dec,\n",
        "          'input_length': self.input_length, 'dec_units': self.dec_units, 'trainable': self.trainable})\n",
        "      return config\n",
        "\n",
        "  def build(self, input_shape):\n",
        "    if self.trainable:\n",
        "      self.embedding = Embedding(input_dim=self.tar_vocab_size, output_dim=self.embedding_dim_dec, input_length=self.input_length,\n",
        "                        mask_zero=True, name=\"embedding_layer_decoder\", trainable=True)\n",
        "    else:\n",
        "      self.embedding = Embedding(input_dim=self.tar_vocab_size, output_dim=self.embedding_dim_dec, input_length=self.input_length,\n",
        "                        mask_zero=True, name=\"embedding_layer_decoder\", weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    self.lstm = LSTM(self.dec_units, return_sequences=True, return_state=True,dropout=self.lstm_dropout,recurrent_dropout=self.recurrent_dropout, name=\"Encoder_LSTM\")\n",
        "    self.attention = AttentionMechanism()\n",
        "    self.dense_layer = Dense(self.tar_vocab_size,activation=None)\n",
        "\n",
        "  def call(self,input_to_decoder, encoder_output): \n",
        "    '''takes decoder single input over batch, encoder outputs, performs cross attention and returns returns logits'''\n",
        "    # embedding decoder single input\n",
        "    target_embedd = self.embedding(input_to_decoder) # [b,1,embedding_dim_dec]\n",
        "    # getting attention updated embedding vector\n",
        "    context_vector,_ = self.attention(target_embedd,encoder_output) #[b,encoder_lstm_units]\n",
        "    # concataneting embeded input and attention updated input\n",
        "    concat_input = tf.concat([target_embedd, tf.expand_dims(context_vector, 1)], -1)  # [b,1,(embedding_dim_dec+encoder_lstm_units)] \n",
        "    # passing to lstm\n",
        "    self.lstm_output, self.lstm_state_h, self.lstm_state_c = self.lstm(concat_input) # [b,1,dec_lstm_units] , [b,1,dec_lstm_units] , [b,1,dec_lstm_units]\n",
        "    # getting logits\n",
        "    output = self.dense_layer(self.lstm_output)  # [b,1,tar_vocab_size]\n",
        "    output = tf.squeeze(output,axis=1) # [b,tar_vocab_size]\n",
        "    return output, self.lstm_state_h,self.lstm_state_c, context_vector                                                   "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "88okyUfSPUXa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7cafe97e-bf6a-4cc3-e27c-5681f38077a4"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([16, 150])"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ],
      "source": [
        "# checking code\n",
        "\n",
        "tar_vocab_size=150\n",
        "embedding_dim_dec=50\n",
        "input_length=10\n",
        "dec_units=48\n",
        "batch_size=16\n",
        "lstm_dropout = 0.5\n",
        "recurrent_dropout=0.5\n",
        "\n",
        "input_to_decoder=tf.random.uniform(shape=(batch_size,1),maxval=10,minval=0,dtype=tf.int32)\n",
        "encoder_output=tf.random.uniform(shape=[batch_size,input_length,32])\n",
        "\n",
        "d = DecoderEncoderCrossAttention(tar_vocab_size,embedding_dim_dec,input_length,dec_units,lstm_dropout,recurrent_dropout)\n",
        "o,_,_,_ = d(input_to_decoder,encoder_output)\n",
        "o.shape"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Decoder Layer:"
      ],
      "metadata": {
        "id": "IphUR5eos1bB"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8Jduzh3ZP7sm"
      },
      "outputs": [],
      "source": [
        "class DecoderBlock(tf.keras.Model):\n",
        "  '''class which gives logits values for full decoder input length '''\n",
        "  def __init__(self,out_vocab_size, embedding_dim_dec, input_length, dec_units,lstm_dropout,recurrent_dropout):\n",
        "    super().__init__()\n",
        "    self.out_vocab_size = out_vocab_size\n",
        "    self.embedding_dim_dec = embedding_dim_dec\n",
        "    self.input_length = input_length\n",
        "    self.dec_units = dec_units\n",
        "    self.lstm_dropout = lstm_dropout\n",
        "    self.recurrent_dropout = recurrent_dropout\n",
        "\n",
        "  def get_config(self):\n",
        "    config = super().get_config()\n",
        "    config.update({\n",
        "        'out_vocab_size': self.out_vocab_size, 'embedding_dim_dec': self.embedding_dim_dec,\n",
        "        'input_length': self.input_length, 'dec_units': self.dec_units})\n",
        "    return config\n",
        "\n",
        "  def build(self,input_shapes):\n",
        "    self.crossattention = DecoderEncoderCrossAttention(self.out_vocab_size, self.embedding_dim_dec, self.input_length,\\\n",
        "                                                        self.dec_units ,self.lstm_dropout,self.recurrent_dropout,False)\n",
        "\n",
        "  def call(self, input_to_decoder,encoder_output):\n",
        "    # creating a empty array of length equal to input length to fill logits value\n",
        "    all_outputs = tf.TensorArray(tf.float32,size=self.input_length)\n",
        "    # iterating over individual input word\n",
        "    for timestep in range(self.input_length):\n",
        "      # getting logits value for current input word\n",
        "      output, decoder_hidden_state,decoder_cell_state,_ = self.crossattention(input_to_decoder[:,timestep:timestep+1], encoder_output)\n",
        "      all_outputs = all_outputs.write(timestep, output) #[max_len,b,tar_vocab_size]\n",
        "    all_outputs = tf.transpose(all_outputs.stack(), [1,0,2]) #[b,max_len,tar_vocab_size]\n",
        "    return all_outputs "
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# checking code\n",
        "\n",
        "tar_vocab_size=150 \n",
        "embedding_dim_dec=50\n",
        "input_length=10\n",
        "dec_units=48\n",
        "enc_units = 32\n",
        "batch_size=16\n",
        "lstm_dropout=0.4\n",
        "recurrent_dropout=0.4\n",
        "\n",
        "input_to_decoder=tf.random.uniform(shape=(batch_size,input_length),maxval=10,minval=0,dtype=tf.int32)\n",
        "encoder_output=tf.random.uniform(shape=[batch_size,input_length,enc_units])\n",
        "\n",
        "d =  DecoderBlock(tar_vocab_size,embedding_dim_dec,input_length,dec_units,lstm_dropout,recurrent_dropout)\n",
        "d(input_to_decoder,encoder_output).shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KclSF4i_l6m-",
        "outputId": "942c1f5b-d899-471e-f772-dac3281ee7ab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([16, 10, 150])"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Model:"
      ],
      "metadata": {
        "id": "ImkfLXpts7CZ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iqypHJSjy9qD"
      },
      "outputs": [],
      "source": [
        "class TranslationModel(tf.keras.Model):\n",
        "\n",
        "  def __init__(self, encoder_inputs_length,decoder_inputs_length, vocab_size_ita,vocab_size_eng,\\\n",
        "                embedding_dim_enc,embedding_dim_dec,enc_units,dec_units,lstm_dropout,recurrent_dropout):\n",
        "    super().__init__()\n",
        "    self.encoder_inputs_length = encoder_inputs_length\n",
        "    self.decoder_inputs_length = decoder_inputs_length\n",
        "    self.vocab_size_ita = vocab_size_ita\n",
        "    self.vocab_size_eng = vocab_size_eng\n",
        "    self.embedding_dim_enc = embedding_dim_enc\n",
        "    self.embedding_dim_dec = embedding_dim_dec\n",
        "    self.enc_units = enc_units\n",
        "    self.dec_units = dec_units\n",
        "    self.lstm_dropout = lstm_dropout\n",
        "    self.recurrent_dropout = recurrent_dropout\n",
        "\n",
        "  def get_config(self):\n",
        "    config = super().get_config()\n",
        "    config.update({'encoder_inputs_length': self.encoder_inputs_length, 'decoder_inputs_length': self.decoder_inputs_length,\n",
        "        'vocab_size_ita': self.vocab_size_ita, 'vocab_size_eng': self.vocab_size_eng, 'embedding_dim_enc': self.embedding_dim_enc,\n",
        "        'embedding_dim_dec': self.embedding_dim_dec , 'enc_units': self.enc_units, 'dec_units': self.dec_units, 'att_units': self.att_units})\n",
        "    return config\n",
        "\n",
        "  def build(self,input_shapes):\n",
        "      self.encoder = Encoder(self.vocab_size_ita+1, self.embedding_dim_enc, self.encoder_inputs_length,\\\n",
        "                             self.enc_units,self.lstm_dropout,self.recurrent_dropout)\n",
        "      self.decoder = DecoderBlock(self.vocab_size_eng+1, self.embedding_dim_dec, decoder_inputs_length,\\\n",
        "                                  self.lstm_dropout,self.recurrent_dropout)\n",
        "      \n",
        "  def call(self, data):\n",
        "      input,output = data[0], data[1]\n",
        "      #passing input to encoder\n",
        "      encoder_output = self.encoder(input) #[b,max_len,encoder_lstm_units]\n",
        "      #passing output to decoder\n",
        "      decoder_output = self.decoder(output, encoder_output) #[b,max_len,tar_vocab_size] #logits\n",
        "      return decoder_output"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Loss Function:"
      ],
      "metadata": {
        "id": "Fidzi9Ams_ng"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z6AokS4N84R2"
      },
      "outputs": [],
      "source": [
        "# https://www.tensorflow.org/tutorials/text/image_captioning#model\n",
        "\n",
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n",
        "    from_logits=True, reduction='none')\n",
        "\n",
        "def loss_function(real, pred):\n",
        "    \"\"\" Custom loss function that will not consider the loss for padded zero.\"\"\"\n",
        "    mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
        "    loss_ = loss_object(real, pred)\n",
        "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
        "    loss_ *= mask\n",
        "    return tf.reduce_mean(loss_)\n",
        "\n",
        "def masked_acc(labels, preds):\n",
        "  mask = tf.cast(labels!=0, tf.float32)\n",
        "  preds = tf.argmax(preds, axis=-1)\n",
        "  labels = tf.cast(labels, tf.int64)\n",
        "  match = tf.cast(preds == labels, mask.dtype)\n",
        "  acc = tf.reduce_sum(match*mask)/tf.reduce_sum(mask)\n",
        "  return acc"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Callbacks:"
      ],
      "metadata": {
        "id": "xgdBekgItS9h"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from itertools import combinations\n",
        "import os\n",
        "import datetime\n",
        "log_dir = os.path.join('/content/drive/MyDrive/cross_attention/',\"logs\",'model','fits', datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n",
        "tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,histogram_freq=1,write_graph=True)\n",
        "\n",
        "modelsave_wt = tf.keras.callbacks.ModelCheckpoint(\n",
        "                     filepath='/content/drive/MyDrive/cross_attention/model.{epoch:02d}-{masked_acc:.4f}.h5', \n",
        "                     save_freq='epoch', verbose=1, monitor='masked_acc', \n",
        "                     save_weights_only=True, save_best_only=True\n",
        "                 ) "
      ],
      "metadata": {
        "id": "ByVgmrHNjWQO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Selected Hyperparameters:"
      ],
      "metadata": {
        "id": "jarUOOF8tveQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#vocab_size_ita=27260\n",
        "#vocab_size_eng=13356"
      ],
      "metadata": {
        "id": "9SKneqQJt6a0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JJ60N_XTB-80"
      },
      "outputs": [],
      "source": [
        "# after trying different values, following were selected\n",
        "\n",
        "encoder_inputs_length = 20\n",
        "decoder_inputs_length = 20\n",
        "vocab_size_ita = vocab_size_ita\n",
        "vocab_size_eng = vocab_size_eng\n",
        "embedding_dim_enc = 100\n",
        "embedding_dim_dec = 100\n",
        "enc_units = 128\n",
        "dec_units = 128\n",
        "lstm_dropout = 0.2\n",
        "recurrent_dropout = 0.2\n",
        "optimizer = tf.keras.optimizers.Adam()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Data Loader:"
      ],
      "metadata": {
        "id": "xhaBrnzxuFbY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size=1024\n",
        "\n",
        "train_dataset = Dataset(train, tknizer_ita, tknizer_eng, 20)\n",
        "val_dataset  = Dataset(val, tknizer_ita, tknizer_eng, 20)\n",
        "test_dataset  = Dataset(test, tknizer_ita, tknizer_eng, 20)\n",
        "\n",
        "train_dataloader = Dataloder(train_dataset, batch_size=batch_size)\n",
        "val_dataloader = Dataloder(val_dataset, batch_size=batch_size)\n",
        "test_dataloader = Dataloder(test_dataset, batch_size=batch_size)\n",
        "\n",
        "train_steps=train.shape[0]//batch_size\n",
        "valid_steps=val.shape[0]//batch_size"
      ],
      "metadata": {
        "id": "l5zitOnljwSA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Training:"
      ],
      "metadata": {
        "id": "o-9roTDYuJnE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model  = TranslationModel(encoder_inputs_length,decoder_inputs_length, vocab_size_ita,vocab_size_eng,embedding_dim_enc,\\\n",
        "                          embedding_dim_dec,enc_units,dec_units, lstm_dropout, recurrent_dropout)\n",
        "\n",
        "model.compile(optimizer=optimizer, loss=loss_function, metrics=[masked_acc])"
      ],
      "metadata": {
        "id": "bY44-saA-FNa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.fit(train_dataloader, steps_per_epoch=train_steps, epochs=10, validation_data=val_dataloader, validation_steps=valid_steps,\\\n",
        "          callbacks=[modelsave_wt,tensorboard_callback])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4cmHvCDqFgLN",
        "outputId": "797d1c33-7502-492b-fc92-51b2c6d5da45"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Layer Encoder_LSTM will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
            "WARNING:tensorflow:Layer Encoder_LSTM will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
            "/usr/local/lib/python3.8/dist-packages/keras/initializers/initializers_v2.py:120: UserWarning: The initializer GlorotUniform is unseeded and being called multiple times, which will return identical values  each time (even if the initializer is unseeded). Please update your code to provide a seed to the initializer, or avoid using the same initalizer instance more than once.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "296/296 [==============================] - ETA: 0s - loss: 1.9879 - masked_acc: 0.1504\n",
            "Epoch 1: masked_acc improved from 0.14619 to 0.15041, saving model to /content/drive/MyDrive/cross_attention/model.01-0.1504.h5\n",
            "296/296 [==============================] - 292s 748ms/step - loss: 1.9879 - masked_acc: 0.1504 - val_loss: 1.6638 - val_masked_acc: 0.1911\n",
            "Epoch 2/10\n",
            "296/296 [==============================] - ETA: 0s - loss: 1.5426 - masked_acc: 0.2328\n",
            "Epoch 2: masked_acc improved from 0.15041 to 0.23276, saving model to /content/drive/MyDrive/cross_attention/model.02-0.2328.h5\n",
            "296/296 [==============================] - 197s 665ms/step - loss: 1.5426 - masked_acc: 0.2328 - val_loss: 1.4218 - val_masked_acc: 0.2831\n",
            "Epoch 3/10\n",
            "296/296 [==============================] - ETA: 0s - loss: 1.3747 - masked_acc: 0.3093\n",
            "Epoch 3: masked_acc improved from 0.23276 to 0.30925, saving model to /content/drive/MyDrive/cross_attention/model.03-0.3093.h5\n",
            "296/296 [==============================] - 195s 659ms/step - loss: 1.3747 - masked_acc: 0.3093 - val_loss: 1.3202 - val_masked_acc: 0.3340\n",
            "Epoch 4/10\n",
            "296/296 [==============================] - ETA: 0s - loss: 1.2947 - masked_acc: 0.3424\n",
            "Epoch 4: masked_acc improved from 0.30925 to 0.34244, saving model to /content/drive/MyDrive/cross_attention/model.04-0.3424.h5\n",
            "296/296 [==============================] - 193s 652ms/step - loss: 1.2947 - masked_acc: 0.3424 - val_loss: 1.2543 - val_masked_acc: 0.3572\n",
            "Epoch 5/10\n",
            "296/296 [==============================] - ETA: 0s - loss: 1.2362 - masked_acc: 0.3634\n",
            "Epoch 5: masked_acc improved from 0.34244 to 0.36345, saving model to /content/drive/MyDrive/cross_attention/model.05-0.3634.h5\n",
            "296/296 [==============================] - 196s 661ms/step - loss: 1.2362 - masked_acc: 0.3634 - val_loss: 1.2015 - val_masked_acc: 0.3737\n",
            "Epoch 6/10\n",
            "296/296 [==============================] - ETA: 0s - loss: 1.1868 - masked_acc: 0.3822\n",
            "Epoch 6: masked_acc improved from 0.36345 to 0.38222, saving model to /content/drive/MyDrive/cross_attention/model.06-0.3822.h5\n",
            "296/296 [==============================] - 194s 656ms/step - loss: 1.1868 - masked_acc: 0.3822 - val_loss: 1.1559 - val_masked_acc: 0.3972\n",
            "Epoch 7/10\n",
            "296/296 [==============================] - ETA: 0s - loss: 1.1442 - masked_acc: 0.4000\n",
            "Epoch 7: masked_acc improved from 0.38222 to 0.40001, saving model to /content/drive/MyDrive/cross_attention/model.07-0.4000.h5\n",
            "296/296 [==============================] - 191s 645ms/step - loss: 1.1442 - masked_acc: 0.4000 - val_loss: 1.1166 - val_masked_acc: 0.4130\n",
            "Epoch 8/10\n",
            "296/296 [==============================] - ETA: 0s - loss: 1.1061 - masked_acc: 0.4158\n",
            "Epoch 8: masked_acc improved from 0.40001 to 0.41582, saving model to /content/drive/MyDrive/cross_attention/model.08-0.4158.h5\n",
            "296/296 [==============================] - 194s 654ms/step - loss: 1.1061 - masked_acc: 0.4158 - val_loss: 1.0793 - val_masked_acc: 0.4311\n",
            "Epoch 9/10\n",
            "296/296 [==============================] - ETA: 0s - loss: 1.0695 - masked_acc: 0.4324\n",
            "Epoch 9: masked_acc improved from 0.41582 to 0.43236, saving model to /content/drive/MyDrive/cross_attention/model.09-0.4324.h5\n",
            "296/296 [==============================] - 195s 658ms/step - loss: 1.0695 - masked_acc: 0.4324 - val_loss: 1.0414 - val_masked_acc: 0.4501\n",
            "Epoch 10/10\n",
            "296/296 [==============================] - ETA: 0s - loss: 1.0310 - masked_acc: 0.4501\n",
            "Epoch 10: masked_acc improved from 0.43236 to 0.45006, saving model to /content/drive/MyDrive/cross_attention/model.10-0.4501.h5\n",
            "296/296 [==============================] - 200s 677ms/step - loss: 1.0310 - masked_acc: 0.4501 - val_loss: 1.0026 - val_masked_acc: 0.4677\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f0db44b4fd0>"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.fit(train_dataloader, steps_per_epoch=train_steps, epochs=20, validation_data=val_dataloader, validation_steps=valid_steps,\\\n",
        "          callbacks=[modelsave_wt,tensorboard_callback],initial_epoch=10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mk98ChmsOvsf",
        "outputId": "9fe2f224-b9a6-4933-b659-d9d92b4f30a7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11/20\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.9896 - masked_acc: 0.4684\n",
            "Epoch 11: masked_acc improved from 0.45006 to 0.46839, saving model to /content/drive/MyDrive/cross_attention/model.11-0.4684.h5\n",
            "296/296 [==============================] - 197s 656ms/step - loss: 0.9896 - masked_acc: 0.4684 - val_loss: 0.9552 - val_masked_acc: 0.4896\n",
            "Epoch 12/20\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.9457 - masked_acc: 0.4873\n",
            "Epoch 12: masked_acc improved from 0.46839 to 0.48731, saving model to /content/drive/MyDrive/cross_attention/model.12-0.4873.h5\n",
            "296/296 [==============================] - 193s 651ms/step - loss: 0.9457 - masked_acc: 0.4873 - val_loss: 0.9081 - val_masked_acc: 0.5109\n",
            "Epoch 13/20\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.8992 - masked_acc: 0.5079\n",
            "Epoch 13: masked_acc improved from 0.48731 to 0.50789, saving model to /content/drive/MyDrive/cross_attention/model.13-0.5079.h5\n",
            "296/296 [==============================] - 195s 656ms/step - loss: 0.8992 - masked_acc: 0.5079 - val_loss: 0.8567 - val_masked_acc: 0.5352\n",
            "Epoch 14/20\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.8475 - masked_acc: 0.5315\n",
            "Epoch 14: masked_acc improved from 0.50789 to 0.53154, saving model to /content/drive/MyDrive/cross_attention/model.14-0.5315.h5\n",
            "296/296 [==============================] - 195s 659ms/step - loss: 0.8475 - masked_acc: 0.5315 - val_loss: 0.7992 - val_masked_acc: 0.5628\n",
            "Epoch 15/20\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.7918 - masked_acc: 0.5580\n",
            "Epoch 15: masked_acc improved from 0.53154 to 0.55796, saving model to /content/drive/MyDrive/cross_attention/model.15-0.5580.h5\n",
            "296/296 [==============================] - 194s 655ms/step - loss: 0.7918 - masked_acc: 0.5580 - val_loss: 0.7398 - val_masked_acc: 0.5911\n",
            "Epoch 16/20\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.7354 - masked_acc: 0.5856\n",
            "Epoch 16: masked_acc improved from 0.55796 to 0.58560, saving model to /content/drive/MyDrive/cross_attention/model.16-0.5856.h5\n",
            "296/296 [==============================] - 196s 661ms/step - loss: 0.7354 - masked_acc: 0.5856 - val_loss: 0.6804 - val_masked_acc: 0.6243\n",
            "Epoch 17/20\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.6817 - masked_acc: 0.6125\n",
            "Epoch 17: masked_acc improved from 0.58560 to 0.61246, saving model to /content/drive/MyDrive/cross_attention/model.17-0.6125.h5\n",
            "296/296 [==============================] - 194s 655ms/step - loss: 0.6817 - masked_acc: 0.6125 - val_loss: 0.6288 - val_masked_acc: 0.6501\n",
            "Epoch 18/20\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.6336 - masked_acc: 0.6362\n",
            "Epoch 18: masked_acc improved from 0.61246 to 0.63623, saving model to /content/drive/MyDrive/cross_attention/model.18-0.6362.h5\n",
            "296/296 [==============================] - 194s 657ms/step - loss: 0.6336 - masked_acc: 0.6362 - val_loss: 0.5833 - val_masked_acc: 0.6736\n",
            "Epoch 19/20\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.5894 - masked_acc: 0.6576\n",
            "Epoch 19: masked_acc improved from 0.63623 to 0.65760, saving model to /content/drive/MyDrive/cross_attention/model.19-0.6576.h5\n",
            "296/296 [==============================] - 195s 658ms/step - loss: 0.5894 - masked_acc: 0.6576 - val_loss: 0.5431 - val_masked_acc: 0.6940\n",
            "Epoch 20/20\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.5522 - masked_acc: 0.6756\n",
            "Epoch 20: masked_acc improved from 0.65760 to 0.67557, saving model to /content/drive/MyDrive/cross_attention/model.20-0.6756.h5\n",
            "296/296 [==============================] - 197s 664ms/step - loss: 0.5522 - masked_acc: 0.6756 - val_loss: 0.5135 - val_masked_acc: 0.7092\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f0c913a68b0>"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.fit(train_dataloader, steps_per_epoch=train_steps, epochs=30, validation_data=val_dataloader, validation_steps=valid_steps,\\\n",
        "          callbacks=[modelsave_wt,tensorboard_callback],initial_epoch=20)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rVw2Ul52YT_o",
        "outputId": "113f2278-d869-4a18-d1a0-be0f95b5ba64"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21/30\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.5190 - masked_acc: 0.6916\n",
            "Epoch 21: masked_acc improved from 0.67557 to 0.69156, saving model to /content/drive/MyDrive/cross_attention/model.21-0.6916.h5\n",
            "296/296 [==============================] - 198s 665ms/step - loss: 0.5190 - masked_acc: 0.6916 - val_loss: 0.4794 - val_masked_acc: 0.7257\n",
            "Epoch 22/30\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.4882 - masked_acc: 0.7065\n",
            "Epoch 22: masked_acc improved from 0.69156 to 0.70654, saving model to /content/drive/MyDrive/cross_attention/model.22-0.7065.h5\n",
            "296/296 [==============================] - 197s 666ms/step - loss: 0.4882 - masked_acc: 0.7065 - val_loss: 0.4541 - val_masked_acc: 0.7388\n",
            "Epoch 23/30\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.4623 - masked_acc: 0.7191\n",
            "Epoch 23: masked_acc improved from 0.70654 to 0.71911, saving model to /content/drive/MyDrive/cross_attention/model.23-0.7191.h5\n",
            "296/296 [==============================] - 194s 656ms/step - loss: 0.4623 - masked_acc: 0.7191 - val_loss: 0.4318 - val_masked_acc: 0.7501\n",
            "Epoch 24/30\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.4396 - masked_acc: 0.7302\n",
            "Epoch 24: masked_acc improved from 0.71911 to 0.73017, saving model to /content/drive/MyDrive/cross_attention/model.24-0.7302.h5\n",
            "296/296 [==============================] - 192s 648ms/step - loss: 0.4396 - masked_acc: 0.7302 - val_loss: 0.4130 - val_masked_acc: 0.7592\n",
            "Epoch 25/30\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.4194 - masked_acc: 0.7398\n",
            "Epoch 25: masked_acc improved from 0.73017 to 0.73983, saving model to /content/drive/MyDrive/cross_attention/model.25-0.7398.h5\n",
            "296/296 [==============================] - 192s 647ms/step - loss: 0.4194 - masked_acc: 0.7398 - val_loss: 0.3975 - val_masked_acc: 0.7674\n",
            "Epoch 26/30\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.4018 - masked_acc: 0.7485\n",
            "Epoch 26: masked_acc improved from 0.73983 to 0.74851, saving model to /content/drive/MyDrive/cross_attention/model.26-0.7485.h5\n",
            "296/296 [==============================] - 191s 645ms/step - loss: 0.4018 - masked_acc: 0.7485 - val_loss: 0.3822 - val_masked_acc: 0.7748\n",
            "Epoch 27/30\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.3851 - masked_acc: 0.7563\n",
            "Epoch 27: masked_acc improved from 0.74851 to 0.75629, saving model to /content/drive/MyDrive/cross_attention/model.27-0.7563.h5\n",
            "296/296 [==============================] - 191s 645ms/step - loss: 0.3851 - masked_acc: 0.7563 - val_loss: 0.3703 - val_masked_acc: 0.7808\n",
            "Epoch 28/30\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.3707 - masked_acc: 0.7634\n",
            "Epoch 28: masked_acc improved from 0.75629 to 0.76343, saving model to /content/drive/MyDrive/cross_attention/model.28-0.7634.h5\n",
            "296/296 [==============================] - 190s 642ms/step - loss: 0.3707 - masked_acc: 0.7634 - val_loss: 0.3592 - val_masked_acc: 0.7858\n",
            "Epoch 29/30\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.3582 - masked_acc: 0.7693\n",
            "Epoch 29: masked_acc improved from 0.76343 to 0.76926, saving model to /content/drive/MyDrive/cross_attention/model.29-0.7693.h5\n",
            "296/296 [==============================] - 192s 649ms/step - loss: 0.3582 - masked_acc: 0.7693 - val_loss: 0.3495 - val_masked_acc: 0.7911\n",
            "Epoch 30/30\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.3465 - masked_acc: 0.7753\n",
            "Epoch 30: masked_acc improved from 0.76926 to 0.77534, saving model to /content/drive/MyDrive/cross_attention/model.30-0.7753.h5\n",
            "296/296 [==============================] - 190s 640ms/step - loss: 0.3465 - masked_acc: 0.7753 - val_loss: 0.3406 - val_masked_acc: 0.7958\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f0f6291c790>"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.fit(train_dataloader, steps_per_epoch=train_steps, epochs=40, validation_data=val_dataloader, validation_steps=valid_steps,\\\n",
        "          callbacks=[modelsave_wt,tensorboard_callback],initial_epoch=30)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CiU_eqtZg6ux",
        "outputId": "eff6ff40-0f3c-49ac-dc6d-b76af8b784b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 31/40\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.3364 - masked_acc: 0.7803\n",
            "Epoch 31: masked_acc improved from 0.77534 to 0.78026, saving model to /content/drive/MyDrive/cross_attention/model.31-0.7803.h5\n",
            "296/296 [==============================] - 198s 662ms/step - loss: 0.3364 - masked_acc: 0.7803 - val_loss: 0.3332 - val_masked_acc: 0.7998\n",
            "Epoch 32/40\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.3272 - masked_acc: 0.7846\n",
            "Epoch 32: masked_acc improved from 0.78026 to 0.78462, saving model to /content/drive/MyDrive/cross_attention/model.32-0.7846.h5\n",
            "296/296 [==============================] - 196s 663ms/step - loss: 0.3272 - masked_acc: 0.7846 - val_loss: 0.3257 - val_masked_acc: 0.8036\n",
            "Epoch 33/40\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.3184 - masked_acc: 0.7889\n",
            "Epoch 33: masked_acc improved from 0.78462 to 0.78893, saving model to /content/drive/MyDrive/cross_attention/model.33-0.7889.h5\n",
            "296/296 [==============================] - 194s 656ms/step - loss: 0.3184 - masked_acc: 0.7889 - val_loss: 0.3189 - val_masked_acc: 0.8065\n",
            "Epoch 34/40\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.3124 - masked_acc: 0.7917\n",
            "Epoch 34: masked_acc improved from 0.78893 to 0.79167, saving model to /content/drive/MyDrive/cross_attention/model.34-0.7917.h5\n",
            "296/296 [==============================] - 193s 653ms/step - loss: 0.3124 - masked_acc: 0.7917 - val_loss: 0.3132 - val_masked_acc: 0.8094\n",
            "Epoch 35/40\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.3029 - masked_acc: 0.7966\n",
            "Epoch 35: masked_acc improved from 0.79167 to 0.79662, saving model to /content/drive/MyDrive/cross_attention/model.35-0.7966.h5\n",
            "296/296 [==============================] - 195s 658ms/step - loss: 0.3029 - masked_acc: 0.7966 - val_loss: 0.3076 - val_masked_acc: 0.8127\n",
            "Epoch 36/40\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2963 - masked_acc: 0.7999\n",
            "Epoch 36: masked_acc improved from 0.79662 to 0.79986, saving model to /content/drive/MyDrive/cross_attention/model.36-0.7999.h5\n",
            "296/296 [==============================] - 196s 662ms/step - loss: 0.2963 - masked_acc: 0.7999 - val_loss: 0.3029 - val_masked_acc: 0.8154\n",
            "Epoch 37/40\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2900 - masked_acc: 0.8032\n",
            "Epoch 37: masked_acc improved from 0.79986 to 0.80323, saving model to /content/drive/MyDrive/cross_attention/model.37-0.8032.h5\n",
            "296/296 [==============================] - 196s 663ms/step - loss: 0.2900 - masked_acc: 0.8032 - val_loss: 0.2976 - val_masked_acc: 0.8180\n",
            "Epoch 38/40\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2842 - masked_acc: 0.8061\n",
            "Epoch 38: masked_acc improved from 0.80323 to 0.80605, saving model to /content/drive/MyDrive/cross_attention/model.38-0.8061.h5\n",
            "296/296 [==============================] - 195s 658ms/step - loss: 0.2842 - masked_acc: 0.8061 - val_loss: 0.2938 - val_masked_acc: 0.8202\n",
            "Epoch 39/40\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2788 - masked_acc: 0.8084\n",
            "Epoch 39: masked_acc improved from 0.80605 to 0.80838, saving model to /content/drive/MyDrive/cross_attention/model.39-0.8084.h5\n",
            "296/296 [==============================] - 206s 696ms/step - loss: 0.2788 - masked_acc: 0.8084 - val_loss: 0.2898 - val_masked_acc: 0.8221\n",
            "Epoch 40/40\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2739 - masked_acc: 0.8111\n",
            "Epoch 40: masked_acc improved from 0.80838 to 0.81110, saving model to /content/drive/MyDrive/cross_attention/model.40-0.8111.h5\n",
            "296/296 [==============================] - 196s 661ms/step - loss: 0.2739 - masked_acc: 0.8111 - val_loss: 0.2861 - val_masked_acc: 0.8243\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f0c922c7ca0>"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.fit(train_dataloader, steps_per_epoch=train_steps, epochs=50, validation_data=val_dataloader, validation_steps=valid_steps,\\\n",
        "          callbacks=[modelsave_wt,tensorboard_callback],initial_epoch=40)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AcbY4FcLnUpK",
        "outputId": "8b955606-27d1-4484-813d-235cf8a2736e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 41/50\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2689 - masked_acc: 0.8138\n",
            "Epoch 41: masked_acc improved from 0.81110 to 0.81379, saving model to /content/drive/MyDrive/cross_attention/model.41-0.8138.h5\n",
            "296/296 [==============================] - 199s 667ms/step - loss: 0.2689 - masked_acc: 0.8138 - val_loss: 0.2834 - val_masked_acc: 0.8253\n",
            "Epoch 42/50\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2649 - masked_acc: 0.8155\n",
            "Epoch 42: masked_acc improved from 0.81379 to 0.81554, saving model to /content/drive/MyDrive/cross_attention/model.42-0.8155.h5\n",
            "296/296 [==============================] - 195s 658ms/step - loss: 0.2649 - masked_acc: 0.8155 - val_loss: 0.2801 - val_masked_acc: 0.8273\n",
            "Epoch 43/50\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2602 - masked_acc: 0.8180\n",
            "Epoch 43: masked_acc improved from 0.81554 to 0.81800, saving model to /content/drive/MyDrive/cross_attention/model.43-0.8180.h5\n",
            "296/296 [==============================] - 192s 650ms/step - loss: 0.2602 - masked_acc: 0.8180 - val_loss: 0.2771 - val_masked_acc: 0.8290\n",
            "Epoch 44/50\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2560 - masked_acc: 0.8202\n",
            "Epoch 44: masked_acc improved from 0.81800 to 0.82016, saving model to /content/drive/MyDrive/cross_attention/model.44-0.8202.h5\n",
            "296/296 [==============================] - 191s 647ms/step - loss: 0.2560 - masked_acc: 0.8202 - val_loss: 0.2738 - val_masked_acc: 0.8310\n",
            "Epoch 45/50\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2520 - masked_acc: 0.8224\n",
            "Epoch 45: masked_acc improved from 0.82016 to 0.82238, saving model to /content/drive/MyDrive/cross_attention/model.45-0.8224.h5\n",
            "296/296 [==============================] - 193s 652ms/step - loss: 0.2520 - masked_acc: 0.8224 - val_loss: 0.2719 - val_masked_acc: 0.8325\n",
            "Epoch 46/50\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2484 - masked_acc: 0.8242\n",
            "Epoch 46: masked_acc improved from 0.82238 to 0.82418, saving model to /content/drive/MyDrive/cross_attention/model.46-0.8242.h5\n",
            "296/296 [==============================] - 196s 661ms/step - loss: 0.2484 - masked_acc: 0.8242 - val_loss: 0.2689 - val_masked_acc: 0.8340\n",
            "Epoch 47/50\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2453 - masked_acc: 0.8256\n",
            "Epoch 47: masked_acc improved from 0.82418 to 0.82559, saving model to /content/drive/MyDrive/cross_attention/model.47-0.8256.h5\n",
            "296/296 [==============================] - 193s 649ms/step - loss: 0.2453 - masked_acc: 0.8256 - val_loss: 0.2674 - val_masked_acc: 0.8346\n",
            "Epoch 48/50\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2418 - masked_acc: 0.8276\n",
            "Epoch 48: masked_acc improved from 0.82559 to 0.82761, saving model to /content/drive/MyDrive/cross_attention/model.48-0.8276.h5\n",
            "296/296 [==============================] - 194s 654ms/step - loss: 0.2418 - masked_acc: 0.8276 - val_loss: 0.2646 - val_masked_acc: 0.8368\n",
            "Epoch 49/50\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2389 - masked_acc: 0.8293\n",
            "Epoch 49: masked_acc improved from 0.82761 to 0.82928, saving model to /content/drive/MyDrive/cross_attention/model.49-0.8293.h5\n",
            "296/296 [==============================] - 194s 655ms/step - loss: 0.2389 - masked_acc: 0.8293 - val_loss: 0.2627 - val_masked_acc: 0.8381\n",
            "Epoch 50/50\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2362 - masked_acc: 0.8307\n",
            "Epoch 50: masked_acc improved from 0.82928 to 0.83072, saving model to /content/drive/MyDrive/cross_attention/model.50-0.8307.h5\n",
            "296/296 [==============================] - 192s 650ms/step - loss: 0.2362 - masked_acc: 0.8307 - val_loss: 0.2601 - val_masked_acc: 0.8389\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f0caa8cf100>"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.fit(train_dataloader, steps_per_epoch=train_steps, epochs=60, validation_data=val_dataloader, validation_steps=valid_steps,\\\n",
        "          callbacks=[modelsave_wt,tensorboard_callback],initial_epoch=50)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uIS5VLddxqau",
        "outputId": "16ca3c4f-f1de-4a03-bee8-86a4f1c3a724"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 51/60\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2331 - masked_acc: 0.8322\n",
            "Epoch 51: masked_acc improved from 0.83072 to 0.83216, saving model to /content/drive/MyDrive/cross_attention/model.51-0.8322.h5\n",
            "296/296 [==============================] - 201s 670ms/step - loss: 0.2331 - masked_acc: 0.8322 - val_loss: 0.2590 - val_masked_acc: 0.8400\n",
            "Epoch 52/60\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2304 - masked_acc: 0.8339\n",
            "Epoch 52: masked_acc improved from 0.83216 to 0.83391, saving model to /content/drive/MyDrive/cross_attention/model.52-0.8339.h5\n",
            "296/296 [==============================] - 194s 657ms/step - loss: 0.2304 - masked_acc: 0.8339 - val_loss: 0.2566 - val_masked_acc: 0.8417\n",
            "Epoch 53/60\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2278 - masked_acc: 0.8351\n",
            "Epoch 53: masked_acc improved from 0.83391 to 0.83507, saving model to /content/drive/MyDrive/cross_attention/model.53-0.8351.h5\n",
            "296/296 [==============================] - 197s 666ms/step - loss: 0.2278 - masked_acc: 0.8351 - val_loss: 0.2554 - val_masked_acc: 0.8423\n",
            "Epoch 54/60\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2252 - masked_acc: 0.8365\n",
            "Epoch 54: masked_acc improved from 0.83507 to 0.83648, saving model to /content/drive/MyDrive/cross_attention/model.54-0.8365.h5\n",
            "296/296 [==============================] - 195s 658ms/step - loss: 0.2252 - masked_acc: 0.8365 - val_loss: 0.2537 - val_masked_acc: 0.8434\n",
            "Epoch 55/60\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2229 - masked_acc: 0.8377\n",
            "Epoch 55: masked_acc improved from 0.83648 to 0.83771, saving model to /content/drive/MyDrive/cross_attention/model.55-0.8377.h5\n",
            "296/296 [==============================] - 194s 654ms/step - loss: 0.2229 - masked_acc: 0.8377 - val_loss: 0.2529 - val_masked_acc: 0.8442\n",
            "Epoch 56/60\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2206 - masked_acc: 0.8390\n",
            "Epoch 56: masked_acc improved from 0.83771 to 0.83903, saving model to /content/drive/MyDrive/cross_attention/model.56-0.8390.h5\n",
            "296/296 [==============================] - 191s 646ms/step - loss: 0.2206 - masked_acc: 0.8390 - val_loss: 0.2505 - val_masked_acc: 0.8454\n",
            "Epoch 57/60\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2183 - masked_acc: 0.8400\n",
            "Epoch 57: masked_acc improved from 0.83903 to 0.83997, saving model to /content/drive/MyDrive/cross_attention/model.57-0.8400.h5\n",
            "296/296 [==============================] - 193s 651ms/step - loss: 0.2183 - masked_acc: 0.8400 - val_loss: 0.2500 - val_masked_acc: 0.8459\n",
            "Epoch 58/60\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2167 - masked_acc: 0.8413\n",
            "Epoch 58: masked_acc improved from 0.83997 to 0.84133, saving model to /content/drive/MyDrive/cross_attention/model.58-0.8413.h5\n",
            "296/296 [==============================] - 193s 650ms/step - loss: 0.2167 - masked_acc: 0.8413 - val_loss: 0.2482 - val_masked_acc: 0.8472\n",
            "Epoch 59/60\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2143 - masked_acc: 0.8424\n",
            "Epoch 59: masked_acc improved from 0.84133 to 0.84239, saving model to /content/drive/MyDrive/cross_attention/model.59-0.8424.h5\n",
            "296/296 [==============================] - 194s 654ms/step - loss: 0.2143 - masked_acc: 0.8424 - val_loss: 0.2470 - val_masked_acc: 0.8478\n",
            "Epoch 60/60\n",
            " 76/296 [======>.......................] - ETA: 2:16 - loss: 0.2113 - masked_acc: 0.8441"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.fit(train_dataloader, steps_per_epoch=train_steps, epochs=65, validation_data=val_dataloader, validation_steps=valid_steps,\\\n",
        "          callbacks=[modelsave_wt,tensorboard_callback],initial_epoch=59)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k1ZdRLy4fi4N",
        "outputId": "70d22014-98ac-46db-fe44-0738d8ac450e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 60/65\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2113 - masked_acc: 0.8438\n",
            "Epoch 60: masked_acc improved from 0.84217 to 0.84383, saving model to /content/drive/MyDrive/cross_attention/model.60-0.8438.h5\n",
            "296/296 [==============================] - 190s 636ms/step - loss: 0.2113 - masked_acc: 0.8438 - val_loss: 0.2349 - val_masked_acc: 0.8526\n",
            "Epoch 61/65\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2085 - masked_acc: 0.8453\n",
            "Epoch 61: masked_acc improved from 0.84383 to 0.84530, saving model to /content/drive/MyDrive/cross_attention/model.61-0.8453.h5\n",
            "296/296 [==============================] - 186s 628ms/step - loss: 0.2085 - masked_acc: 0.8453 - val_loss: 0.2329 - val_masked_acc: 0.8539\n",
            "Epoch 62/65\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2060 - masked_acc: 0.8466\n",
            "Epoch 62: masked_acc improved from 0.84530 to 0.84659, saving model to /content/drive/MyDrive/cross_attention/model.62-0.8466.h5\n",
            "296/296 [==============================] - 187s 633ms/step - loss: 0.2060 - masked_acc: 0.8466 - val_loss: 0.2320 - val_masked_acc: 0.8542\n",
            "Epoch 63/65\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2040 - masked_acc: 0.8477\n",
            "Epoch 63: masked_acc improved from 0.84659 to 0.84775, saving model to /content/drive/MyDrive/cross_attention/model.63-0.8477.h5\n",
            "296/296 [==============================] - 187s 633ms/step - loss: 0.2040 - masked_acc: 0.8477 - val_loss: 0.2302 - val_masked_acc: 0.8553\n",
            "Epoch 64/65\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.2016 - masked_acc: 0.8488\n",
            "Epoch 64: masked_acc improved from 0.84775 to 0.84884, saving model to /content/drive/MyDrive/cross_attention/model.64-0.8488.h5\n",
            "296/296 [==============================] - 188s 636ms/step - loss: 0.2016 - masked_acc: 0.8488 - val_loss: 0.2302 - val_masked_acc: 0.8557\n",
            "Epoch 65/65\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.1996 - masked_acc: 0.8502\n",
            "Epoch 65: masked_acc improved from 0.84884 to 0.85021, saving model to /content/drive/MyDrive/cross_attention/model.65-0.8502.h5\n",
            "296/296 [==============================] - 187s 632ms/step - loss: 0.1996 - masked_acc: 0.8502 - val_loss: 0.2278 - val_masked_acc: 0.8569\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f7e4eccbb20>"
            ]
          },
          "metadata": {},
          "execution_count": 101
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.fit(train_dataloader, steps_per_epoch=train_steps, epochs=70, validation_data=val_dataloader, validation_steps=valid_steps,\\\n",
        "          callbacks=[modelsave_wt,tensorboard_callback],initial_epoch=65)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vn5ON3p3nLtd",
        "outputId": "47592a7b-de78-45de-b169-2f74a16cacb8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 66/70\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.1974 - masked_acc: 0.8510\n",
            "Epoch 66: masked_acc improved from 0.85021 to 0.85099, saving model to /content/drive/MyDrive/cross_attention/model.66-0.8510.h5\n",
            "296/296 [==============================] - 192s 640ms/step - loss: 0.1974 - masked_acc: 0.8510 - val_loss: 0.2268 - val_masked_acc: 0.8570\n",
            "Epoch 67/70\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.1952 - masked_acc: 0.8524\n",
            "Epoch 67: masked_acc improved from 0.85099 to 0.85240, saving model to /content/drive/MyDrive/cross_attention/model.67-0.8524.h5\n",
            "296/296 [==============================] - 188s 635ms/step - loss: 0.1952 - masked_acc: 0.8524 - val_loss: 0.2265 - val_masked_acc: 0.8577\n",
            "Epoch 68/70\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.1935 - masked_acc: 0.8533\n",
            "Epoch 68: masked_acc improved from 0.85240 to 0.85333, saving model to /content/drive/MyDrive/cross_attention/model.68-0.8533.h5\n",
            "296/296 [==============================] - 189s 640ms/step - loss: 0.1935 - masked_acc: 0.8533 - val_loss: 0.2249 - val_masked_acc: 0.8589\n",
            "Epoch 69/70\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.1915 - masked_acc: 0.8547\n",
            "Epoch 69: masked_acc improved from 0.85333 to 0.85469, saving model to /content/drive/MyDrive/cross_attention/model.69-0.8547.h5\n",
            "296/296 [==============================] - 190s 641ms/step - loss: 0.1915 - masked_acc: 0.8547 - val_loss: 0.2240 - val_masked_acc: 0.8597\n",
            "Epoch 70/70\n",
            "296/296 [==============================] - ETA: 0s - loss: 0.1899 - masked_acc: 0.8552\n",
            "Epoch 70: masked_acc improved from 0.85469 to 0.85520, saving model to /content/drive/MyDrive/cross_attention/model.70-0.8552.h5\n",
            "296/296 [==============================] - 191s 644ms/step - loss: 0.1899 - masked_acc: 0.8552 - val_loss: 0.2234 - val_masked_acc: 0.8599\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f7e4ece4100>"
            ]
          },
          "metadata": {},
          "execution_count": 105
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Prediction:"
      ],
      "metadata": {
        "id": "K1vJhIpUug9Y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def predict(input_sentence):\n",
        "  '''takes italian input sentence and outputs translated english sentence'''\n",
        "\n",
        "  encoder_seq = tknizer_ita.texts_to_sequences([input_sentence]) # tokenizing\n",
        "  encoder_seq = pad_sequences(encoder_seq, maxlen=20, dtype='int32', padding='post') #padding to len 20\n",
        "  encoder_output, encoder_h, encoder_c = model.encoder(encoder_seq) # encoding\n",
        "\n",
        "  decoder_input = tknizer_eng.texts_to_sequences(['<start>']) # feeding <start> token as 1st decoder input\n",
        "  decoder_state_h = encoder_h\n",
        "  decoder_state_c = encoder_c\n",
        "  prediction=[]\n",
        "  att_wt = []\n",
        "  for i in range(20): # iterating over full max_length\n",
        "    if i>0 and prediction[-1]==['<end>']: # end translation when <end> token predicted\n",
        "      break\n",
        "    else:\n",
        "      input=tf.expand_dims(decoder_input[0][-1],0)  \n",
        "      if i==0:\n",
        "       input=tf.expand_dims(input,0)\n",
        "      decoder_output, state_h,state_c,attention_weights, context_vector= model.layers[1].cross_attention(input, encoder_output,decoder_state_h,decoder_state_c)\n",
        "      att_wt.append(attention_weights)\n",
        "      index = tf.argmax(decoder_output,axis=-1).numpy()\n",
        "      prediction.append(tknizer_eng.sequences_to_texts([index]))\n",
        "      decoder_input[0].append(index)\n",
        "      decoder_state_h = state_h\n",
        "      decoder_state_c = state_c\n",
        "    \n",
        "  predicted_sent= prediction[0][0]\n",
        "  for word in prediction:\n",
        "    predicted_sent = predicted_sent + ' ' + word[0]\n",
        "  return predicted_sent,att_wt"
      ],
      "metadata": {
        "id": "TWgja-m6i3IU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# predicting on random unseen 5 italian sentences\n",
        "\n",
        "for doc in test.sample(5).values:\n",
        "  sent=doc[0]\n",
        "  eng_sent = doc[-1]\n",
        "  print('italian: ',sent)\n",
        "  print('English True: ',eng_sent)\n",
        "  trans,_ = predict(sent)\n",
        "  print('Model Translation: ',trans,'\\n')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1c5a7e1c-b708-4990-d82f-66bd41cef5c9",
        "id": "zVvkF0RBi3IV"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "italian:  vedo cosavete fatto lì\n",
            "English True:  i see what you did there <end>\n",
            "Model Translation:  i i see what you have done there <end> \n",
            "\n",
            "italian:  tom non è un fisico\n",
            "English True:  tom is not a physician <end>\n",
            "Model Translation:  tom tom is not a physician <end> \n",
            "\n",
            "italian:  cè un costo di consegna\n",
            "English True:  is there a delivery charge <end>\n",
            "Model Translation:  there there is a charge of the delivery <end> \n",
            "\n",
            "italian:  è un tizio strano\n",
            "English True:  he is a strange guy <end>\n",
            "Model Translation:  it it is a strange guy <end> \n",
            "\n",
            "italian:  tutti qua sanno che non mangiamo la carne di maiale\n",
            "English True:  everyone here knows that we do not eat pork <end>\n",
            "Model Translation:  everyone everyone here knows we do not eat pork <end> \n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Bleu_score on Test Dataset:"
      ],
      "metadata": {
        "id": "gRVWaNIaxW1T"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk.translate.bleu_score as bleu"
      ],
      "metadata": {
        "id": "xpJs8vK9i3IX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def bleu_score(input,weights):\n",
        "  bleu_score=[]\n",
        "  for doc in input:\n",
        "    input_sentence = doc[0]\n",
        "    eng_sent = doc[-1]\n",
        "    predicted_sent,_ = predict(input_sentence)\n",
        "    bleu_score.append(bleu.sentence_bleu(eng_sent, predicted_sent,weights=weights))\n",
        "  return np.mean(bleu_score)"
      ],
      "metadata": {
        "id": "byxKkWbFi3KW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# bleu score by only matching uni-grams\n",
        "\n",
        "bleu_score_test = bleu_score( input = test.values, weights = (1,0,0,0) )\n",
        "print('average test data bleu score: ',bleu_score_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "524451ae-fb0b-4bf9-c7cb-a5e3afa88607",
        "id": "TAYTqLwYi3KY"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "average test data bleu score:  0.4451662890214658\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# bleu score by matching 1 to 4-grams\n",
        "\n",
        "bleu_score_test = bleu_score( input = test.values, weights = (0.25,0.25,0.25,0.25) )\n",
        "print('average test data cumulative 4-gram bleu score: ',bleu_score_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cCiIpbHRtbdn",
        "outputId": "f2fd6ff8-129c-4b0b-c18e-e4d2070513a6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "average test data cumulative 4-gram bleu score:  1.479362713798278e-231\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}