Capsule Network basic - SampleDataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "train_filename = \"data_preprocessed/train_data_sample_gray.p\"\n",
    "(train_labels, train_data, train_tensors) = pickle.load(open(train_filename, mode='rb'))\n",
    "\n",
    "valid_filename = \"data_preprocessed/valid_data_sample_gray.p\"\n",
    "(valid_labels, valid_data, valid_tensors) = pickle.load(open(valid_filename, mode='rb'))\n",
    "\n",
    "test_filename = \"data_preprocessed/test_data_sample_gray.p\"\n",
    "(test_labels, test_data, test_tensors) = pickle.load(open(test_filename, mode='rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def onhotLabels(label):\n",
    "    from sklearn.preprocessing import OneHotEncoder\n",
    "    enc = OneHotEncoder()\n",
    "    enc.fit(label)\n",
    "    return enc.transform(label).toarray()\n",
    "\n",
    "train_labels = onhotLabels(train_labels)\n",
    "valid_labels = onhotLabels(valid_labels)\n",
    "test_labels = onhotLabels(test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CapsNet model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_4 (InputLayer)            (None, 64, 64, 1)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv1 (Conv2D)                  (None, 56, 56, 256)  20992       input_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "primarycap_conv2d (Conv2D)      (None, 24, 24, 256)  5308672     conv1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "primarycap_reshape (Reshape)    (None, 18432, 8)     0           primarycap_conv2d[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "primarycap_squash (Lambda)      (None, 18432, 8)     0           primarycap_reshape[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "digitcaps (CapsuleLayer)        (None, 2, 16)        4718592     primarycap_squash[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "input_5 (InputLayer)            (None, 2)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "mask_4 (Mask)                   (None, 32)           0           digitcaps[0][0]                  \n",
      "                                                                 input_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "capsnet (Length)                (None, 2)            0           digitcaps[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "decoder (Sequential)            (None, 64, 64, 1)    4740608     mask_4[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 14,788,864\n",
      "Trainable params: 14,788,864\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import argparse\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras import callbacks\n",
    "\n",
    "import numpy as np\n",
    "from keras import layers, models, optimizers\n",
    "from keras import backend as K\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "from capsulelayers import CapsuleLayer, PrimaryCap, Length, Mask\n",
    "\n",
    "def CapsNet(input_shape, n_class, routings):\n",
    "    \"\"\"\n",
    "    A Capsule Network on MNIST.\n",
    "    :param input_shape: data shape, 3d, [width, height, channels]\n",
    "    :param n_class: number of classes\n",
    "    :param routings: number of routing iterations\n",
    "    :return: Two Keras Models, the first one used for training, and the second one for evaluation.\n",
    "            `eval_model` can also be used for training.\n",
    "    \"\"\"\n",
    "    x = layers.Input(shape=input_shape)\n",
    "\n",
    "    # Layer 1: Just a conventional Conv2D layer\n",
    "    conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x)\n",
    "\n",
    "    # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]\n",
    "    primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid')\n",
    "\n",
    "    # Layer 3: Capsule layer. Routing algorithm works here.\n",
    "    digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,\n",
    "                             name='digitcaps')(primarycaps)\n",
    "\n",
    "    # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.\n",
    "    # If using tensorflow, this will not be necessary. :)\n",
    "    out_caps = Length(name='capsnet')(digitcaps)\n",
    "\n",
    "    # Decoder network.\n",
    "    y = layers.Input(shape=(n_class,))\n",
    "    masked_by_y = Mask()([digitcaps, y])  # The true label is used to mask the output of capsule layer. For training\n",
    "    masked = Mask()(digitcaps)  # Mask using the capsule with maximal length. For prediction\n",
    "\n",
    "    # Shared Decoder model in training and prediction\n",
    "    decoder = models.Sequential(name='decoder')\n",
    "    decoder.add(layers.Dense(512, activation='relu', input_dim=16*n_class))\n",
    "    decoder.add(layers.Dense(1024, activation='relu'))\n",
    "    decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))\n",
    "    decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))\n",
    "\n",
    "    # Models for training and evaluation (prediction)\n",
    "    train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])\n",
    "    eval_model = models.Model(x, [out_caps, decoder(masked)])\n",
    "\n",
    "    # manipulate model\n",
    "    noise = layers.Input(shape=(n_class, 16))\n",
    "    noised_digitcaps = layers.Add()([digitcaps, noise])\n",
    "    masked_noised_y = Mask()([noised_digitcaps, y])\n",
    "    manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))\n",
    "    return train_model, eval_model, manipulate_model\n",
    "\n",
    "model, eval_model, manipulate_model = CapsNet(input_shape=train_tensors.shape[1:],\n",
    "                                                  n_class=len(np.unique(train_labels)),\n",
    "                                                  routings=4)\n",
    "decoder.summary()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "\n",
    "def binary_accuracy(y_true, y_pred):\n",
    "    return K.mean(K.equal(y_true, K.round(y_pred)))\n",
    "\n",
    "def precision_threshold(threshold = 0.5):\n",
    "    def precision(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())\n",
    "        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))\n",
    "        predicted_positives = K.sum(y_pred)\n",
    "        precision_ratio = true_positives / (predicted_positives + K.epsilon())\n",
    "        return precision_ratio\n",
    "    return precision\n",
    "\n",
    "def recall_threshold(threshold = 0.5):\n",
    "    def recall(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())\n",
    "        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))\n",
    "        possible_positives = K.sum(K.clip(y_true, 0, 1))\n",
    "        recall_ratio = true_positives / (possible_positives + K.epsilon())\n",
    "        return recall_ratio\n",
    "    return recall\n",
    "\n",
    "def fbeta_score_threshold(beta = 1, threshold = 0.5):\n",
    "    def fbeta_score(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        beta_value = beta\n",
    "        p = precision_threshold(threshold_value)(y_true, y_pred)\n",
    "        r = recall_threshold(threshold_value)(y_true, y_pred)\n",
    "        bb = beta_value ** 2\n",
    "        fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())\n",
    "        return fbeta_score\n",
    "    return fbeta_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def margin_loss(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    Margin loss for Eq.(4). When y_true[i, :] contains not just one `1`, this loss should work too. Not test it.\n",
    "    :param y_true: [None, n_classes]\n",
    "    :param y_pred: [None, num_capsule]\n",
    "    :return: a scalar loss value.\n",
    "    \"\"\"\n",
    "    L = y_true * K.square(K.maximum(0., 0.9 - y_pred)) + \\\n",
    "        0.5 * (1 - y_true) * K.square(K.maximum(0., y_pred - 0.1))\n",
    "\n",
    "    return K.mean(K.sum(L, 1))\n",
    "\n",
    "def train(model, data, lr, lr_decay, lam_recon, batch_size, shift_fraction, epochs):\n",
    "    \"\"\"\n",
    "    Training a CapsuleNet\n",
    "    :param model: the CapsuleNet model\n",
    "    :param data: a tuple containing training and testing data, like `((x_train, y_train), (x_test, y_test))`\n",
    "    :param args: arguments\n",
    "    :return: The trained model\n",
    "    \"\"\"\n",
    "    # unpacking the data\n",
    "    (x_train, y_train), (x_test, y_test) = data\n",
    "\n",
    "    # callbacks\n",
    "    log = callbacks.CSVLogger('saved_models/CapsNet_log.csv')\n",
    "    tb = callbacks.TensorBoard(log_dir='saved_models/tensorboard-logs',\n",
    "                               batch_size=batch_size, histogram_freq=0)\n",
    "    checkpoint = callbacks.ModelCheckpoint(filepath='saved_models/CapsNet.best.from_scratch.hdf5', \n",
    "                               verbose=1, save_best_only=True)\n",
    "    cb_lr_decay = callbacks.LearningRateScheduler(schedule=lambda epoch: lr * (lr_decay ** epoch))\n",
    "\n",
    "    # compile the model\n",
    "    model.compile(optimizer='sgd', loss='binary_crossentropy', \n",
    "              metrics=[precision_threshold(threshold = 0.5), \n",
    "              recall_threshold(threshold = 0.5), \n",
    "              fbeta_score_threshold(beta=0.5, threshold = 0.5),\n",
    "              'accuracy'])\n",
    "\n",
    "    # Training without data augmentation:\n",
    "#     model.fit([x_train, y_train], [y_train, x_train], batch_size=batch_size, epochs=epochs,\n",
    "#               validation_data=[[x_test, y_test], [y_test, x_test]], callbacks=[log, tb, checkpoint, cb_lr_decay])\n",
    "\n",
    "    # Begin: Training with data augmentation ---------------------------------------------------------------------#\n",
    "    def train_generator(x, y, batch_size, shift_fraction=0.):\n",
    "        train_datagen = ImageDataGenerator(width_shift_range=shift_fraction,\n",
    "                                           height_shift_range=shift_fraction)  # shift up to 2 pixel for MNIST\n",
    "        generator = train_datagen.flow(x, y, batch_size=batch_size)\n",
    "        while 1:\n",
    "            x_batch, y_batch = generator.next()\n",
    "            yield ([x_batch, y_batch], [y_batch, x_batch])\n",
    "\n",
    "    # Training with data augmentation. If shift_fraction=0., also no augmentation.\n",
    "    model.fit_generator(generator=train_generator(x_train, y_train, batch_size, shift_fraction),\n",
    "                        steps_per_epoch=int(y_train.shape[0] / batch_size),\n",
    "                        epochs=epochs,\n",
    "                        validation_data=[[x_test, y_test], [y_test, x_test]],\n",
    "                        callbacks=[log, tb, checkpoint, cb_lr_decay])\n",
    "    # End: Training with data augmentation -----------------------------------------------------------------------#\n",
    "\n",
    "    from utils import plot_log\n",
    "    plot_log('saved_models/CapsNet_log.csv', show=True)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3930 - capsnet_loss: 0.6998 - decoder_loss: 0.6932 - capsnet_precision: 0.5239 - capsnet_recall: 0.5429 - capsnet_fbeta_score: 0.5255 - capsnet_acc: 0.5246 - decoder_precision: 0.4958 - decoder_recall: 0.5000 - decoder_fbeta_score: 0.4965 - decoder_acc: 0.0140Epoch 00001: val_loss improved from inf to 1.38791, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 75s 706ms/step - loss: 1.3927 - capsnet_loss: 0.6995 - decoder_loss: 0.6932 - capsnet_precision: 0.5240 - capsnet_recall: 0.5425 - capsnet_fbeta_score: 0.5255 - capsnet_acc: 0.5246 - decoder_precision: 0.4957 - decoder_recall: 0.5000 - decoder_fbeta_score: 0.4964 - decoder_acc: 0.0141 - val_loss: 1.3879 - val_capsnet_loss: 0.6947 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5193 - val_capsnet_recall: 0.5609 - val_capsnet_fbeta_score: 0.5261 - val_capsnet_acc: 0.5205 - val_decoder_precision: 0.4936 - val_decoder_recall: 0.4995 - val_decoder_fbeta_score: 0.4947 - val_decoder_acc: 0.0134\n",
      "Epoch 2/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3884 - capsnet_loss: 0.6952 - decoder_loss: 0.6932 - capsnet_precision: 0.5248 - capsnet_recall: 0.5390 - capsnet_fbeta_score: 0.5255 - capsnet_acc: 0.5240 - decoder_precision: 0.4951 - decoder_recall: 0.5003 - decoder_fbeta_score: 0.4960 - decoder_acc: 0.0144Epoch 00002: val_loss improved from 1.38791 to 1.38579, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 75s 704ms/step - loss: 1.3887 - capsnet_loss: 0.6955 - decoder_loss: 0.6932 - capsnet_precision: 0.5251 - capsnet_recall: 0.5389 - capsnet_fbeta_score: 0.5257 - capsnet_acc: 0.5242 - decoder_precision: 0.4952 - decoder_recall: 0.5003 - decoder_fbeta_score: 0.4961 - decoder_acc: 0.0144 - val_loss: 1.3858 - val_capsnet_loss: 0.6926 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5335 - val_capsnet_recall: 0.5736 - val_capsnet_fbeta_score: 0.5403 - val_capsnet_acc: 0.5350 - val_decoder_precision: 0.4935 - val_decoder_recall: 0.4998 - val_decoder_fbeta_score: 0.4947 - val_decoder_acc: 0.0134\n",
      "Epoch 3/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3855 - capsnet_loss: 0.6923 - decoder_loss: 0.6932 - capsnet_precision: 0.5366 - capsnet_recall: 0.5429 - capsnet_fbeta_score: 0.5357 - capsnet_acc: 0.5366 - decoder_precision: 0.4947 - decoder_recall: 0.5006 - decoder_fbeta_score: 0.4958 - decoder_acc: 0.0144Epoch 00003: val_loss improved from 1.38579 to 1.38230, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 75s 705ms/step - loss: 1.3858 - capsnet_loss: 0.6926 - decoder_loss: 0.6932 - capsnet_precision: 0.5361 - capsnet_recall: 0.5422 - capsnet_fbeta_score: 0.5352 - capsnet_acc: 0.5361 - decoder_precision: 0.4948 - decoder_recall: 0.5006 - decoder_fbeta_score: 0.4959 - decoder_acc: 0.0143 - val_loss: 1.3823 - val_capsnet_loss: 0.6891 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5379 - val_capsnet_recall: 0.5927 - val_capsnet_fbeta_score: 0.5471 - val_capsnet_acc: 0.5414 - val_decoder_precision: 0.4935 - val_decoder_recall: 0.5000 - val_decoder_fbeta_score: 0.4947 - val_decoder_acc: 0.0134\n",
      "Epoch 4/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3836 - capsnet_loss: 0.6904 - decoder_loss: 0.6932 - capsnet_precision: 0.5364 - capsnet_recall: 0.5399 - capsnet_fbeta_score: 0.5356 - capsnet_acc: 0.5354 - decoder_precision: 0.4955 - decoder_recall: 0.5006 - decoder_fbeta_score: 0.4965 - decoder_acc: 0.0139Epoch 00004: val_loss improved from 1.38230 to 1.37998, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 75s 706ms/step - loss: 1.3836 - capsnet_loss: 0.6904 - decoder_loss: 0.6932 - capsnet_precision: 0.5365 - capsnet_recall: 0.5416 - capsnet_fbeta_score: 0.5360 - capsnet_acc: 0.5357 - decoder_precision: 0.4956 - decoder_recall: 0.5006 - decoder_fbeta_score: 0.4965 - decoder_acc: 0.0140 - val_loss: 1.3800 - val_capsnet_loss: 0.6868 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5505 - val_capsnet_recall: 0.5827 - val_capsnet_fbeta_score: 0.5558 - val_capsnet_acc: 0.5514 - val_decoder_precision: 0.4935 - val_decoder_recall: 0.4998 - val_decoder_fbeta_score: 0.4947 - val_decoder_acc: 0.0133\n",
      "Epoch 5/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3824 - capsnet_loss: 0.6892 - decoder_loss: 0.6932 - capsnet_precision: 0.5426 - capsnet_recall: 0.5467 - capsnet_fbeta_score: 0.5415 - capsnet_acc: 0.5408 - decoder_precision: 0.4951 - decoder_recall: 0.5004 - decoder_fbeta_score: 0.4961 - decoder_acc: 0.0138Epoch 00005: val_loss did not improve\n",
      "106/106 [==============================] - 74s 702ms/step - loss: 1.3823 - capsnet_loss: 0.6891 - decoder_loss: 0.6932 - capsnet_precision: 0.5431 - capsnet_recall: 0.5463 - capsnet_fbeta_score: 0.5418 - capsnet_acc: 0.5411 - decoder_precision: 0.4951 - decoder_recall: 0.5004 - decoder_fbeta_score: 0.4961 - decoder_acc: 0.0139 - val_loss: 1.3801 - val_capsnet_loss: 0.6869 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5561 - val_capsnet_recall: 0.5482 - val_capsnet_fbeta_score: 0.5539 - val_capsnet_acc: 0.5545 - val_decoder_precision: 0.4935 - val_decoder_recall: 0.4997 - val_decoder_fbeta_score: 0.4946 - val_decoder_acc: 0.0133\n",
      "Epoch 6/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3810 - capsnet_loss: 0.6878 - decoder_loss: 0.6932 - capsnet_precision: 0.5544 - capsnet_recall: 0.5488 - capsnet_fbeta_score: 0.5516 - capsnet_acc: 0.5513 - decoder_precision: 0.4948 - decoder_recall: 0.5006 - decoder_fbeta_score: 0.4958 - decoder_acc: 0.0144Epoch 00006: val_loss improved from 1.37998 to 1.37739, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 75s 706ms/step - loss: 1.3809 - capsnet_loss: 0.6877 - decoder_loss: 0.6932 - capsnet_precision: 0.5544 - capsnet_recall: 0.5495 - capsnet_fbeta_score: 0.5517 - capsnet_acc: 0.5515 - decoder_precision: 0.4949 - decoder_recall: 0.5006 - decoder_fbeta_score: 0.4960 - decoder_acc: 0.0144 - val_loss: 1.3774 - val_capsnet_loss: 0.6842 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5613 - val_capsnet_recall: 0.5745 - val_capsnet_fbeta_score: 0.5635 - val_capsnet_acc: 0.5627 - val_decoder_precision: 0.4934 - val_decoder_recall: 0.4998 - val_decoder_fbeta_score: 0.4946 - val_decoder_acc: 0.0133\n",
      "Epoch 7/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3769 - capsnet_loss: 0.6837 - decoder_loss: 0.6932 - capsnet_precision: 0.5619 - capsnet_recall: 0.5449 - capsnet_fbeta_score: 0.5571 - capsnet_acc: 0.5603 - decoder_precision: 0.4959 - decoder_recall: 0.5007 - decoder_fbeta_score: 0.4968 - decoder_acc: 0.0142Epoch 00007: val_loss improved from 1.37739 to 1.37663, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 75s 707ms/step - loss: 1.3769 - capsnet_loss: 0.6837 - decoder_loss: 0.6932 - capsnet_precision: 0.5619 - capsnet_recall: 0.5460 - capsnet_fbeta_score: 0.5574 - capsnet_acc: 0.5604 - decoder_precision: 0.4960 - decoder_recall: 0.5007 - decoder_fbeta_score: 0.4969 - decoder_acc: 0.0143 - val_loss: 1.3766 - val_capsnet_loss: 0.6834 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5654 - val_capsnet_recall: 0.5855 - val_capsnet_fbeta_score: 0.5689 - val_capsnet_acc: 0.5673 - val_decoder_precision: 0.4934 - val_decoder_recall: 0.4999 - val_decoder_fbeta_score: 0.4947 - val_decoder_acc: 0.0133\n",
      "Epoch 8/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3781 - capsnet_loss: 0.6849 - decoder_loss: 0.6932 - capsnet_precision: 0.5571 - capsnet_recall: 0.5503 - capsnet_fbeta_score: 0.5542 - capsnet_acc: 0.5563 - decoder_precision: 0.4962 - decoder_recall: 0.5008 - decoder_fbeta_score: 0.4970 - decoder_acc: 0.0142Epoch 00008: val_loss improved from 1.37663 to 1.37453, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 75s 706ms/step - loss: 1.3780 - capsnet_loss: 0.6848 - decoder_loss: 0.6932 - capsnet_precision: 0.5576 - capsnet_recall: 0.5507 - capsnet_fbeta_score: 0.5547 - capsnet_acc: 0.5567 - decoder_precision: 0.4961 - decoder_recall: 0.5008 - decoder_fbeta_score: 0.4970 - decoder_acc: 0.0142 - val_loss: 1.3745 - val_capsnet_loss: 0.6813 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5692 - val_capsnet_recall: 0.5855 - val_capsnet_fbeta_score: 0.5720 - val_capsnet_acc: 0.5709 - val_decoder_precision: 0.4934 - val_decoder_recall: 0.5000 - val_decoder_fbeta_score: 0.4947 - val_decoder_acc: 0.0133\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3716 - capsnet_loss: 0.6784 - decoder_loss: 0.6932 - capsnet_precision: 0.5733 - capsnet_recall: 0.5542 - capsnet_fbeta_score: 0.5678 - capsnet_acc: 0.5714 - decoder_precision: 0.4957 - decoder_recall: 0.5008 - decoder_fbeta_score: 0.4966 - decoder_acc: 0.0139Epoch 00009: val_loss did not improve\n",
      "106/106 [==============================] - 75s 704ms/step - loss: 1.3719 - capsnet_loss: 0.6788 - decoder_loss: 0.6932 - capsnet_precision: 0.5729 - capsnet_recall: 0.5545 - capsnet_fbeta_score: 0.5675 - capsnet_acc: 0.5711 - decoder_precision: 0.4961 - decoder_recall: 0.5008 - decoder_fbeta_score: 0.4970 - decoder_acc: 0.0139 - val_loss: 1.3756 - val_capsnet_loss: 0.6824 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5702 - val_capsnet_recall: 0.5464 - val_capsnet_fbeta_score: 0.5647 - val_capsnet_acc: 0.5668 - val_decoder_precision: 0.4934 - val_decoder_recall: 0.5001 - val_decoder_fbeta_score: 0.4947 - val_decoder_acc: 0.0133\n",
      "Epoch 10/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3761 - capsnet_loss: 0.6829 - decoder_loss: 0.6932 - capsnet_precision: 0.5679 - capsnet_recall: 0.5565 - capsnet_fbeta_score: 0.5642 - capsnet_acc: 0.5658 - decoder_precision: 0.4948 - decoder_recall: 0.5010 - decoder_fbeta_score: 0.4959 - decoder_acc: 0.0147Epoch 00010: val_loss improved from 1.37453 to 1.37270, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 75s 708ms/step - loss: 1.3755 - capsnet_loss: 0.6823 - decoder_loss: 0.6932 - capsnet_precision: 0.5698 - capsnet_recall: 0.5575 - capsnet_fbeta_score: 0.5659 - capsnet_acc: 0.5674 - decoder_precision: 0.4947 - decoder_recall: 0.5010 - decoder_fbeta_score: 0.4959 - decoder_acc: 0.0147 - val_loss: 1.3727 - val_capsnet_loss: 0.6795 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5742 - val_capsnet_recall: 0.5727 - val_capsnet_fbeta_score: 0.5734 - val_capsnet_acc: 0.5736 - val_decoder_precision: 0.4935 - val_decoder_recall: 0.5002 - val_decoder_fbeta_score: 0.4947 - val_decoder_acc: 0.0133\n",
      "Epoch 11/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3760 - capsnet_loss: 0.6828 - decoder_loss: 0.6932 - capsnet_precision: 0.5687 - capsnet_recall: 0.5560 - capsnet_fbeta_score: 0.5645 - capsnet_acc: 0.5673 - decoder_precision: 0.4962 - decoder_recall: 0.5010 - decoder_fbeta_score: 0.4971 - decoder_acc: 0.0143Epoch 00011: val_loss improved from 1.37270 to 1.37127, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 75s 709ms/step - loss: 1.3757 - capsnet_loss: 0.6825 - decoder_loss: 0.6932 - capsnet_precision: 0.5689 - capsnet_recall: 0.5566 - capsnet_fbeta_score: 0.5648 - capsnet_acc: 0.5675 - decoder_precision: 0.4961 - decoder_recall: 0.5010 - decoder_fbeta_score: 0.4970 - decoder_acc: 0.0143 - val_loss: 1.3713 - val_capsnet_loss: 0.6781 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5712 - val_capsnet_recall: 0.5836 - val_capsnet_fbeta_score: 0.5730 - val_capsnet_acc: 0.5723 - val_decoder_precision: 0.4935 - val_decoder_recall: 0.5001 - val_decoder_fbeta_score: 0.4947 - val_decoder_acc: 0.0133\n",
      "Epoch 12/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3719 - capsnet_loss: 0.6787 - decoder_loss: 0.6932 - capsnet_precision: 0.5723 - capsnet_recall: 0.5607 - capsnet_fbeta_score: 0.5687 - capsnet_acc: 0.5699 - decoder_precision: 0.4951 - decoder_recall: 0.5008 - decoder_fbeta_score: 0.4962 - decoder_acc: 0.0138Epoch 00012: val_loss improved from 1.37127 to 1.37102, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 75s 710ms/step - loss: 1.3719 - capsnet_loss: 0.6787 - decoder_loss: 0.6932 - capsnet_precision: 0.5727 - capsnet_recall: 0.5610 - capsnet_fbeta_score: 0.5691 - capsnet_acc: 0.5703 - decoder_precision: 0.4949 - decoder_recall: 0.5008 - decoder_fbeta_score: 0.4960 - decoder_acc: 0.0139 - val_loss: 1.3710 - val_capsnet_loss: 0.6778 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5796 - val_capsnet_recall: 0.5873 - val_capsnet_fbeta_score: 0.5806 - val_capsnet_acc: 0.5805 - val_decoder_precision: 0.4934 - val_decoder_recall: 0.4998 - val_decoder_fbeta_score: 0.4947 - val_decoder_acc: 0.0133\n",
      "Epoch 13/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3723 - capsnet_loss: 0.6791 - decoder_loss: 0.6932 - capsnet_precision: 0.5757 - capsnet_recall: 0.5595 - capsnet_fbeta_score: 0.5708 - capsnet_acc: 0.5734 - decoder_precision: 0.4952 - decoder_recall: 0.5007 - decoder_fbeta_score: 0.4962 - decoder_acc: 0.0146Epoch 00013: val_loss improved from 1.37102 to 1.37036, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 75s 712ms/step - loss: 1.3721 - capsnet_loss: 0.6790 - decoder_loss: 0.6932 - capsnet_precision: 0.5762 - capsnet_recall: 0.5599 - capsnet_fbeta_score: 0.5714 - capsnet_acc: 0.5739 - decoder_precision: 0.4951 - decoder_recall: 0.5007 - decoder_fbeta_score: 0.4961 - decoder_acc: 0.0145 - val_loss: 1.3704 - val_capsnet_loss: 0.6772 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5821 - val_capsnet_recall: 0.5773 - val_capsnet_fbeta_score: 0.5805 - val_capsnet_acc: 0.5814 - val_decoder_precision: 0.4935 - val_decoder_recall: 0.4997 - val_decoder_fbeta_score: 0.4947 - val_decoder_acc: 0.0133\n",
      "Epoch 14/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3712 - capsnet_loss: 0.6781 - decoder_loss: 0.6932 - capsnet_precision: 0.5780 - capsnet_recall: 0.5506 - capsnet_fbeta_score: 0.5711 - capsnet_acc: 0.5734 - decoder_precision: 0.4960 - decoder_recall: 0.5005 - decoder_fbeta_score: 0.4968 - decoder_acc: 0.0143Epoch 00014: val_loss did not improve\n",
      "106/106 [==============================] - 75s 707ms/step - loss: 1.3715 - capsnet_loss: 0.6783 - decoder_loss: 0.6932 - capsnet_precision: 0.5778 - capsnet_recall: 0.5498 - capsnet_fbeta_score: 0.5708 - capsnet_acc: 0.5731 - decoder_precision: 0.4959 - decoder_recall: 0.5005 - decoder_fbeta_score: 0.4967 - decoder_acc: 0.0143 - val_loss: 1.3717 - val_capsnet_loss: 0.6785 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5788 - val_capsnet_recall: 0.5809 - val_capsnet_fbeta_score: 0.5786 - val_capsnet_acc: 0.5773 - val_decoder_precision: 0.4935 - val_decoder_recall: 0.4994 - val_decoder_fbeta_score: 0.4946 - val_decoder_acc: 0.0133\n",
      "Epoch 15/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3721 - capsnet_loss: 0.6789 - decoder_loss: 0.6932 - capsnet_precision: 0.5799 - capsnet_recall: 0.5652 - capsnet_fbeta_score: 0.5760 - capsnet_acc: 0.5781 - decoder_precision: 0.4956 - decoder_recall: 0.5005 - decoder_fbeta_score: 0.4965 - decoder_acc: 0.0144Epoch 00015: val_loss improved from 1.37036 to 1.36969, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 81s 767ms/step - loss: 1.3717 - capsnet_loss: 0.6785 - decoder_loss: 0.6932 - capsnet_precision: 0.5808 - capsnet_recall: 0.5666 - capsnet_fbeta_score: 0.5770 - capsnet_acc: 0.5791 - decoder_precision: 0.4958 - decoder_recall: 0.5005 - decoder_fbeta_score: 0.4967 - decoder_acc: 0.0143 - val_loss: 1.3697 - val_capsnet_loss: 0.6765 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5825 - val_capsnet_recall: 0.5882 - val_capsnet_fbeta_score: 0.5831 - val_capsnet_acc: 0.5832 - val_decoder_precision: 0.4935 - val_decoder_recall: 0.4994 - val_decoder_fbeta_score: 0.4946 - val_decoder_acc: 0.0133\n",
      "Epoch 16/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3683 - capsnet_loss: 0.6752 - decoder_loss: 0.6932 - capsnet_precision: 0.5829 - capsnet_recall: 0.5673 - capsnet_fbeta_score: 0.5785 - capsnet_acc: 0.5795 - decoder_precision: 0.4951 - decoder_recall: 0.5004 - decoder_fbeta_score: 0.4960 - decoder_acc: 0.0140Epoch 00016: val_loss did not improve\n",
      "106/106 [==============================] - 86s 816ms/step - loss: 1.3683 - capsnet_loss: 0.6751 - decoder_loss: 0.6932 - capsnet_precision: 0.5823 - capsnet_recall: 0.5663 - capsnet_fbeta_score: 0.5779 - capsnet_acc: 0.5789 - decoder_precision: 0.4949 - decoder_recall: 0.5004 - decoder_fbeta_score: 0.4959 - decoder_acc: 0.0140 - val_loss: 1.3717 - val_capsnet_loss: 0.6786 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5836 - val_capsnet_recall: 0.5655 - val_capsnet_fbeta_score: 0.5794 - val_capsnet_acc: 0.5795 - val_decoder_precision: 0.4935 - val_decoder_recall: 0.4995 - val_decoder_fbeta_score: 0.4946 - val_decoder_acc: 0.0133\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 17/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3685 - capsnet_loss: 0.6753 - decoder_loss: 0.6932 - capsnet_precision: 0.5867 - capsnet_recall: 0.5735 - capsnet_fbeta_score: 0.5830 - capsnet_acc: 0.5848 - decoder_precision: 0.4966 - decoder_recall: 0.5004 - decoder_fbeta_score: 0.4973 - decoder_acc: 0.0138Epoch 00017: val_loss improved from 1.36969 to 1.36935, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 76s 716ms/step - loss: 1.3685 - capsnet_loss: 0.6753 - decoder_loss: 0.6932 - capsnet_precision: 0.5855 - capsnet_recall: 0.5722 - capsnet_fbeta_score: 0.5818 - capsnet_acc: 0.5837 - decoder_precision: 0.4965 - decoder_recall: 0.5004 - decoder_fbeta_score: 0.4972 - decoder_acc: 0.0139 - val_loss: 1.3694 - val_capsnet_loss: 0.6762 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5802 - val_capsnet_recall: 0.5936 - val_capsnet_fbeta_score: 0.5823 - val_capsnet_acc: 0.5814 - val_decoder_precision: 0.4935 - val_decoder_recall: 0.4994 - val_decoder_fbeta_score: 0.4946 - val_decoder_acc: 0.0133\n",
      "Epoch 18/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3700 - capsnet_loss: 0.6768 - decoder_loss: 0.6932 - capsnet_precision: 0.5755 - capsnet_recall: 0.5619 - capsnet_fbeta_score: 0.5716 - capsnet_acc: 0.5731 - decoder_precision: 0.4969 - decoder_recall: 0.5005 - decoder_fbeta_score: 0.4975 - decoder_acc: 0.0139Epoch 00018: val_loss improved from 1.36935 to 1.36907, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 75s 708ms/step - loss: 1.3698 - capsnet_loss: 0.6766 - decoder_loss: 0.6932 - capsnet_precision: 0.5760 - capsnet_recall: 0.5616 - capsnet_fbeta_score: 0.5719 - capsnet_acc: 0.5734 - decoder_precision: 0.4966 - decoder_recall: 0.5005 - decoder_fbeta_score: 0.4973 - decoder_acc: 0.0140 - val_loss: 1.3691 - val_capsnet_loss: 0.6759 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5873 - val_capsnet_recall: 0.5800 - val_capsnet_fbeta_score: 0.5853 - val_capsnet_acc: 0.5859 - val_decoder_precision: 0.4935 - val_decoder_recall: 0.4994 - val_decoder_fbeta_score: 0.4946 - val_decoder_acc: 0.0133\n",
      "Epoch 19/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3711 - capsnet_loss: 0.6779 - decoder_loss: 0.6932 - capsnet_precision: 0.5790 - capsnet_recall: 0.5723 - capsnet_fbeta_score: 0.5765 - capsnet_acc: 0.5774 - decoder_precision: 0.4946 - decoder_recall: 0.5004 - decoder_fbeta_score: 0.4957 - decoder_acc: 0.0145Epoch 00019: val_loss improved from 1.36907 to 1.36821, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 75s 706ms/step - loss: 1.3709 - capsnet_loss: 0.6778 - decoder_loss: 0.6932 - capsnet_precision: 0.5791 - capsnet_recall: 0.5725 - capsnet_fbeta_score: 0.5766 - capsnet_acc: 0.5775 - decoder_precision: 0.4945 - decoder_recall: 0.5004 - decoder_fbeta_score: 0.4956 - decoder_acc: 0.0146 - val_loss: 1.3682 - val_capsnet_loss: 0.6750 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5848 - val_capsnet_recall: 0.5845 - val_capsnet_fbeta_score: 0.5842 - val_capsnet_acc: 0.5850 - val_decoder_precision: 0.4935 - val_decoder_recall: 0.4994 - val_decoder_fbeta_score: 0.4946 - val_decoder_acc: 0.0133\n",
      "Epoch 20/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 1.3658 - capsnet_loss: 0.6726 - decoder_loss: 0.6932 - capsnet_precision: 0.5936 - capsnet_recall: 0.5679 - capsnet_fbeta_score: 0.5874 - capsnet_acc: 0.5887 - decoder_precision: 0.4978 - decoder_recall: 0.5004 - decoder_fbeta_score: 0.4982 - decoder_acc: 0.0133Epoch 00020: val_loss did not improve\n",
      "106/106 [==============================] - 78s 732ms/step - loss: 1.3656 - capsnet_loss: 0.6724 - decoder_loss: 0.6932 - capsnet_precision: 0.5943 - capsnet_recall: 0.5678 - capsnet_fbeta_score: 0.5879 - capsnet_acc: 0.5892 - decoder_precision: 0.4979 - decoder_recall: 0.5004 - decoder_fbeta_score: 0.4983 - decoder_acc: 0.0133 - val_loss: 1.3686 - val_capsnet_loss: 0.6754 - val_decoder_loss: 0.6932 - val_capsnet_precision: 0.5898 - val_capsnet_recall: 0.5891 - val_capsnet_fbeta_score: 0.5890 - val_capsnet_acc: 0.5886 - val_decoder_precision: 0.4935 - val_decoder_recall: 0.4993 - val_decoder_fbeta_score: 0.4946 - val_decoder_acc: 0.0133\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARoAAAG0CAYAAADtpegkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3Xl4VOXZ+PHvnclkg4SwgyACCiiEhB0BWQRRsYpCpdQ99EVeXhForaIt2qYttFWpVfn5iiuKWgEXFkutbykgKIICDbKJCCIGA4QkZCFkm3l+f5yTYRKyTMKcbNyf68o1c855znPuOZlzz1mfR4wxKKWUk0LqOgClVOOniUYp5ThNNEopx2miUUo5ThONUspxmmiUUo7TRKN8RMQlIrki0imYZWsQxzwReS3Y9aq6E1rXAaiaE5Fcv8EooADw2MP/bYx5qzr1GWM8QNNgl1VKE00DZozxbegichiYaoxZW1F5EQk1xhTXRmxK+dNDp0bMPgRZJiJvi0gOcKeIDBGRLSJySkRSReRZEXHb5UNFxIhIZ3v4TXv6hyKSIyKfiUiX6pa1p48Tka9FJEtEForIpyKSGODnmCAie+yY14lID79pvxaRH0QkW0S+EpFR9vgrRWSHPf64iDwZhFWqakgTTeM3Afgb0AxYBhQDs4FWwDDgeuC/K5n/duAxoAVwBPhDdcuKSBtgOfCQvdxvgUGBBC8iVwBvADOB1sBaYLWIuEWklx17P2NMDDDOXi7AQuBJe/xlwLuBLE85QxNN4/eJMeYDY4zXGHPGGPOFMWarMabYGHMIeBEYWcn87xpjthljioC3gD41KHsjkGyMWWVP+ytwMsD4fwqsNsass+f9M1bSHIyVNCOAXvZh4bf2ZwIoArqJSEtjTI4xZmuAy1MO0ETT+H3vPyAil4vIGhE5JiLZwO+x9jIqcszvfR6VnwCuqOxF/nEY60nelABiL5n3O795vfa8HYwx+4FfYn2GE/YhYju76BSgJ7BfRD4XkRsCXJ5ygCaaxq/s4/kvALuBy+zDit8A4nAMqUDHkgEREaBDgPP+AFziN2+IXddRAGPMm8aYYUAXwAX8yR6/3xjzU6AN8BfgPRGJOP+PompCE82FJxrIAk7b5z8qOz8TLH8H+onITSISinWOqHWA8y4HxovIKPuk9UNADrBVRK4QkatFJBw4Y/95AUTkLhFpZe8BZWElXG9wP5YKlCaaC88vgXuwNtYXsE4QO8oYcxyYDDwFpAOXAv/Buu+nqnn3YMX7PJCGdfJ6vH2+Jhx4Aut8zzGgOTDXnvUGYJ99tW0BMNkYUxjEj6WqQbThK1XbRMSFdUh0qzFmU13Ho5ynezSqVojI9SISax/mPIZ1VejzOg5L1RJNNKq2XAUcwjr8uQ6YYIyp8tBJNQ566KSUcpzu0SilHKeJRinluDp7ertVq1amc+fOdbV4pVQNbN++/aQxJtB7oHzqLNF07tyZbdu21dXilVI1ICLfVV3qXHropJRynCYapZTjNNEopRyniUYp5ThNNEopx2nj5ErVMq/XUOT1UuwxFHm8FHkMxX7DxV77NdDpXkNxybBdrtgeX1RcpnzJdK9Vb5HHS4+20fzqhisc/cyaaFSDYYzxbXQlG4n/xlfs8Rtvl/HfuKyNtvSGVmzPW+gpvYEW+23gRcXnbqDFfht8yXCphFA2kfjV662lp37cLsHtCiE0xH51CaEhIbhdQqg9Piw0hNwC5zvG0ERzATDG2BtiyQZV+lfOf8Mt2SjOFHl4eu0BUjLPcHGLSBKHdsbjLW/DLb0Rnd34SsqU/2t89he2/F/jwnLm89TSFuq2N8hQe0N1l7OBlmy47pAQItwhhIaHnjNfaIhVPqxkPrt8hdPLSQi+5VQ1vUy9rhDBasiwftBEUwljDKcLPeTkF5F9ppic/CJy8ovJzi8iO99v+EwRPdpFc3m7GN9GdyI7n999sNf3a/E/oy7l+Q0H6/gT1czJ3AL+cyS5ynLWhmN96d2hZTeM0htJmCuEsNAQolwhuO35Qn3v/TYi3wZe/q/x2Q277MZ3tq6zMZWuw396WD3dQBuLep9o1n91gmPZ+aV+Pcs75qxo+jm72d5zd7cPp+c5/jnqMsncGN+ey9tFl/trXLKBh5X5NQ4RYfXOowB0axPNiO6tzvm1doeGnK1DN1BViXqfaBZ9fJCt32aUOy1E8P0yuUNL//qd3QBK72I2dYf6dllLfgXPJ9G4XUJ0hJtxce0YF9fe9+vqNfB/e44RHeEmOiKUUT3akJZTgNcYoiNCibHHNw0PJdRVPy/+Dbm0ZV2HoBqJOmuPZsCAASaQZ51O5OTj8RrrWNn3C2wlkpAQ/QVVqjaJyHZjzIDqzlfv92jaRGsPGUo1dPVzn10p1ahoolFKOU4TjVLKcZpolFKO00SjlHJclYlGRF4VkRMisruSMqNEJFlE9ojIx8ENUSnV0AWyR/MaVn/H5RKRWOB/sfpD7gVMCk5oSqnGospEY4zZCJR/a67lduB9Y8wRu/yJIMWmlGokgnGOpjvQXEQ2iMh2Ebk7CHUqpRqRYNwZHAr0B8YAkcBnIrLFGPN12YIiMg2YBtCpU6cgLFop1RAEY48mBfjIGHPaGHMS2AgklFfQGPOiMWaAMWZA69bV7oNKKdVABSPRrAKuEpFQEYkCBgP7glCvUqqRqPLQSUTeBkYBrUQkBfgt4AYwxiwyxuwTkX8CXwJe4GVjTIWXwpVSF54qE40x5rYAyjwJPBmUiJRSjY7eGayUcpwmGqWU4zTRKKUcp4lGKeU4TTRKKcdpolFKOU4TjVLKcZpolFKO00SjlHKcJhqllOM00SilHKeJRinlOE00SinHaaJRSjlOE41SynGaaJRSjtNEo5RynCYapZTjNNEopRyniUYp5ThNNEopx2miUUo5ThONUspxVSYaEXlVRE6ISKWdwonIQBEpFpFbgxeeUqoxCGSP5jXg+soKiIgLeBz4vyDEpJRqZKpMNMaYjUBGFcVmAu8BJ4IRlFKqcTnvczQi0gGYADx//uEopRqjKvveDsDTwMPGGK+IVFpQRKYB0wA6deoUhEUr1cAYA54i8BZZr/7vvcXnN81TeG65CqcV26+F0KYXXP9HRz92MBLNAGCpnWRaATeISLExZmXZgsaYF4EXAQYMGGCCsGx1oSi7gQa8URb7laloYys7f2XTytuQCytZbmHpuoyndtZXiBtcbvs1FFxhZ9/7ptnjvUWOh3PeicYY06XkvYi8Bvy9vCSj6ogxFW9cJcOewnOnFZ6GdX+AzO+gRRcYMgOMt5xfybJ1ns+vayUbtbe4dtZXSNkN0W1vpKGlN1z/Mu6o0uWrNb/9Wt60UnX5Twsrpw6/ZBLigiqOLmpblYlGRN4GRgGtRCQF+C3gBjDGLHI0urpkDBTlQX42FGRbr/lZUJB19n1+lj0tC9rFw0V9zm4g2T/A338B2Dtug/8HtjbQ01jHd8PK/6m8jLgC/AX123jckYFtlOXWFehGWVnd5UyrZxtoYyHG1M0RzIABA8y2bduqLrj/Q8g+WubXruwvaGH1fh3L/opnHXH+A9el+J9Cm8sr3uBLdqH9p4kL9q0GBFp3h87DK04AIaEQovd+XghEZLsxZkB15wvGORpnbf5/8N0n5UyQ6u96hkWVP21nDRNNSCiEx0BEDFx+I3S//my9AN/86+z0LiMgz75LICIGwptBeDSEhtVs2bXhkiF1HYFqJOr/Hs3pk+D1lJM4XM4HqZQqpfHu0TRpVdcRqFpQVFRESkoK+fn5dR2KAiIiIujYsSNutzso9dX/RKMuCCkpKURHR9O5c2equh9LOcsYQ3p6OikpKXTp0qXqGQKgZ/BUvZCfn0/Lli01ydQDIkLLli2DunepiUbVG5pk6o9g/y800SilHKeJRinlOE00StUzGzZsYPPmzZWWSUpKYsGCBbUU0fnTRKNUPRNIomlo9PK2qnd+98Ee9v6QHdQ6e14Uw29v6lVpmSVLlrBgwQJEhPj4eH7yk58wb948CgsLadmyJW+99RZt27YlKSmJgwcP8s0333Dy5EnmzJnDvffeS2pqKpMnTyY7O5vi4mKef/55hg8fTtOmTZk9ezZ///vfiYyMZNWqVbRt25a0tDSmT5/OkSPWnelPP/00HTp0YNGiRbhcLt58800WLlzI8OHDK407OTmZ6dOnk5eXx6WXXsqrr75K8+bNefbZZ1m0aBGhoaH07NmTpUuX8vHHHzN79mzAOuG7ceNGoqOjg7OSK6GJRilgz549zJs3j82bN9OqVSsyMjIQEbZs2YKI8PLLL/PEE0/wl7/8BYAvv/ySLVu2cPr0afr27cuPfvQj3n77ba677jrmzp2Lx+MhLy8PgNOnT3PllVcyf/585syZw0svvcSjjz7K7Nmz+cUvfsFVV13FkSNHuO6669i3bx/Tp0+nadOmPPjggwHFfvfdd7Nw4UJGjhzJb37zG373u9/x9NNP8+c//5lvv/2W8PBwTp06BcCCBQt47rnnGDZsGLm5uURERDizQsvQRKPqnar2PJywbt06Jk2aRKtW1p3oLVq0YNeuXUyePJnU1FQKCwtL3bx28803ExkZSWRkJFdffTWff/45AwcO5Gc/+xlFRUXccsst9OnTB4CwsDBuvPFGAPr378+//vUvANauXcvevXt9dWZnZ5Obm1utuLOysjh16hQjR44E4J577mHSpEkAxMfHc8cdd3DLLbdwyy23ADBs2DAeeOAB7rjjDiZOnEjHjh1rsrqqTc/RKFWBmTNncv/997Nr1y5eeOGFUjewlb3PREQYMWIEGzdupEOHDiQmJrJkyRIA3G63r7zL5aK42Gpbx+v1smXLFpKTk0lOTubo0aM0bdo0aPGvWbOGGTNmsGPHDgYOHEhxcTGPPPIIL7/8MmfOnGHYsGF89dVXQVteZTTRKAWMHj2ad955h/T0dAAyMjLIysqiQ4cOALz++uulyq9atYr8/HzS09PZsGEDAwcO5LvvvqNt27bce++9TJ06lR07dlS6zGuvvZaFCxf6hpOTkwGIjo4mJycnoLibNWtG8+bN2bRpEwBvvPEGI0eOxOv18v3333P11Vfz+OOPk5WVRW5uLgcPHqR37948/PDDDBw4sNYSjR46KQX06tWLuXPnMnLkSFwuF3379iUpKYlJkybRvHlzRo8ezbfffusrHx8fz9VXX83Jkyd57LHHuOiii3j99dd58skncbvdNG3a1LdHU5Fnn32WGTNmEB8fT3FxMSNGjGDRokXcdNNN3HrrraxatSqgk8Gvv/6672Rw165dWbx4MR6PhzvvvJOsrCyMMcyaNYvY2Fgee+wx1q9fT0hICL169WLcuHFBWX9Vqf/NRKgLwr59+7jiiivqOoyAJCUlVetkbUNV3v+kps1E6KGTUspxeuikVDUlJSXV2rLmz5/PO++8U2rcpEmTmDt3bq3FEAyaaJSqx+bOndvgkkp59NBJKeU4TTRKKcdpolFKOa7KRCMir4rICRHZXcH0O0TkSxHZJSKbRSQh+GEq5bxg3pWrSgtkj+Y14PpKpn8LjDTG9Ab+gN23tlJKlagy0RhjNgIZlUzfbIzJtAe3ALXzlJZSDjHG8NBDDxEXF0fv3r1ZtmwZAKmpqYwYMYI+ffoQFxfHpk2b8Hg8JCYm+sr+9a9/rePo66dgX97+L+DDINepLjQfPgLHdgW3zna9YdyfAyr6/vvvk5yczM6dOzl58iQDBw5kxIgR/O1vfzunGYiShyF377bOLJQ0x6BKC1qiEZGrsRLNVZWUmQZMA+jUqVOwFq1UUH3yySfcdtttuFwu2rZty8iRI/niiy/KbQaia9euHDp0iJkzZ/KjH/2Ia6+9tq7Dr5eCkmhEJB54GRhnjEmvqJwx5kXsczgDBgyom4esVP0X4J5HbStpBmLNmjUkJibywAMPcPfdd7Nz504++ugjFi1axPLly3n11VfrOtR657wvb4tIJ+B94C5jzNfnH5JSdWv48OEsW7YMj8dDWloaGzduZNCgQeU2A3Hy5Em8Xi8//vGPmTdvXpVNQ1yoqtyjEZG3gVFAKxFJAX4LuAGMMYuA3wAtgf+1G/cprsnTnUrVFxMmTOCzzz4jISEBEeGJJ56gXbt25TYDcfToUaZMmYLX6wXgT3/6Ux1HXz9pMxGqXmhIzURcKLSZCKVUg6KJRinlOE00SinHaaJRSjlOE41SynGaaJRSjtNEo1Q5kpKSWLBgQb2u+7XXXuP+++8PQkTO00SjVD1X0rNlQ6aJRinb/Pnz6d69O1dddRX79+8H4ODBg1x//fX079+f4cOH+3p2PH78OBMmTCAhIYGEhAQ2b94MwFNPPUVcXBxxcXE8/fTTldZdWf2JiYlMnz6dwYMHM2fOnCpjP3z4MKNHjyY+Pp4xY8Zw5MgRAN555x3i4uJISEhgxIgRAOzZs4dBgwbRp08f4uPjOXDgQBDWXuW0FwRV7zz++eN8lRHcrlovb3E5Dw96uMLp27dvZ+nSpSQnJ1NcXEy/fv3o378/06ZNY9GiRXTr1o2tW7dy3333sW7dOmbNmsXIkSNZsWIFHo+H3Nxctm/fzuLFi9m6dSvGGAYPHuzrnra8uoEK6wdISUlh8+bNuFyuKj/fzJkzueeee7jnnnt49dVXmTVrFitXruT3v/89H330ER06dPA1YbFo0SJmz57NHXfcQWFhIR6PJwhruHKaaJQCNm3axIQJE4iKigJg/Pjx5Ofns3nzZiZNmuQrV1BQAMC6det8Xd66XC6aNWvGJ598woQJE2jSpAkAEydOZNOmTXi93nPqBsjNza2wfrD6bwokyQB89tlnvP/++wDcddddvr2gYcOGkZiYyE9+8hMmTpwIwJAhQ5g/fz4pKSlMnDiRbt26VXNtVZ8mGlXvVLbnUZu8Xi+xsbEkJyfXSf0lCet8LFq0iK1bt7JmzRr69+/P9u3buf322xk8eDBr1qzhhhtu4IUXXmD06NHnvazK6DkapbDamlm5ciVnzpwhJyeHDz74gKioKLp06eLrKdIYw86dOwEYM2YMzz//PAAej4esrCyGDx/OypUrycvL4/Tp06xYsYLhw4eXWzdATExMhfVX19ChQ1m6dCkAb731FsOHDwesc0CDBw/m97//Pa1bt+b777/n0KFDdO3alVmzZnHzzTfz5Zdf1nzFBUgTjVJAv379mDx5MgkJCYwbN46BAwcC1kb7yiuvkJCQQK9evVi1ahUAzzzzDOvXr6d3797079+fvXv30q9fPxITExk0aBCDBw9m6tSp9O3bt8K6K6u/uhYuXMjixYuJj4/njTfe4JlnngHgoYceonfv3sTFxTF06FASEhJYvnw5cXFx9OnTh927d3P33Xef59qrmjYToeoFbSai/tFmIpRSDYqeDFaqnlu8eLHvUKjEsGHDeO655+ooourTRKNUPTdlyhSmTJlS12GcFz10UvVGXZ0vVOcK9v9CE42qFyIiIkhPT9dkUw8YY0hPTyciIiJodeqhk6oXOnbsSEpKCmlpaXUdisJK/B07Bq93a000ql5wu9106dKlrsNQDtFDJ6WU46pMNCLyqoicEJHdFUwXEXlWRL4RkS9FpF/ww1RKNWSB7NG8BlxfyfRxQDf7bxrw/PmHpZRqTKpMNMaYjUBGJUVuBpYYyxYgVkTaBytApVTDF4xzNB2A7/2GU+xxSikF1PLJYBGZJiLbRGSbXsZU6sIRjERzFLjYb7ijPe4cxpgXjTEDjDEDWrduHYRFK6UagmAkmtXA3fbVpyuBLGNMahDqVUo1ElXesCcibwOjgFYikgL8FnADGGMWAf8AbgC+AfKAhv30l1Iq6KpMNMaY26qYboAZQYtIKdXo6J3BSinHaaJRSjlOE41SynGaaJRSjtNEo5RynCYapZTjNNEopRyniUYp5ThNNEopx2miUUo5ThONUspxmmiUUo7TRKOUcpwmGqWU4zTRKKUcp4lGKeU4TTRKKcdpolFKOU4TjVLKcZpolFKO00SjlHKcJhqllOM00SilHBdQohGR60Vkv4h8IyKPlDO9k4isF5H/iMiXInJD8ENVSjVUVSYaEXEBzwHjgJ7AbSLSs0yxR4Hlxpi+wE+B/w12oEqphiuQPZpBwDfGmEPGmEJgKXBzmTIGiLHfNwN+CF6ISqmGrsoucYEOwPd+wynA4DJlkoD/E5GZQBPgmqBEp5RqFIJ1Mvg24DVjTEfgBuANETmnbhGZJiLbRGRbWlpakBatlKrvAkk0R4GL/YY72uP8/RewHMAY8xkQAbQqW5Ex5kVjzABjzIDWrVvXLGKlVIMTSKL5AugmIl1EJAzrZO/qMmWOAGMAROQKrESjuyxKKSCARGOMKQbuBz4C9mFdXdojIr8XkfF2sV8C94rITuBtINEYY5wKWinVsARyMhhjzD+Af5QZ9xu/93uBYcENTSnVWOidwUopx2miUUo5ThONUspxmmiUUo7TRKOUcpwmGqWU4zTRKKUcp4lGKeU4TTRKKcdpolFKOS6gRxDqUl5RHl7jdXQZHuPh5JmT51VHdFg0UaFRvmGD4UTeCd9wm6g2ZBVkUeApOK/lVJeI0CayTY3mTc9Pp9hbTERoBM3CmgU5MlVfhEgIUe6oqgueh3qfaGb8ewbbjm+r6zCUarQGtB3A4usXO7qMep9oJl8+mVEXj3J0GZ+lfsanRz89rzoiQyOZ0WeGb/jbrG9578B7vuGxl4zlX9/967yWUVN3XnEn7Zq0q9Y86fnpLN599ss3s+9Mwl3hwQ5N1QNtm7R1fBlSV605DBgwwGzbpnsqSjUkIrLdGDOguvPpyWCllOM00SilHKeJRinlOE00SinHaaJRSjlOE41SynF1dnlbRNKA7+pk4ZZWwPndDuwsja/m6nNs0LDju8QYU+1O2eos0dQ1EdlWk/sBaovGV3P1OTa4MOPTQyellOM00SilHHchJ5oX6zqAKmh8NVefY4MLML4L9hyNUqr2XMh7NEqpWtIoEo2IvCoiJ0RkdyVlRolIsojsEZGP7XE97HElf9ki8nN7WpKIHPWbdkNtx2eP/4U9breIvC0iEfb4LiKyVUS+EZFlIhJWj2J7TUS+9Vt3fWoSWxDim23Htqfk/2qPbyEi/xKRA/Zr83oWX61990TkIb/l7BYRj4i0sKddLyL77e/YI37zVP+7Z4xp8H/ACKAfsLuC6bHAXqCTPdymnDIu4BjWfQIAScCDdRkf0AH4Foi0h5cDiX7vf2q/XwT8Tz2K7TXg1jped3HAbiAKq92ltcBl9rQngEfs948Aj9ez+Grtu1em7E3AOr/t4SDQFQgDdgI9a/rdaxR7NMaYjUBGJUVuB943xhyxy58op8wY4KAxJug3EZ5nfKFApIiEYn0pfxARAUYD79plXgduqQ+x1SQGh+K7AthqjMkzxhQDHwMT7Wk3Y60zOI9152B8QRNAfP5uA9623w8CvjHGHDLGFAJLgZtr+t1rFIkmAN2B5iKyQUS2i8jd5ZT5KWdXcon7ReRLe/ezxrvXNY3PGHMUWAAcAVKBLGPM/wEtgVP2FxQgBWsPoz7EVmK+ve7+KiJONs1X0f92NzBcRFqKSBRwA3CxPa2tMSbVfn8McLKJuZrEB7X33QPAjuF6oKRZyA7A935FSr5jNfruXSiJJhToD/wIuA54TES6l0y0jzHHA+/4zfM8cCnQB2tD+kttx2d/wW4GugAXAU1E5E4H4whWbL8CLgcGAi2Ah2s7PmPMPuBx4P+AfwLJgKfszMba/3fy0mtN4qvN716Jm4BPjTGB7v1Uy4WSaFKAj4wxp40xJ4GNQILf9HHADmPM8ZIRxpjjxhiPMcYLvIS1K1nb8V0DfGuMSTPGFAHvA0OBdCDWPmQB6AgcrSexYYxJNZYCYDF1s+4wxrxijOlvjBkBZAJf2/McF5H2APZreYfSdRZfLX/3SpTdoz9K6T2sku9Yjb57F0qiWQVcJSKh9i7iYGCf33T/Y1PA9wUsMQFrV7e24zsCXCkiUfax8Rhgn/0rvB641Z7/HruOOo8Nzq47e/wt1M26Q0Ta2K+dsM5//M2eZzXWOgNn112N4qvl7x4i0gwYSen18AXQzb7CFIaViFbX+LsXjDPbdf2HlSRSgSKsX5D/AqYD0/3KPIR19n838HO/8U2wsnSzMnW+AewCvsT6Yravo/h+B3xlj38DCLfHdwU+B77BOuQLr0exrbPX3W7gTaBpHa27Tfb4ncAYv/EtgX8DB7Cu9rSoZ/HV9ncvEVhazrw3YO1lHQTm+o2v9ndP7wxWSjnuQjl0UkrVIU00SinHaaJRSjlOE00FRMQlIrn2FYGgla1LInKZiAT9pJyIXCMih/2G94vI8EDK1mBZL4vIr2s6v6ob9b7v7UCJSK7fYBRQwNkboP7bGPNWdeozxniApsEueyEwxvQIRj0iMhW40xgzyq/uqcGoW9WuRpNojDG+Dd3+xZxqjFlbUXkRCTVnb6NWqk419u/jBXPoJCLz7Efa3xaRHOBOERkiIltE5JSIpIrIsyLitsuHiogRkc728Jv29A9FJEdEPhORLtUta08fJyJfi0iWiCwUkU9FJLGCuAOJ8b/FemQ/U0Se9ZvXZT9rlC4ih7CeZalo/cwVkaVlxj0nIk/Z76eKyD778xy09zYqqitFREbZ76NE5A07tj1Yt+P7l31URA7Z9e4RkfH2+N7A/8N6HihXRE76rdskv/mn2589XURW+t0sWOm6qc56LolHRNaKSIaIHBOROX7LecxeJ9kisk1ELirvMFVEPin5P9vrc6O9nAzgURHpJiLr7WWctNdbM7/5L7E/Y5o9/RkRibBjvsKvXHsRyRORlhV93lpX0xuB6vMfcBi4psy4eUAh1jMdIUAk1rM4g7H27Lpi3Zx0v10+FOsZmM728JtYXVAMANzAMuDNGpRtA+RgPSfkBh7AupkqsYLPEkiMq4BmQGesJ3WvsaffD+zBuk28Jdbt76aC5XQFcoEmfnWfAAbYwzfZZUqe3j0DxNvTrgEO+9WVAoyy3y8ANgDNgUuwblDzL/sToL39P7ndjqGtPW0qsKFMnG8CSfb7a+0Y+wARwP9ytpmDStdNNddzM+A4MBsIB2KAQfYZTu2kAAAgAElEQVS0X2HdcNfN/gx9sJ7vuqzsugY+4WxTGlOBYuB/sJpkiMR6AHMMVrMMbYBPgQV+n2e3vT6b2OWH2dNeBOb7LeeXwIq63g5Lffa6DsCRD1VxollXxXwPAu+U+aL6J49FfmXHY7fxUc2yPwM2+U0TrDs3y000AcZ4pd/097HbMsFKLFP9pt1Q9stfpu4twO32+3HA/krK/h2YYb+vLNEc8f9fAPf5ly2n3t3Aj+z3VSWa14E/+k2LwTov17GqdVPN9XwX8EUF5Q6WxFtmfCCJ5lAVMdxaslxgONaT5q5yyg3Dahuo5AbcZGBisLer8/m7YA6dbP6PvSMil4vIGntXOBv4PVbnWRU55vc+j8pPAFdU9iL/OIz1zUipqJIAYwxoWVTdYd/fsJ77AmvvouTZIETkRrFaVcsQkVNYexOVrasS7SuLQUQSRWSnvft/Cuup70DqBevz+eozxmRjPZzo32xBQP+zKtbzxVgJpTyVTatK2e9jOxFZLlbretlYDYj5x3DYWBceSjHGfIq1d3SViMQBnYA1NYzJERdaoil7afcFrF/Qy4wxMcBvsPYwnJSK9YsL+B48rKw9j/OJMZXST+BWdfl9OXCNiHTAOrQrecgvEquhoz9hHdbEYjVvEEgcxyqKQUS6YjWJ8D9AS7ver/zqrepS/A9Yh2Ml9UVjHaLV5En2ytbz91jNNpSnommn7Zii/Ma1K1Om7Od7HOtqaW87hsQyMVwiIq4K4lgC3Im197XcWE/O1xsXWqIpKxrIAk7bJ9P+uxaW+Xegn4jcJNaj9rOByroYPZ8YlwM/F5EO9onBStuFMcYcw9q9fw3rsOmAPSkc67xBGuARkRuxziUEGsOvRSRWrPuM7veb1hRrY0vDyrn3Yu3RlDgOdPQ/KVvG28B/iUi8WI1r/QnrsLTCPcRKVLaeVwOdROR+EQkXkRgRKWm64WVgnohcKpY+YrW5e8z+u1Osk/LT8EuKlcRwGsgSkYuxDt9KfIb18O8fxTrBHikiw/ymv4F1qHU7VtKpVy70RPNLrMfcc7B+0ZY5vUBjtXkzGXgK64tzKfAfrF+yYMf4PNZTyruwHvt/t/LigLUXcw1+h03GmFPAL4AVWCdUb8VKmIH4Ldae1WHgQ/w2AmPMl8BCrCeBU4EewFa/ef+F9YT1cRHxPwQqmf+fWIc4K+z5OwF3BBhXWRWuZ2NMFjAW+DFW8vsaq1kFgCeBlVjrORvrxGyEfUh8L/BrrAsDl5X5bOX5LVbbM1lYya2ktTuMden7RqwmQL/HOvd1q9/0w1j/5wJjzOZqfnbH6dPbdczeFf4BqzHvTXUdj2q4RGQJ1gnmpLqOpaxGc8NeQyIi12Nd4TmDdXm0COtXXakasc933Qz0rutYynOhHzrVlauAQ1jnJq4DJtS3k3eq4RCRP2Hdy/NHY/e2UN/ooZNSynG6R6OUcpwmGqWU4+rsZHCrVq1M586d62rxSqka2L59+0ljTGX3fZUroERjXyV5Buvhr5eNMX8up8xPsPoMNsBOY8ztldXZuXNntm3bVt14lVJ1SERq1GV0lYnGvs/jOawbllKAL0RktTFmr1+ZbliXaYcZYzLF7q9GKaUgsHM05Xb2XabMvcBzxphMOKcjeKXUBS6QRFNRZ9/+ugPdxWrAaYt9qKWUUkDwTgaHYjX8MwrryeSNItLbfkbGx36wbBpAp071uh1vpVQQBbJHU1Fn3/5SsPrlLTLGfIv10Fm3shUZY140xgwwxgxo3braJ66VUg1UIHs0vs6+sRLMT7EeRfe3EqvBpMUi0grrUOpQMAL89++nU5zyA8XRkXhjmuCNaYKJboo0i4Zm0YTExEBsDGERUbhD3LhD3IS5wnzv3S43YSFhvvelXkPchIbo415KOa3KrcwYUywi9wMfYV3eftUYs0dEfg9sM8astqddKyJ7sZpSfMgYkx6MAHO+/oo2+0/QJM8Qdk7bYmfluyE3EnIiISdSyI2E3AhrODdSfO990+zp4nKVSlChIaG4Q9ycKT5DRn5GwHGOvWQsT454EleI1S6Rx2sFWzKs1IWszp51GjBggKnOfTTGGLx5eRScyqAw4ySFmekUncqkKCOD4lOZeLKy8J7KwpuVhcnKhuxcJDsXyTmNeMv/jEagOCocvF7cZ4p843dfIuy9WOwkdW6COhMGyLmNy7WJakOxt5i8ojzyPfkANHE3YXr8dKLcUUSGRhLuCiciNILY8FhaRLSgRUQLIkMjkXLqU6q+EZHtxpgB1Z6voSSamjJeL97cXDynTll/WVnWa+bZ95lvVatvuQod6dMelyuUUBNCSvYR0qPhvWEhZEZXnkQiXBE0j2hOi4gWvteSv/KGI0MjgxKvUtVV00TT6E9QSEgIrpgYXDExUMGVrhOpB3Gv24IJkQr3fgJxaXooIRER4HLR9hurnmv/4+F4LLz366HMGZXEGc8ZzhSfIasgi4z8DDLyM8jMzyz1/uCpg2TkZ1DgKb/liMjQSCvphDenRWQLokKjCJEQXOLCFeLCJS6auJsQ3zqefm360TpKT7yrutXo92gCYYwp99DFm5/P6c8+I3fDx+SuX0/xCes+RImKIjIujsiEBCL7JBAZH09omatoWatX88OcSpvo9Qlp2pSwzp0JueRiilvGUBjhIi9COBVaSKpkc7T4JN8XHedwfioFbihwQ2Go9VrsotzDOH83db2JCd0mUOQtoshTZL16iyj0FJb7vqRMoaeQ7ce30zSsKU+NeooWES0CW6Gq0dJDJ4cZYyjYvx+A8MsuQ0ID2xksOHCAo798kOK803hSatI4//n77R0u9nUK/ByQS1x4yvTqMan7JHq27EkTdxOauJsQFRpFlDuq1LCea2r8NNHUc8knkrnrw7sILzR0PQZN8g3Nc6FljqFlNrTMgTanDG2ygr/snMemwehhpS7rl7wPCwk7+94VRqiE4gpx8e8j/+bn639ereUIYiWf0CZEuf0Skf9wqJ2Y7OGm7qZ0iulEt9huhLnCgv/hVVBpomkAirxFhEpotX/1TVERxRmZFJ9Mw3PyJMUn0yk+eZLi9JPWcNpJitOtcd7s7HLrcDVrhqt1K0JbtiK0VStCW7XE1aoVoa1aE9qqpT2uFa4WLRCXC+/p02T94x+Ed+0KzZtR0DSc/EgXeZ4z5BXlcbr4NKeLTpNXlHfucHEep4vKHz5TbJ2jKitUQukS24UrWlzB5S0u9/1Fh0XXaF0rZ+jJ4AbAHVJR90SVE7cbd9s2uNtW/VC8t6AAj510/P/8E9SZ3bvwpJ3Em5dXzsIEV4sWeNLLuQ1KBFdMDFGxsUQ3b44rNrbMX5uz79vZ05vHEhJWek/F4/X4kk9uYS4Hsw7yVcZXfJXxFZt/2Mzqg6t9ZTs27cgVLa+gR/MeXNHSSkKtIlsRItpmW0OiezQXMG9enrUnlHbS2lsqeZ+eTvGxY+R+/LGvbLMfT8Tdtu3Z2wROnaK45DaBU6cwZ87dSykhUVG4Ypvhio0lNDYWV6xfkmrenPDu3YmM701IpHXZ/uSZk+xL38dXGV+xL2Mf+zP2cyTnbJvbLnERGx5L84jm1l948wrfd4rppLcDBJHu0ahqC4mKIiwqirCLL666cBW8BQVnk1DmqVIJ6ez4TDynTlF09AdrODsbSn7oQkOJ6NmTqL59iezfjyH9+jE8friv/tzCXPZn7md/xn5OnjlJZkEmmfnW34FTB8jMzySrIAtTppdZQbgk5hK6N+9O9+bd6dGiB92bd6d9k/Z64roW6R6NqjPG48Fz6hT5u3eTt30HZ3bs4MyuXZgC6/4h9yWdiOrbj8h+fYnq35+wrl0rTQ4er4eswixfAkrPT+fQqUN8nfk1+zP3833O2dZOot3RdGvezZd4ejTvwWXNL9O9nyroyWDVKJjCQvL37iVvx3/I27GdMzv+gyfDeubM1awZkX37EtGzJ+HduxHerRthl1wS8K0GeUV5fJ35te9vf8Z+vs78mrxi61xV2b2fTjGdaBbWjJjwGN9rdFj0BX1+SBONapSMMRQePsyZHf8h7z87OLPjPxQePgxeL2CdKA+79FLCu1mJJ7x7NyK6dSP0oosCOjTyFBVyNOMwB0/s49vj+/ku/Ru+Tz9ERtYxCt1wPBYK3WfrEYTosGhiwmJoFt6s0teY8JhSw43hPiNNNOqC4c3Pp/DQIQoOHKDgwAHyv/6aggPfUJya6isT0qQJYZdeioS5MQWFmIICvAX5vvfWcAF4KmkSwOZp2YzCdi043TaarNZRpLd0c7xFCEdjijkZeobsgmyyC7PJKsg650ZHf+4Qd7lJqWVkSzo06UCH6A5c1PQiLmpyERGhEUFZV8GmJ4PVBSMkIoKInj2J6Nmz1HhPTo6VfL62ElDBwYPg9RISG4uEhxESHoGEh1fxPhwJj0DCw/Dmnqbo+yMUHvmewiNHiN51hFZpaVzqt0xXbCzuTp0I63Q57k4XYzq0pcBOStlRkF2UQ1ZBli8RlbzPLsjmRN4JDmQeIO1MGkXeolKfpVVkKzo0tRJPx6YdrQRkv2/fpD1uV81ulagrukejVDV48/Io/D6FwiPfUXSkJAl9R9GR7ylKTfUd0oF1Vc99ySWEXXwxYZd0wn3xxYR1uoSwSzoR2rYtEmKd6/EaL2l5afxw+geO5h7laM5R633OUY7mHuXY6WMUm2JfvYLQJqoNHZp28CUj//ftmrRzrEE3PXRSqo6ZwkIKjx61EtB3Ryj8/mwSKkxJgaKzey0SFmYlnosvxhUbe/Yyv3153hhT8hbj9ZDvKSCv5M7qojzOFJ3hTLH1WuAp8JsfQowQHhpOlCuSiNAIIl2RRIZGEOGKINIVQZgrHN+ZImMI79aNNg/8IqDPqIdOStUxCQsjvEsXwrt0OWea8XgoSj1mHYp9d4TC74/4ElLB11/bFci5r773EAU0KUkREgrSDGhmNeDmLabYeCj2FltP4Jtiir1nKPJmc8YUk4fV0FtJZf7N2hp3Nk53xKaJRqlaIC4XYR07ENaxA02GDKnVZRd4CkjNTeWH3B9IyU3hh1zrEK1k+IqWTRnmcAyaaJRq5MJd4XRu1pnOzTqXO73YW1zu+GAK6M4jEbleRPaLyDci8kg50xNFJE1Eku2/qcEPVSnlhNroCSQofW/blhlj7ncgRqVUAxesvreVUqpCwep7G+DHIvKliLwrIuf/OLBSqtEI1tNhHwCdjTHxwL+A18srJCLTRGSbiGxLS0sL0qKVUvVdUPreNsakG2NK+gZ5GehfXkXa97ZSF6ZAEo2v720RCcPqe3u1fwERae83OB7YF7wQlVINXbD63p4lIuOBYiADSHQwZqVUA6PPOimlAlbTZ50u3KbClFK1RhONUspxmmiUUo7TRKOUcpwmGqWU4zTRKKUcp4lGKeU4TTRKKcdpolFKOU4TjVLKcZpolFKO00SjlHKcJhqllOM00SilHKeJRinlOE00SinHaaJRSjlOu8RV56WoqIiUlBTy8/PrOhQVRBEREXTs2BG32x2U+jTRqPOSkpJCdHQ0nTt3RkTqOhwVBMYY0tPTSUlJoUuXLkGpMyh9b/uV+7GIGBGpdpuiqmHKz8+nZcuWmmQaERGhZcuWQd1LrTLR+PW9PQ7oCdwmIj3LKRcNzAa2Bi061SBokml8gv0/DWbf238AHgf0YF0pVUpQ+t4WkX7AxcaYNUGMTakaSUpKYsGCBQ2u7sbsvC9vi0gI8BTwywDKat/b6oJSXFxc1yHUC8HoezsaiAM2iMhh4EpgdXknhLXvbeWU+fPn0717d6666ir2798PwMGDB7n++uvp378/w4cP56uvvgLg+PHjTJgwgYSEBBISEti8eTMATz31FHFxccTFxfH0009XWndl9ScmJjJ9+nQGDx7MnDlzyo33888/Z8iQIfTt25ehQ4f66vV4PDz44IPExcURHx/PwoULAfjiiy8YOnQoCQkJDBo0iJycnCCvQWcFcnnb1/c2VoL5KXB7yURjTBbQqmRYRDYADxpjtBvKC8zvPtjD3h+yg1pnz4ti+O1NvSots337dpYuXUpycjLFxcX069eP/v37M23aNBYtWkS3bt3YunUr9913H+vWrWPWrFmMHDmSFStW4PF4yM3NZfv27SxevJitW7dijGHw4MGMHDkSr9dbbt1AhfWDddl/8+bNuFyucmO+/PLL2bRpE6Ghoaxdu5Zf//rXvPfee7z44oscPnyY5ORkQkNDycjIoLCwkMmTJ7Ns2TIGDhxIdnY2kZGRQV3PTgtW39tK1ZlNmzYxYcIEoqKiABg/fjz5+fls3ryZSZMm+coVFBQAsG7dOpYsWQKAy+WiWbNmfPLJJ0yYMIEmTZoAMHHiRDZt2oTX6z2nboDc3NwK6weYNGlShUkGICsri3vuuYcDBw4gIhQVFQGwdu1apk+fTmiotWm2aNGCXbt20b59ewYOHAhATEzMeaytuhHQDXvGmH8A/ygz7jcVlB11/mGphqiqPY/a5PV6iY2NJTk5uU7qL0lYFXnssce4+uqrWbFiBYcPH2bUqFEORFl/6LNOqsEbMWIEK1eu5MyZM+Tk5PDBBx8QFRVFly5deOeddwDrbtedO3cCMGbMGJ5//nnAOieSlZXF8OHDWblyJXl5eZw+fZoVK1YwfPjwcusGa6+iovoDkZWVRYcO1sXb1157zTd+7NixvPDCC76TyBkZGfTo0YPU1FS++OILAHJychrcSWZNNKrB69evH5MnTyYhIYFx48b5DjHeeustXnnlFRISEujVqxerVq0C4JlnnmH9+vX07t2b/v37s3fvXvr160diYiKDBg1i8ODBTJ06lb59+1ZYd2X1B2LOnDn86le/om/fvqWSxtSpU+nUqRPx8fEkJCTwt7/9jbCwMJYtW8bMmTNJSEhg7NixDe7ZMjHG1MmCBwwYYLZt0/PFDd2+ffu44oor6joM5YDy/rcist0YU+1HjHSPRinlOH16WykHLV68mGeeeabUuGHDhvHcc8/VUUR1QxONUg6aMmUKU6ZMqesw6pweOimlHKeJRinlOE00SinHaaJRSjlOE426oDRt2jTgsomJibz77ruOxeJ0/fWJJhqlGoiG9tiBP000qkF75JFHSt2TkpSUxLx58xgzZgz9+vWjd+/eAT8aYIzh/vvvp0ePHlxzzTWcOHHCN2379u2MHDmS/v37c91115GamgrAN998wzXXXENCQgL9+vXj4MGDGGN46KGHiIuLo3fv3ixbtqzG9Y8aNYqf//znDBgw4Jz7cUp88MEHDB48mL59+3LNNddw/PhxwHrCfMqUKfTu3Zv4+Hjee+89AP75z3/Sr18/EhISGDNmTKCr+rzofTQqeD58BI7tCm6d7XrDuD9XOHny5Mn8/Oc/Z8aMGQAsX76cjz76iFmzZhETE8PJkye58sorGT9+fJUNbq9YsYL9+/ezd+9ejh8/Ts+ePfnZz35GUVERM2fOZNWqVbRu3Zply5Yxd+5cXn31Ve644w4eeeQRJkyYQH5+Pl6vl/fff5/k5GR27tzJyZMnGThwICNGjOCzzz6rdv0AhYWFVPa4zlVXXcWWLVsQEV5++WWeeOIJ/vKXv/CHP/yBZs2asWuX9T/JzMwkLS2Ne++9l40bN9KlSxcyMjKq+x+pEU00qkHr27cvJ06c4IcffiAtLY3mzZvTrl07fvGLX7Bx40ZCQkI4evQox48fp127dpXWtXHjRm677TZcLhcXXXQRo0ePBmD//v3s3r2bsWPHAtYT3+3btycnJ4ejR48yYcIEwOp0DeCTTz7x1dO2bVtGjhzJF198Ue36S0yePLnSuFNSUpg8eTKpqakUFhb6+mJau3YtS5cu9ZVr3rw5H3zwASNGjPCVadGiRWAr+jxpolHBU8meh5MmTZrEu+++y7Fjx5g8eTJvvfUWaWlpbN++HbfbTefOnc/raWdjDL169eKzzz4rNT5YzWlWVH+Jqtq2mTlzJg888ADjx49nw4YNJCUlBSWuYNJzNKrBmzx5MkuXLuXdd99l0qRJZGVl0aZNG9xuN+vXr+e7774LqJ4RI0awbNkyPB4PqamprF+/HoAePXqQlpbmSwRFRUXs2bOH6OhoOnbsyMqVKwGrhb28vDyGDx/uqyctLY2NGzcyaNCgatcfKP+2bV5//XXf+LFjx5Y6f5WZmcmVV17Jxo0b+fbbbwFq7dBJE41q8Hr16kVOTg4dOnSgffv23HHHHWzbto3evXuzZMkSLr/88oDqmTBhAt26daNnz57cfffdDBkyBICwsDDeffddHn74YRISEujTp4+vQfM33niDZ599lvj4eIYOHcqxY8eYMGGCrz2Z0aNH88QTT9CuXbsa1R+IpKQkJk2aRP/+/WnVytd8N48++iiZmZnExcWRkJDA+vXrad26NS+++CITJ04kISGhysOyYNH2aNR50fZoGq9ab4+mqr63RWS6iOwSkWQR+aS8LnOVUheuKk8G+/W9PRarl8ovRGS1MWavX7G/GWMW2eXHY3Uod70D8Sp13nbt2sVdd91Valx4eDhbt9bvbuPnz5/va6O4xKRJk5g7d24dRRS4QK46+freBhCRkr63fYnGGOPfmU8ToG6Ox5QKQO/evR3rHcFJc+fObRBJpTyBJJry+t4eXLaQiMwAHgDCgNFBiU4p1SgE7aqTMeY5Y8ylwMPAo+WV0b63lbowBaPv7bKWAreUN0H73lbqwhRIovH1vS0iYVh9b5fqBldEuvkN/gg4ELwQlVINXZWJxhhTDJT0vb0PWF7S97Z9hQngfhHZIyLJWOdp7nEsYqXOQ3Xao6lrK1euZO/evVUXbACC0ve2MWZ2kONS6oK3cuVKbrzxRnr2bPi3pelDlSpoHv/8cb7K+CqodV7e4nIeHvRwhdMfeeQRLr74Yl8zEUlJSYSGhrJ+/XoyMzMpKipi3rx53HzzzQEt7/HHH+fNN98kJCSEcePG8ec//5mXXnqJF198kcLCQi677DLeeOMNoqKiSExMJCIigm3btpGdnc1TTz3FjTfeyJ49e5gyZQqFhYV4vV7ee+893G4348aN46qrrmLz5s106NCBVatWERkZycGDB5kxYwZpaWlERUXx0ksvkZGRwerVq/n444+ZN28e7733Hpdeeuk58VYU2/Hjx5k+fTqHDh0C4Pnnn2fo0KEsWbKEBQsWICLEx8fzxhtv1OC/Un36rJNq0CZPnszy5ct9w8uXL+eee+5hxYoV7Nixg/Xr1/PLX/6SQB61+fDDD1m1ahVbt25l586dzJkzB4CJEyfyxRdfsHPnTq644gpeeeUV3zyHDx/m888/Z82aNUyfPp38/HwWLVrE7NmzSU5OZtu2bXTs2BGAAwcOMGPGDPbs2UNsbKyvIapp06axcOFCtm/fzoIFC7jvvvsYOnQo48eP58knnyQ5ObncJFNZbLNmzWLkyJHs3LmTHTt20KtXL/bs2cO8efNYt24dO3furLAhLSfoHo0Kmsr2PJwSzPZo1q5dy5QpU4iKigLOttWye/duHn30UU6dOkVubi7XXXedb56f/OQnhISE0K1bN7p27cpXX33FkCFDmD9/PikpKUycOJFu3axrJV26dKFPnz4A9O/fn8OHD5Obm8vmzZuZNGmSr86CgoKAP39Fsa1bt44lS5YA4HK5aNasGUuWLGHSpEm+By9rqy0a0ESjGgGn26NJTExk5cqVJCQk8Nprr7FhwwbftLKt9okIt99+O4MHD2bNmjXccMMNvPDCC3Tt2pXw8HBfOZfLxZkzZ/B6vcTGxtb4TuXKYqtP9NBJNXjBao9m7NixLF68mLy8POBsWy05OTm0b9+eoqIi3nrrrVLzvPPOO3i9Xg4ePMihQ4fo0aMHhw4domvXrsyaNYubb76ZL7/8ssJlxsTE0KVLF98zTMYYdu7cCUB0dHSVjWtVFNuYMWN4/vnnAavFvqysLEaPHs0777xDenp6qc9XGzTRqAYvWO3RXH/99YwfP54BAwbQp08fFixYAMAf/vAHBg8ezLBhw86pq1OnTgwaNIhx48axaNEiIiIiWL58OXFxcfTp04fdu3dz9913V7rct956i1deeYWEhAR69erla0z9pz/9KU8++SR9+/bl4MGD5c5bUWzPPPMM69evp3fv3vTv35+9e/fSq1cv5s6dy8iRI0lISOCBBx4IaL0Eg7ZHo87LhdweTWJiIjfeeCO33nprXYfiiFpvj0Yppc6HngxWF5xgtUfz2muvBTGqys2YMYNPP/201LjZs2czZcqUWovhfGiiURechtgejX8j4w2RHjoppRyniUYp5ThNNEopx2miUSqINmzYUK0+mS4UmmiUCiJNNOXTRKMahSVLlvh6h7zrrrv44IMPGDx4MH379uWaa67h+PHjgNWMxF133cWQIUPo1q0bL730EgCpqamMGDGCPn36EBcXx6ZNmwCroay5c+eSkJDAlVde6asnLS2NH//4xwwcOJCBAwfy6aefcvjwYRYtWsRf//pX+vTp46ujrIpiy83NZcqUKfTu3Zv4+Hjf093//Oc/6devHwkJCYwZM8bR9egUvTNYnRf/u0eP/fGPFOwLbns04VdcTrtf/7rSMnv27GHChAls3ryZVq1akZGRgYgQGxuLiPDyyy+zb98+/vKXv5CUlMSKFSvYsmULp0+fpm/fvmzdupW3336b/Px85s6di8fjIS8vj+joaESE1atXc9NNNzFnzhxiYmJ49NFHuf3227nvvvu46qqrOHLkCNdddx379u0jKSmJpk2b8uCDD1YYb2ZmZrmxPfzwwxQUFPD000/7yhUXF9OvXz82btxIly5dyMjIqLWnroN5Z7DeR6MavHXr1p3T/MGuXbuYPHkyqampFBYW0qVLF1/5m2++mcjISCIjI7n66qv5/PPPGThwID/72c8oKirilltu8TXnEBYWxo033ghYTTv861//AqwmJfyb2ZcO8OsAAAtrSURBVMzOziY3NzegeFNSUsqNbe3atSxdutRXrnnz5nzwwQeMGDHCV6Y2m3YIJk00Kmiq2vOoTTNnzuSBBx5g/PjxbNiwgaSkJN+08pp2GDFiBBs3bmTNmjUkJibywAMPcPfdd+N2u33lXS4XxcXFAHi9XrZs2UJERERQY2usgtX39gMisldEvhSRf4vIJcEPVanyldf8QVZWFh06dADg9ddfL1V+1apV5Ofnk56ezoYNGxg4cCDfffcdbdu25d5772Xq1Kns2LGj0mVee+21LFy40DdccqdxIE07VBTb2LFjS90BnJmZyZVXXsnGjRv59ttvfZ+tIaoy0fj1vT0O6AncJiJlW0v+DzDAGBMPvAs8EexAlapIec0fJCUlMWnSJPr37+87pCoRHx/P1VdfzZVXXsljjz3GRRddxIYNG0hISKBv374sW7aM2bMrb2//2WefZdu2bcTHx9OzZ08WLVoEwE033cSKFSsqPRlcUWyPPvoomZmZxMXFkZCQwPr162ndujUvvvgiEydOJCEhgcmTJ5/n2qobVZ4MFpEhQJIx5jp7+FcAxpg/VVC+L/D/jDHDKqtXTwY3Dg2tmYhATtYqS203E1Fe39sdKin/X8CH1Q1EKdV4BfVksIjcCQwARlYwfRowDayWyZSqbbV54nX+/Pm+JjpLTJo0iblz5/7/9u4+Rq6qjOP499mXdvtC+rK2gGlQKqEUQVEWNUaLWogtBgpaQzFGm5iYiiaikYg2JpioEV9i/EupgigEUBRDE6MiokVNRJYGcLtVaS00paUsy5uUsO3OPP5xz525MzvbnZ29Z2a28/skk5k599x7n549febMzJ1zmhZDu6gn0dS19raZXQRsBS5095rTuLv7NmAbJG+dph2tyCyydevWjkwqteS19vZbgBuBy9z9mfzDFJHZLK+1t78NLATuMrNHzGz7JIeTE1Crri6XePL+m+a19vZFuUYls0ZfXx+jo6P09/dPuBBOZid3Z3R0tKGLESejK4NlRlasWMGBAwcYGRlpdSiSo76+vtJSvnlQopEZ6e3trfgdkUgtmiZCRKJTohGR6JRoRCQ6JRoRiU6JRkSiU6IRkeiUaEQkOiUaEYlOiUZEolOiEZHolGhEJDolGhGJTolGRKJTohGR6JRoRCQ6JRoRiU6JRkSiy2vt7TVmttPMxs1sY/5hishsltfa2/uBzcDteQcoIrNfPXMGvw3Y4+7/BTCzO4ENwHBawd2fCNuKEWIUkVkuxtrbIiIVmvphsJl90swGzWxQy3OIdI56Ek1da2/Xw923ufuAuw8sW7askUOIyCyUy9rbIiLHk8va22Z2gZkdAD4M3Ghmu2IGLSKzS15rbz9E8pZKRGQCXRksItEp0YhIdEo0IhKdEo2IRKdEIyLRKdGISHRKNCISnRKNiESnRCMi0SnRiEh0SjQiEp0SjYhEp0QjItEp0YhIdEo0IhKdEo2IRKdEIyLR1TXDXisVik7Rk5s7FfdFBw/32Tpe9byYqeOlsnKd544c5b7hwwCMh/ONF5xC0RkvpvdFCkXnaME5Ol5gbLzI2LEiY+MFXj1W5MJVy1h/zinJ+YGR/43xtd8MM7+3m4V9PXz+4lXsHXmZ544cpafbcIfxQjhXsUihCMVwvqTMKYRzlm5OqaxYJNnPy/sVisVSvADzervpXziHT7/3DPp6u3HPtmz5SbY8fThecG77+5O89OoxXt+/gPXnnjKhkmf29/DMJ2zzyrOVtpdPahhdXUa3GV1d0G1Gd1e5rLvL6CrdV4aRnrt03kmeZ/sKpedk+kwSUUU/m6xTHs90d8o2vkH6zzMzDDBLNxmW2Z7UtVCXULf6ebmy1Th2Wnfh3B5WLj9pmoFPj7k31JwzNjAw4IODg1PWG/rGGlaPPRY1lm7Lrw3GPRkk9rTZWnoFt6krZdRqk+I0j1GPrhzbXhqza865vPHLf62rrpk97O4D0z1HXSMaM1sHfB/oBn7s7t+s2j4X+BlwPjAKXJmuXjlTY6s/xM7nB0J2Lmf5NINXZH3LbrfSK4RNyP6W2QeWPvlblrwy83APnns1hBjmPTvE4oM7StteXHw2i14YnnzniJ5esY5XF60Eyu1QzaqK57x8kFOe+HXp+cGzNlPsPYnq3StfYSccteaxq+s55dFJdoRRhNIINB2NeI1jlF/1K4MJZy+/mqf9o1TFMo8rRwS14qyXVf+Dp+AYFWO8qlEhpAOfzOjQaw+e0noT982er3wCBxYsOW1a8TZiyhFNWHv7P8DFJKtUPgRc5e7DmTpXA29y9y1mtgm4wt2vPN5x6x3RiEj7aHREU8+HwaW1t939KJCuvZ21AfhpePxLYK1NN62LyAkrr7W3S3XCOlAvAv3VB9KSuCKdqalfb2tJXJHOlNfa26U6ZtYDLCL5UFhEJLe1t7cDHw+PNwL3e6u+NxeRtjPl19vuPm5m6drb3cDN6drbwKC7bwduAm41sz3AcyTJSEQEaOEFe2Y2AjzZkpMnXgM828LzT0XxNa6dY4PZHd/r3H3aH7C2LNG0mpkNNnI9QLMovsa1c2zQmfHpR5UiEp0SjYhE18mJZlurA5iC4mtcO8cGHRhfx35GIyLN08kjGhFpkhMi0ZjZzWb2jJkNHafOe8zsETPbZWY7QtmqUJbeXjKza8K2683sqcy2S5odXyj/XCgbMrM7zKwvlJ9uZg+a2R4z+3m4mLJdYrvFzPZl2u68RmLLIb7Phth2pX/XUL7UzP5gZo+H+yVtFl/T+p6ZXZs5z5CZFcxsadi2zsz+HfrYdZl9pt/3vDTr3Oy9AWuAtwJDk2xfDAwDp4Xny2vU6QaeJrlOAOB64AutjI/kx6r7gHnh+S+AzZnHm8LjHwKfaqPYbgE2trjtzgGGgPkkF6beB5wRtn0LuC48vg64oc3ia1rfq6p7KclV/en/h73ASmAO8ChwdqN974QY0bj7AyRXJE/mI8Dd7r4/1H+mRp21wF53z/0iwhnG1wPMC78hmw8cDFNwvI9kSg5Ipui4vB1iaySGSPGtBh5091c8mVFgB/DBsC07rUnDbRcxvtzUEV/WVcAd4XHN6WEa7XsnRKKpw5nAEjP7s5k9bGYfq1FnE+VGTn3GzB4Lw8+Gh9eNxufuTwHfAfYDh4AX3f1ekik4XggdFGpP3dGq2FJfD233PUtmYIxlsr/tEPBuM+s3s/nAJZR/HHyyux8Kj58GTm6z+KB5fQ+AEMM64FehaLLpYRrqe52SaHpIphn9APB+4Ctmdma6MbzHvAy4K7PPD4A3AOeR/Ef6brPjCx1sA3A68FpggZl9NGIcecX2JeAs4AJgKfDFZsfn7ruBG4B7gd8BjwCF6p09Gf/H/Oq1kfia2fdSlwJ/c/d6Rz/T0imJ5gDwe3c/4u7PAg8Ab85sXw/sdPfDaYG7H3b3grsXgR+RDCWbHd9FwD53H3H3Y8DdwDtJpuBYHN6yQO2pO1oVG+5+yBNjwE9oTdvh7je5+/nuvgZ4nmRKWoDDZnYqQLiv9Va6ZfE1ue+lqkf0k00P01Df65REcw/wLjPrCUPEtwO7M9uz702BUgdMXUEy1G12fPuBd5jZ/PDeeC2wO7wK/4lkSg5Ipui4px1ig3LbhfLLaU3bYWbLw/1pJJ9/3B72yU5rErPtGoqvyX0PM1sEXEhlO9ScHqbhvpfHJ9utvpEkiUPAMZJXkE8AW4AtmTrXknz6PwRckylfQJKlF1Ud81bgn8BjJB3z1BbF91XgX6H8VmBuKF8J/APYQ/KWb24bxXZ/aLsh4DZgYYva7i+h/FFgbaa8H/gj8DjJtz1L2yy+Zve9zcCdNfa9hGSUtRfYmimfdt/TlcEiEl2nvHUSkRZSohGR6JRoRCQ6JRoRiU6JRkSiU6IRkeiUaEQkOiUaEYnu/9g+GfcbqLKzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fa22be05ed0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<keras.engine.training.Model at 0x7fa28725a190>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train(model=model, data=((train_tensors, train_labels), (valid_tensors, valid_labels)), \n",
    "      lr=0.001, lr_decay=0.9, lam_recon=0.392, batch_size=32, shift_fraction=0.1, epochs=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights('saved_models/CapsNet.best.from_scratch.hdf5')\n",
    "prediction = eval_model.predict(test_tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.614100 %\n",
      "Recall: 0.599095 %\n",
      "Fscore: 0.611039 %\n"
     ]
    }
   ],
   "source": [
    "threshold = 0.5\n",
    "beta = 0.5\n",
    "\n",
    "pre = K.eval(precision_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction[0])))\n",
    "rec = K.eval(recall_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction[0])))\n",
    "fsc = K.eval(fbeta_score_threshold(beta = beta, threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction[0])))\n",
    "\n",
    "print (\"Precision: %f %%\\nRecall: %f %%\\nFscore: %f %%\"% (pre, rec, fsc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.53556561"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K.eval(binary_accuracy(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[ 0.47410309,  0.56007147],\n",
       "        [ 0.50858134,  0.52763808],\n",
       "        [ 0.54589927,  0.55975068],\n",
       "        ..., \n",
       "        [ 0.70406932,  0.33741596],\n",
       "        [ 0.44626307,  0.58754826],\n",
       "        [ 0.49341762,  0.46686408]], dtype=float32), array([[[[ 0.50309771],\n",
       "          [ 0.50112057],\n",
       "          [ 0.5002538 ],\n",
       "          ..., \n",
       "          [ 0.50152695],\n",
       "          [ 0.50125098],\n",
       "          [ 0.5023585 ]],\n",
       " \n",
       "         [[ 0.49969226],\n",
       "          [ 0.5027166 ],\n",
       "          [ 0.4964819 ],\n",
       "          ..., \n",
       "          [ 0.50036359],\n",
       "          [ 0.49782547],\n",
       "          [ 0.50023651]],\n",
       " \n",
       "         [[ 0.49756637],\n",
       "          [ 0.5003103 ],\n",
       "          [ 0.50059479],\n",
       "          ..., \n",
       "          [ 0.49671668],\n",
       "          [ 0.50313598],\n",
       "          [ 0.49973908]],\n",
       " \n",
       "         ..., \n",
       "         [[ 0.49682194],\n",
       "          [ 0.50349993],\n",
       "          [ 0.50171423],\n",
       "          ..., \n",
       "          [ 0.50015706],\n",
       "          [ 0.50112081],\n",
       "          [ 0.50063396]],\n",
       " \n",
       "         [[ 0.4977133 ],\n",
       "          [ 0.49982035],\n",
       "          [ 0.50248063],\n",
       "          ..., \n",
       "          [ 0.50077718],\n",
       "          [ 0.50034511],\n",
       "          [ 0.50067711]],\n",
       " \n",
       "         [[ 0.49837404],\n",
       "          [ 0.50260931],\n",
       "          [ 0.49941072],\n",
       "          ..., \n",
       "          [ 0.49900562],\n",
       "          [ 0.50133705],\n",
       "          [ 0.49597567]]],\n",
       " \n",
       " \n",
       "        [[[ 0.50297087],\n",
       "          [ 0.5010711 ],\n",
       "          [ 0.50028545],\n",
       "          ..., \n",
       "          [ 0.50140631],\n",
       "          [ 0.50116962],\n",
       "          [ 0.50223225]],\n",
       " \n",
       "         [[ 0.49970216],\n",
       "          [ 0.50267249],\n",
       "          [ 0.49681318],\n",
       "          ..., \n",
       "          [ 0.50039089],\n",
       "          [ 0.49798495],\n",
       "          [ 0.50024897]],\n",
       " \n",
       "         [[ 0.49782321],\n",
       "          [ 0.50037122],\n",
       "          [ 0.50057453],\n",
       "          ..., \n",
       "          [ 0.49695575],\n",
       "          [ 0.50292414],\n",
       "          [ 0.49968037]],\n",
       " \n",
       "         ..., \n",
       "         [[ 0.49702901],\n",
       "          [ 0.50333041],\n",
       "          [ 0.50164652],\n",
       "          ..., \n",
       "          [ 0.50013524],\n",
       "          [ 0.50098395],\n",
       "          [ 0.50057793]],\n",
       " \n",
       "         [[ 0.49788007],\n",
       "          [ 0.49977717],\n",
       "          [ 0.50238633],\n",
       "          ..., \n",
       "          [ 0.50073546],\n",
       "          [ 0.50033033],\n",
       "          [ 0.50065261]],\n",
       " \n",
       "         [[ 0.49845672],\n",
       "          [ 0.50245953],\n",
       "          [ 0.49945796],\n",
       "          ..., \n",
       "          [ 0.49908295],\n",
       "          [ 0.50129837],\n",
       "          [ 0.49622327]]],\n",
       " \n",
       " \n",
       "        [[[ 0.5031004 ],\n",
       "          [ 0.50118017],\n",
       "          [ 0.50021112],\n",
       "          ..., \n",
       "          [ 0.50147212],\n",
       "          [ 0.50116909],\n",
       "          [ 0.50232601]],\n",
       " \n",
       "         [[ 0.49967918],\n",
       "          [ 0.50274622],\n",
       "          [ 0.4964799 ],\n",
       "          ..., \n",
       "          [ 0.50036114],\n",
       "          [ 0.49782032],\n",
       "          [ 0.50030607]],\n",
       " \n",
       "         [[ 0.49767661],\n",
       "          [ 0.50038308],\n",
       "          [ 0.50057667],\n",
       "          ..., \n",
       "          [ 0.49678221],\n",
       "          [ 0.50305486],\n",
       "          [ 0.49979636]],\n",
       " \n",
       "         ..., \n",
       "         [[ 0.49685448],\n",
       "          [ 0.50355422],\n",
       "          [ 0.50179178],\n",
       "          ..., \n",
       "          [ 0.50008303],\n",
       "          [ 0.50107443],\n",
       "          [ 0.5006426 ]],\n",
       " \n",
       "         [[ 0.49775872],\n",
       "          [ 0.49978682],\n",
       "          [ 0.5024938 ],\n",
       "          ..., \n",
       "          [ 0.50076693],\n",
       "          [ 0.50037873],\n",
       "          [ 0.5006609 ]],\n",
       " \n",
       "         [[ 0.49837133],\n",
       "          [ 0.50259185],\n",
       "          [ 0.49943131],\n",
       "          ..., \n",
       "          [ 0.49898604],\n",
       "          [ 0.50137013],\n",
       "          [ 0.49598163]]],\n",
       " \n",
       " \n",
       "        ..., \n",
       "        [[[ 0.50132859],\n",
       "          [ 0.50077802],\n",
       "          [ 0.50201225],\n",
       "          ..., \n",
       "          [ 0.49914995],\n",
       "          [ 0.50211668],\n",
       "          [ 0.50164026]],\n",
       " \n",
       "         [[ 0.50170207],\n",
       "          [ 0.50146788],\n",
       "          [ 0.49131441],\n",
       "          ..., \n",
       "          [ 0.50106287],\n",
       "          [ 0.49963859],\n",
       "          [ 0.49795586]],\n",
       " \n",
       "         [[ 0.4980112 ],\n",
       "          [ 0.49591315],\n",
       "          [ 0.49441251],\n",
       "          ..., \n",
       "          [ 0.50096703],\n",
       "          [ 0.50041258],\n",
       "          [ 0.50084758]],\n",
       " \n",
       "         ..., \n",
       "         [[ 0.49680889],\n",
       "          [ 0.50274843],\n",
       "          [ 0.49734154],\n",
       "          ..., \n",
       "          [ 0.50293809],\n",
       "          [ 0.49873814],\n",
       "          [ 0.49933937]],\n",
       " \n",
       "         [[ 0.50134218],\n",
       "          [ 0.50294209],\n",
       "          [ 0.49944434],\n",
       "          ..., \n",
       "          [ 0.49585247],\n",
       "          [ 0.49929646],\n",
       "          [ 0.50280833]],\n",
       " \n",
       "         [[ 0.49598333],\n",
       "          [ 0.50808591],\n",
       "          [ 0.49838376],\n",
       "          ..., \n",
       "          [ 0.50053132],\n",
       "          [ 0.50304258],\n",
       "          [ 0.49899873]]],\n",
       " \n",
       " \n",
       "        [[[ 0.50324202],\n",
       "          [ 0.50124478],\n",
       "          [ 0.50022024],\n",
       "          ..., \n",
       "          [ 0.50157017],\n",
       "          [ 0.50122947],\n",
       "          [ 0.50246215]],\n",
       " \n",
       "         [[ 0.49963072],\n",
       "          [ 0.50292343],\n",
       "          [ 0.49630708],\n",
       "          ..., \n",
       "          [ 0.50041968],\n",
       "          [ 0.49773431],\n",
       "          [ 0.50031084]],\n",
       " \n",
       "         [[ 0.4975453 ],\n",
       "          [ 0.50040787],\n",
       "          [ 0.50059742],\n",
       "          ..., \n",
       "          [ 0.49663675],\n",
       "          [ 0.50321668],\n",
       "          [ 0.49982911]],\n",
       " \n",
       "         ..., \n",
       "         [[ 0.49669638],\n",
       "          [ 0.50372249],\n",
       "          [ 0.50191563],\n",
       "          ..., \n",
       "          [ 0.50006706],\n",
       "          [ 0.50113738],\n",
       "          [ 0.50065249]],\n",
       " \n",
       "         [[ 0.49768475],\n",
       "          [ 0.49977094],\n",
       "          [ 0.50264305],\n",
       "          ..., \n",
       "          [ 0.50077158],\n",
       "          [ 0.50041401],\n",
       "          [ 0.50071961]],\n",
       " \n",
       "         [[ 0.49832946],\n",
       "          [ 0.50270629],\n",
       "          [ 0.49941632],\n",
       "          ..., \n",
       "          [ 0.49890763],\n",
       "          [ 0.50142556],\n",
       "          [ 0.49578637]]],\n",
       " \n",
       " \n",
       "        [[[ 0.50093186],\n",
       "          [ 0.50050974],\n",
       "          [ 0.50136507],\n",
       "          ..., \n",
       "          [ 0.4994275 ],\n",
       "          [ 0.50148839],\n",
       "          [ 0.50121099]],\n",
       " \n",
       "         [[ 0.50113606],\n",
       "          [ 0.50101948],\n",
       "          [ 0.49394822],\n",
       "          ..., \n",
       "          [ 0.50074559],\n",
       "          [ 0.49967906],\n",
       "          [ 0.49860048]],\n",
       " \n",
       "         [[ 0.49858364],\n",
       "          [ 0.49717045],\n",
       "          [ 0.49608582],\n",
       "          ..., \n",
       "          [ 0.50068372],\n",
       "          [ 0.50024158],\n",
       "          [ 0.50051838]],\n",
       " \n",
       "         ..., \n",
       "         [[ 0.49777076],\n",
       "          [ 0.50189704],\n",
       "          [ 0.49827281],\n",
       "          ..., \n",
       "          [ 0.50202733],\n",
       "          [ 0.49909422],\n",
       "          [ 0.49951661]],\n",
       " \n",
       "         [[ 0.50094479],\n",
       "          [ 0.50218296],\n",
       "          [ 0.49955514],\n",
       "          ..., \n",
       "          [ 0.49716699],\n",
       "          [ 0.49947473],\n",
       "          [ 0.50195897]],\n",
       " \n",
       "         [[ 0.49717793],\n",
       "          [ 0.50562781],\n",
       "          [ 0.49891198],\n",
       "          ..., \n",
       "          [ 0.50034142],\n",
       "          [ 0.50210541],\n",
       "          [ 0.49931785]]]], dtype=float32)]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}